hubconf.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161
  1. # Copyright 2020 The HuggingFace Team. All rights reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os
  15. import sys
  16. SRC_DIR = os.path.join(os.path.dirname(__file__), "src")
  17. sys.path.append(SRC_DIR)
  18. from transformers import (
  19. AutoConfig,
  20. AutoModel,
  21. AutoModelForCausalLM,
  22. AutoModelForMaskedLM,
  23. AutoModelForQuestionAnswering,
  24. AutoModelForSequenceClassification,
  25. AutoTokenizer,
  26. add_start_docstrings,
  27. )
  28. dependencies = ["torch", "numpy", "tokenizers", "filelock", "requests", "tqdm", "regex", "sentencepiece", "sacremoses", "importlib_metadata", "huggingface_hub"]
  29. @add_start_docstrings(AutoConfig.__doc__)
  30. def config(*args, **kwargs):
  31. r"""
  32. # Using torch.hub !
  33. import torch
  34. config = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased') # Download configuration from huggingface.co and cache.
  35. config = torch.hub.load('huggingface/transformers', 'config', './test/bert_saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')`
  36. config = torch.hub.load('huggingface/transformers', 'config', './test/bert_saved_model/my_configuration.json')
  37. config = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased', output_attentions=True, foo=False)
  38. assert config.output_attentions == True
  39. config, unused_kwargs = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased', output_attentions=True, foo=False, return_unused_kwargs=True)
  40. assert config.output_attentions == True
  41. assert unused_kwargs == {'foo': False}
  42. """
  43. return AutoConfig.from_pretrained(*args, **kwargs)
  44. @add_start_docstrings(AutoTokenizer.__doc__)
  45. def tokenizer(*args, **kwargs):
  46. r"""
  47. # Using torch.hub !
  48. import torch
  49. tokenizer = torch.hub.load('huggingface/transformers', 'tokenizer', 'bert-base-uncased') # Download vocabulary from huggingface.co and cache.
  50. tokenizer = torch.hub.load('huggingface/transformers', 'tokenizer', './test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`
  51. """
  52. return AutoTokenizer.from_pretrained(*args, **kwargs)
  53. @add_start_docstrings(AutoModel.__doc__)
  54. def model(*args, **kwargs):
  55. r"""
  56. # Using torch.hub !
  57. import torch
  58. model = torch.hub.load('huggingface/transformers', 'model', 'bert-base-uncased') # Download model and configuration from huggingface.co and cache.
  59. model = torch.hub.load('huggingface/transformers', 'model', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
  60. model = torch.hub.load('huggingface/transformers', 'model', 'bert-base-uncased', output_attentions=True) # Update configuration during loading
  61. assert model.config.output_attentions == True
  62. # Loading from a TF checkpoint file instead of a PyTorch model (slower)
  63. config = AutoConfig.from_pretrained('./tf_model/bert_tf_model_config.json')
  64. model = torch.hub.load('huggingface/transformers', 'model', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
  65. """
  66. return AutoModel.from_pretrained(*args, **kwargs)
  67. @add_start_docstrings(AutoModelForCausalLM.__doc__)
  68. def modelForCausalLM(*args, **kwargs):
  69. r"""
  70. # Using torch.hub !
  71. import torch
  72. model = torch.hub.load('huggingface/transformers', 'modelForCausalLM', 'gpt2') # Download model and configuration from huggingface.co and cache.
  73. model = torch.hub.load('huggingface/transformers', 'modelForCausalLM', './test/saved_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
  74. model = torch.hub.load('huggingface/transformers', 'modelForCausalLM', 'gpt2', output_attentions=True) # Update configuration during loading
  75. assert model.config.output_attentions == True
  76. # Loading from a TF checkpoint file instead of a PyTorch model (slower)
  77. config = AutoConfig.from_pretrained('./tf_model/gpt_tf_model_config.json')
  78. model = torch.hub.load('huggingface/transformers', 'modelForCausalLM', './tf_model/gpt_tf_checkpoint.ckpt.index', from_tf=True, config=config)
  79. """
  80. return AutoModelForCausalLM.from_pretrained(*args, **kwargs)
  81. @add_start_docstrings(AutoModelForMaskedLM.__doc__)
  82. def modelForMaskedLM(*args, **kwargs):
  83. r"""
  84. # Using torch.hub !
  85. import torch
  86. model = torch.hub.load('huggingface/transformers', 'modelForMaskedLM', 'bert-base-uncased') # Download model and configuration from huggingface.co and cache.
  87. model = torch.hub.load('huggingface/transformers', 'modelForMaskedLM', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
  88. model = torch.hub.load('huggingface/transformers', 'modelForMaskedLM', 'bert-base-uncased', output_attentions=True) # Update configuration during loading
  89. assert model.config.output_attentions == True
  90. # Loading from a TF checkpoint file instead of a PyTorch model (slower)
  91. config = AutoConfig.from_pretrained('./tf_model/bert_tf_model_config.json')
  92. model = torch.hub.load('huggingface/transformers', 'modelForMaskedLM', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
  93. """
  94. return AutoModelForMaskedLM.from_pretrained(*args, **kwargs)
  95. @add_start_docstrings(AutoModelForSequenceClassification.__doc__)
  96. def modelForSequenceClassification(*args, **kwargs):
  97. r"""
  98. # Using torch.hub !
  99. import torch
  100. model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', 'bert-base-uncased') # Download model and configuration from huggingface.co and cache.
  101. model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
  102. model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', 'bert-base-uncased', output_attentions=True) # Update configuration during loading
  103. assert model.config.output_attentions == True
  104. # Loading from a TF checkpoint file instead of a PyTorch model (slower)
  105. config = AutoConfig.from_pretrained('./tf_model/bert_tf_model_config.json')
  106. model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
  107. """
  108. return AutoModelForSequenceClassification.from_pretrained(*args, **kwargs)
  109. @add_start_docstrings(AutoModelForQuestionAnswering.__doc__)
  110. def modelForQuestionAnswering(*args, **kwargs):
  111. r"""
  112. # Using torch.hub !
  113. import torch
  114. model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', 'bert-base-uncased') # Download model and configuration from huggingface.co and cache.
  115. model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
  116. model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', 'bert-base-uncased', output_attentions=True) # Update configuration during loading
  117. assert model.config.output_attentions == True
  118. # Loading from a TF checkpoint file instead of a PyTorch model (slower)
  119. config = AutoConfig.from_pretrained('./tf_model/bert_tf_model_config.json')
  120. model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
  121. """
  122. return AutoModelForQuestionAnswering.from_pretrained(*args, **kwargs)