Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support model revision and tokenizer revision in huggingface server #3558

Merged
merged 4 commits into from
Apr 3, 2024
Merged

Support model revision and tokenizer revision in huggingface server #3558

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0c05917
support model revision and tokenizer revision
lizzzcai Mar 29, 2024
2fcd641
point to specified commit in test case
lizzzcai Mar 30, 2024
761c0a2
Merge branch 'master' into support-revision-hfserver-0327
yuzisun Apr 3, 2024
c9dbd04
format code
lizzzcai Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'master' into support-revision-hfserver-0327 
Signed-off-by: Dan Sun <dsun20@bloomberg.net>
  • Loading branch information
@yuzisun
yuzisun authored Apr 3, 2024
commit 761c0a2fc12a451a3486f960b3270287b86f9139
55 changes: 36 additions & 19 deletions python/huggingfaceserver/huggingfaceserver/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,25 +29,42 @@ def list_of_strings(arg):

parser = argparse.ArgumentParser(parents=[kserve.model_server.parser])

parser.add_argument('--model_dir', required=False, default=None,
help='A local path to the model binary')
parser.add_argument('--model_id', required=False,
help='Huggingface model id')
parser.add_argument('--model_revision', required=False, default=None,
help='Huggingface model revision')
parser.add_argument('--tokenizer_revision', required=False, default=None,
help='Huggingface tokenizer revision')
parser.add_argument('--max_length', type=int, default=None,
help='max sequence length for the tokenizer')
parser.add_argument('--disable_lower_case', action='store_true',
help='do not use lower case for the tokenizer')
parser.add_argument('--disable_special_tokens', action='store_true',
help='the sequences will not be encoded with the special tokens relative to their model')
parser.add_argument('--tensor_input_names', type=list_of_strings, default=None,
help='the tensor input names passed to the model')
parser.add_argument('--task', required=False, help="The ML task name")
parser.add_argument('--disable_vllm', action='store_true', help="Do not use vllm as the default runtime")
parser.add_argument('--return_token_type_ids', action="store_true", help="Return token type ids")
parser.add_argument(
"--model_dir", required=False, default=None, help="A local path to the model binary"
)
parser.add_argument("--model_id", required=False, help="Huggingface model id")
parser.add_argument(
"--model_revision", required=False, default=None, help="Huggingface model revision"
)
parser.add_argument(
"--tokenizer_revision", required=False, default=None, help="Huggingface tokenizer revision"
)
parser.add_argument(
"--max_length", type=int, default=None, help="max sequence length for the tokenizer"
)
parser.add_argument(
"--disable_lower_case",
action="store_true",
help="do not use lower case for the tokenizer",
)
parser.add_argument(
"--disable_special_tokens",
action="store_true",
help="the sequences will not be encoded with the special tokens relative to their model",
)
parser.add_argument(
"--tensor_input_names",
type=list_of_strings,
default=None,
help="the tensor input names passed to the model",
)
parser.add_argument("--task", required=False, help="The ML task name")
parser.add_argument(
"--disable_vllm", action="store_true", help="Do not use vllm as the default runtime"
)
parser.add_argument(
"--return_token_type_ids", action="store_true", help="Return token type ids"
)

try:
from vllm.engine.arg_utils import AsyncEngineArgs
Expand Down
35 changes: 21 additions & 14 deletions python/huggingfaceserver/huggingfaceserver/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,14 +83,14 @@ def __init__(
self.model_dir = kwargs.get("model_dir", None)
if not self.model_id and not self.model_dir:
self.model_dir = "/mnt/models"
self.model_revision = kwargs.get('model_revision', None)
self.tokenizer_revision = kwargs.get('tokenizer_revision', None)
self.do_lower_case = not kwargs.get('disable_lower_case', False)
self.add_special_tokens = not kwargs.get('disable_special_tokens', False)
self.max_length = kwargs.get('max_length', None)
self.tensor_input_names = kwargs.get('tensor_input_names', None)
self.return_token_type_ids = kwargs.get('return_token_type_ids', None)
self.task = kwargs.get('task', None)
self.model_revision = kwargs.get("model_revision", None)
self.tokenizer_revision = kwargs.get("tokenizer_revision", None)
self.do_lower_case = not kwargs.get("disable_lower_case", False)
self.add_special_tokens = not kwargs.get("disable_special_tokens", False)
self.max_length = kwargs.get("max_length", None)
self.tensor_input_names = kwargs.get("tensor_input_names", None)
self.return_token_type_ids = kwargs.get("return_token_type_ids", None)
self.task = kwargs.get("task", None)
self.tokenizer = None
self.model = None
self.mapping = None
Expand Down Expand Up @@ -169,6 +169,7 @@ def load(self) -> bool:
revision=tokenizer_revision,
do_lower_case=self.do_lower_case,
device_map=self.device_map)

if not self.tokenizer.pad_token:
self.tokenizer.add_special_tokens({"pad_token": "[PAD]"})
logger.info(f"successfully loaded tokenizer for task: {self.task}")
Expand All @@ -177,22 +178,28 @@ def load(self) -> bool:
if not self.predictor_host:
if self.task == MLTask.sequence_classification.value:
self.model = AutoModelForSequenceClassification.from_pretrained(
model_id_or_path, revision=revision, device_map=self.device_map)
model_id_or_path, revision=revision, device_map=self.device_map
)
elif self.task == MLTask.question_answering.value:
self.model = AutoModelForQuestionAnswering.from_pretrained(
model_id_or_path, revision=revision, device_map=self.device_map)
model_id_or_path, revision=revision, device_map=self.device_map
)
elif self.task == MLTask.token_classification.value:
self.model = AutoModelForTokenClassification.from_pretrained(
model_id_or_path, revision=revision, device_map=self.device_map)
model_id_or_path, revision=revision, device_map=self.device_map
)
elif self.task == MLTask.fill_mask.value:
self.model = AutoModelForMaskedLM.from_pretrained(
model_id_or_path, revision=revision, device_map=self.device_map)
model_id_or_path, revision=revision, device_map=self.device_map
)
elif self.task == MLTask.text_generation.value:
self.model = AutoModelForCausalLM.from_pretrained(
model_id_or_path, revision=revision, device_map=self.device_map)
model_id_or_path, revision=revision, device_map=self.device_map
)
elif self.task == MLTask.text2text_generation.value:
self.model = AutoModelForSeq2SeqLM.from_pretrained(
model_id_or_path, revision=revision, device_map=self.device_map)
model_id_or_path, revision=revision, device_map=self.device_map
)
else:
raise ValueError(
f"Unsupported task {self.task}. Please check the supported `task` option."
Expand Down
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.