From b3cd77f1cc6bc9449f9ea1e2fc9a492d75e7f71d Mon Sep 17 00:00:00 2001 From: codingl2k1 <138426806+codingl2k1@users.noreply.github.com> Date: Sat, 19 Oct 2024 18:02:03 +0200 Subject: [PATCH 1/9] ENH: Support ChatTTS 0.2 (#2449) --- .github/workflows/python.yaml | 2 +- setup.cfg | 4 +- xinference/deploy/docker/requirements.txt | 2 +- xinference/deploy/docker/requirements_cpu.txt | 2 +- xinference/model/audio/chattts.py | 39 ++++++++++++------- xinference/model/audio/model_spec.json | 2 +- .../model/audio/model_spec_modelscope.json | 2 +- xinference/model/audio/tests/test_chattts.py | 14 ++++--- 8 files changed, 40 insertions(+), 27 deletions(-) diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index 3fedc5eea5..ecd92bd794 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -162,7 +162,7 @@ jobs: ${{ env.SELF_HOST_PYTHON }} -m pip install -U WeTextProcessing<1.0.4 ${{ env.SELF_HOST_PYTHON }} -m pip install -U librosa ${{ env.SELF_HOST_PYTHON }} -m pip install -U xxhash - ${{ env.SELF_HOST_PYTHON }} -m pip install -U "ChatTTS>0.1,<0.2" + ${{ env.SELF_HOST_PYTHON }} -m pip install -U "ChatTTS>=0.2" ${{ env.SELF_HOST_PYTHON }} -m pip install -U HyperPyYAML ${{ env.SELF_HOST_PYTHON }} -m pip uninstall -y matcha-tts ${{ env.SELF_HOST_PYTHON }} -m pip install -U onnxruntime-gpu==1.16.0; sys_platform == 'linux' diff --git a/setup.cfg b/setup.cfg index cd3a752de6..04867b9f4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -110,7 +110,7 @@ all = librosa # For ChatTTS xxhash # For ChatTTS torchaudio # For ChatTTS - ChatTTS>0.1,<0.2 + ChatTTS>=0.2 lightning>=2.0.0 # For CosyVoice, matcha hydra-core>=1.3.2 # For CosyVoice, matcha inflect # For CosyVoice, matcha @@ -185,7 +185,7 @@ audio = librosa xxhash torchaudio - ChatTTS>0.1,<0.2 + ChatTTS>=0.2 tiktoken # For CosyVoice, openai-whisper torch>=2.0.0 # For CosyVoice, matcha lightning>=2.0.0 # For CosyVoice, matcha diff --git a/xinference/deploy/docker/requirements.txt b/xinference/deploy/docker/requirements.txt index 6de778c5b9..89e301761a 100644 --- a/xinference/deploy/docker/requirements.txt +++ b/xinference/deploy/docker/requirements.txt @@ -49,7 +49,7 @@ nemo_text_processing<1.1.0 # 1.1.0 requires pynini==2.1.6.post1 WeTextProcessing<1.0.4 # 1.0.4 requires pynini==2.1.6 librosa # For ChatTTS torchaudio # For ChatTTS -ChatTTS>0.1,<0.2 +ChatTTS>=0.2 xxhash # For ChatTTS torch>=2.0.0 # For CosyVoice lightning>=2.0.0 # For CosyVoice, matcha diff --git a/xinference/deploy/docker/requirements_cpu.txt b/xinference/deploy/docker/requirements_cpu.txt index 50cd791995..c517a69717 100644 --- a/xinference/deploy/docker/requirements_cpu.txt +++ b/xinference/deploy/docker/requirements_cpu.txt @@ -46,7 +46,7 @@ nemo_text_processing<1.1.0 # 1.1.0 requires pynini==2.1.6.post1 WeTextProcessing<1.0.4 # 1.0.4 requires pynini==2.1.6 librosa # For ChatTTS torchaudio # For ChatTTS -ChatTTS>0.1,<0.2 +ChatTTS>=0.2 xxhash # For ChatTTS torch>=2.0.0 # For CosyVoice lightning>=2.0.0 # For CosyVoice, matcha diff --git a/xinference/model/audio/chattts.py b/xinference/model/audio/chattts.py index b636fe59b4..2a5f4ee7c9 100644 --- a/xinference/model/audio/chattts.py +++ b/xinference/model/audio/chattts.py @@ -54,7 +54,11 @@ def load(self): torch.set_float32_matmul_precision("high") self._model = ChatTTS.Chat() logger.info("Load ChatTTS model with kwargs: %s", self._kwargs) - self._model.load(source="custom", custom_path=self._model_path, **self._kwargs) + ok = self._model.load( + source="custom", custom_path=self._model_path, **self._kwargs + ) + if not ok: + raise Exception(f"The ChatTTS model is not correct: {self._model_path}") def speech( self, @@ -114,16 +118,15 @@ def _generator(): last_pos = 0 with writer.open(): for it in iter: - for itt in it: - for chunk in itt: - chunk = np.array([chunk]).transpose() - writer.write_audio_chunk(i, torch.from_numpy(chunk)) - new_last_pos = out.tell() - if new_last_pos != last_pos: - out.seek(last_pos) - encoded_bytes = out.read() - yield encoded_bytes - last_pos = new_last_pos + for chunk in it: + chunk = np.array([chunk]).transpose() + writer.write_audio_chunk(i, torch.from_numpy(chunk)) + new_last_pos = out.tell() + if new_last_pos != last_pos: + out.seek(last_pos) + encoded_bytes = out.read() + yield encoded_bytes + last_pos = new_last_pos return _generator() else: @@ -131,7 +134,15 @@ def _generator(): # Save the generated audio with BytesIO() as out: - torchaudio.save( - out, torch.from_numpy(wavs[0]), 24000, format=response_format - ) + try: + torchaudio.save( + out, + torch.from_numpy(wavs[0]).unsqueeze(0), + 24000, + format=response_format, + ) + except: + torchaudio.save( + out, torch.from_numpy(wavs[0]), 24000, format=response_format + ) return out.getvalue() diff --git a/xinference/model/audio/model_spec.json b/xinference/model/audio/model_spec.json index bf51b3da3a..e0328dd375 100644 --- a/xinference/model/audio/model_spec.json +++ b/xinference/model/audio/model_spec.json @@ -127,7 +127,7 @@ "model_name": "ChatTTS", "model_family": "ChatTTS", "model_id": "2Noise/ChatTTS", - "model_revision": "ce5913842aebd78e4a01a02d47244b8d62ac4ee3", + "model_revision": "3b34118f6d25850440b8901cef3e71c6ef8619c8", "model_ability": "text-to-audio", "multilingual": true }, diff --git a/xinference/model/audio/model_spec_modelscope.json b/xinference/model/audio/model_spec_modelscope.json index e3f46f84bc..e47f1f8e3a 100644 --- a/xinference/model/audio/model_spec_modelscope.json +++ b/xinference/model/audio/model_spec_modelscope.json @@ -42,7 +42,7 @@ "model_name": "ChatTTS", "model_family": "ChatTTS", "model_hub": "modelscope", - "model_id": "pzc163/chatTTS", + "model_id": "AI-ModelScope/ChatTTS", "model_revision": "master", "model_ability": "text-to-audio", "multilingual": true diff --git a/xinference/model/audio/tests/test_chattts.py b/xinference/model/audio/tests/test_chattts.py index 93739d409e..cadd732351 100644 --- a/xinference/model/audio/tests/test_chattts.py +++ b/xinference/model/audio/tests/test_chattts.py @@ -46,12 +46,14 @@ def test_chattts(setup): response = model.speech(input_string, stream=True) assert inspect.isgenerator(response) - i = 0 - for chunk in response: - i += 1 - assert type(chunk) is bytes - assert len(chunk) > 0 - assert i > 5 + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=True) as f: + i = 0 + for chunk in response: + f.write(chunk) + i += 1 + assert type(chunk) is bytes + assert len(chunk) > 0 + assert i > 5 # Test openai API import openai From 043b673dbd93f1cd5c8d6b0bb481f8d605cd24d2 Mon Sep 17 00:00:00 2001 From: Adam Ning Date: Mon, 21 Oct 2024 11:00:43 +0800 Subject: [PATCH 2/9] Remove duplicated call of model_install (#2457) --- xinference/deploy/supervisor.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/xinference/deploy/supervisor.py b/xinference/deploy/supervisor.py index aac1e78d3e..ed12a9f7c2 100644 --- a/xinference/deploy/supervisor.py +++ b/xinference/deploy/supervisor.py @@ -31,10 +31,6 @@ logger = logging.getLogger(__name__) -from ..model import _install as install_model - -install_model() - async def _start_supervisor(address: str, logging_conf: Optional[Dict] = None): logging.config.dictConfig(logging_conf) # type: ignore From ef21a716508b9c5cb4cdc291da0e57046a25f309 Mon Sep 17 00:00:00 2001 From: Xuye Qin Date: Mon, 21 Oct 2024 14:35:52 +0800 Subject: [PATCH 3/9] DOC: update enterprise doc links (#2461) --- README.md | 2 +- README_zh_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cd622e7b07..d63cbb42ff 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

Xinference Cloud · - Xinference Enterprise · + Xinference Enterprise · Self-hosting · Documentation

diff --git a/README_zh_CN.md b/README_zh_CN.md index 2ace2e1bcc..2df28e2632 100644 --- a/README_zh_CN.md +++ b/README_zh_CN.md @@ -5,7 +5,7 @@

Xinference 云服务 · - Xinference 企业版 · + Xinference 企业版 · 自托管 · 文档

From 48a07e8af3d015908c8a55f91c7baea3afbbb809 Mon Sep 17 00:00:00 2001 From: Adam Ning Date: Wed, 23 Oct 2024 00:14:17 +0800 Subject: [PATCH 4/9] FEAT: Add support for Qwen/Qwen2.5-Coder-7B-Instruct gptq format (#2408) --- xinference/model/llm/llm_family.json | 9 +++++++++ xinference/model/llm/llm_family_modelscope.json | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/xinference/model/llm/llm_family.json b/xinference/model/llm/llm_family.json index ac09f0ea5b..abdcfd4f62 100644 --- a/xinference/model/llm/llm_family.json +++ b/xinference/model/llm/llm_family.json @@ -8176,6 +8176,15 @@ ], "model_id": "Qwen/Qwen2.5-Coder-7B-Instruct" }, + { + "model_format": "gptq", + "model_size_in_billions": "7", + "quantizations": [ + "Int4", + "Int8" + ], + "model_id": "Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-{quantization}" + }, { "model_format": "ggufv2", "model_size_in_billions": "1_5", diff --git a/xinference/model/llm/llm_family_modelscope.json b/xinference/model/llm/llm_family_modelscope.json index acf6e09fd3..7a91e561e6 100644 --- a/xinference/model/llm/llm_family_modelscope.json +++ b/xinference/model/llm/llm_family_modelscope.json @@ -5880,6 +5880,17 @@ "model_revision": "master", "model_hub": "modelscope" }, + { + "model_format": "gptq", + "model_size_in_billions": 7, + "quantizations": [ + "Int4", + "Int8" + ], + "model_id": "qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-{quantization}", + "model_revision": "master", + "model_hub": "modelscope" + }, { "model_format": "ggufv2", "model_size_in_billions": "1_5", From 0d10b38daa60f299b76a986a6ebcc581903c15ee Mon Sep 17 00:00:00 2001 From: codingl2k1 <138426806+codingl2k1@users.noreply.github.com> Date: Thu, 24 Oct 2024 07:57:03 +0200 Subject: [PATCH 5/9] FEAT: Support GOT-OCR2_0 (#2458) Co-authored-by: qinxuye --- .github/workflows/python.yaml | 6 +- setup.cfg | 14 +++- xinference/api/restful_api.py | 48 ++++++++++++ xinference/client/restful/restful_client.py | 19 +++++ xinference/core/model.py | 19 +++++ xinference/deploy/docker/requirements.txt | 7 +- xinference/deploy/docker/requirements_cpu.txt | 5 +- xinference/model/image/core.py | 37 ++++++++- xinference/model/image/model_spec.json | 9 +++ .../model/image/model_spec_modelscope.json | 10 +++ xinference/model/image/ocr/__init__.py | 13 ++++ xinference/model/image/ocr/got_ocr2.py | 76 +++++++++++++++++++ xinference/model/image/tests/test_got_ocr2.py | 41 ++++++++++ 13 files changed, 293 insertions(+), 11 deletions(-) create mode 100644 xinference/model/image/ocr/__init__.py create mode 100644 xinference/model/image/ocr/got_ocr2.py create mode 100644 xinference/model/image/tests/test_got_ocr2.py diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml index ecd92bd794..81a16122c5 100644 --- a/.github/workflows/python.yaml +++ b/.github/workflows/python.yaml @@ -175,9 +175,13 @@ jobs: ${{ env.SELF_HOST_PYTHON }} -m pip uninstall -y opencc ${{ env.SELF_HOST_PYTHON }} -m pip uninstall -y "faster_whisper" ${{ env.SELF_HOST_PYTHON }} -m pip install -U accelerate + ${{ env.SELF_HOST_PYTHON }} -m pip install -U verovio ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \ -W ignore::PendingDeprecationWarning \ --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/image/tests/test_stable_diffusion.py && \ + ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \ + -W ignore::PendingDeprecationWarning \ + --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/image/tests/test_got_ocr2.py && \ ${{ env.SELF_HOST_PYTHON }} -m pytest --timeout=1500 \ -W ignore::PendingDeprecationWarning \ --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/model/audio/tests/test_whisper.py && \ @@ -203,6 +207,6 @@ jobs: --cov-config=setup.cfg --cov-report=xml --cov=xinference xinference/client/tests/test_client.py pytest --timeout=1500 \ -W ignore::PendingDeprecationWarning \ - --cov-config=setup.cfg --cov-report=xml --cov=xinference --ignore xinference/client/tests/test_client.py --ignore xinference/model/image/tests/test_stable_diffusion.py --ignore xinference/model/audio/tests xinference + --cov-config=setup.cfg --cov-report=xml --cov=xinference --ignore xinference/client/tests/test_client.py --ignore xinference/model/image/tests/test_stable_diffusion.py --ignore xinference/model/image/tests/test_got_ocr2.py --ignore xinference/model/audio/tests xinference fi working-directory: . diff --git a/setup.cfg b/setup.cfg index 04867b9f4d..ccde851616 100644 --- a/setup.cfg +++ b/setup.cfg @@ -80,13 +80,13 @@ all = llama-cpp-python>=0.2.25,!=0.2.58 transformers>=4.43.2 torch>=2.0.0 # >=2.0 For CosyVoice - accelerate>=0.27.2 + accelerate>=0.28.0 sentencepiece transformers_stream_generator bitsandbytes protobuf einops - tiktoken + tiktoken>=0.6.0 sentence-transformers>=3.1.0 vllm>=0.2.6 ; sys_platform=='linux' diffusers>=0.30.0 @@ -131,6 +131,8 @@ all = qwen-vl-utils # For qwen2-vl datamodel_code_generator # for minicpm-4B jsonschema # for minicpm-4B + verovio>=4.3.1 # For got_ocr2 + accelerate>=0.28.0 # For got_ocr2 intel = torch==2.1.0a0 intel_extension_for_pytorch==2.1.10+xpu @@ -139,7 +141,7 @@ llama_cpp = transformers = transformers>=4.43.2 torch - accelerate>=0.27.2 + accelerate>=0.28.0 sentencepiece transformers_stream_generator bitsandbytes @@ -174,6 +176,12 @@ image = diffusers>=0.30.0 # fix conflict with matcha-tts controlnet_aux deepcache + verovio>=4.3.1 # For got_ocr2 + transformers>=4.37.2 # For got_ocr2 + tiktoken>=0.6.0 # For got_ocr2 + accelerate>=0.28.0 # For got_ocr2 + torch # For got_ocr2 + torchvision # For got_ocr2 video = diffusers>=0.30.0 imageio-ffmpeg diff --git a/xinference/api/restful_api.py b/xinference/api/restful_api.py index 56d6c89f2d..ed3a2eab90 100644 --- a/xinference/api/restful_api.py +++ b/xinference/api/restful_api.py @@ -567,6 +567,16 @@ async def internal_exception_handler(request: Request, exc: Exception): else None ), ) + self._router.add_api_route( + "/v1/images/ocr", + self.create_ocr, + methods=["POST"], + dependencies=( + [Security(self._auth_service, scopes=["models:read"])] + if self.is_authenticated() + else None + ), + ) # SD WebUI API self._router.add_api_route( "/sdapi/v1/options", @@ -1754,6 +1764,44 @@ async def create_inpainting( await self._report_error_event(model_uid, str(e)) raise HTTPException(status_code=500, detail=str(e)) + async def create_ocr( + self, + model: str = Form(...), + image: UploadFile = File(media_type="application/octet-stream"), + kwargs: Optional[str] = Form(None), + ) -> Response: + model_uid = model + try: + model_ref = await (await self._get_supervisor_ref()).get_model(model_uid) + except ValueError as ve: + logger.error(str(ve), exc_info=True) + await self._report_error_event(model_uid, str(ve)) + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as e: + logger.error(e, exc_info=True) + await self._report_error_event(model_uid, str(e)) + raise HTTPException(status_code=500, detail=str(e)) + + try: + if kwargs is not None: + parsed_kwargs = json.loads(kwargs) + else: + parsed_kwargs = {} + im = Image.open(image.file) + text = await model_ref.ocr( + image=im, + **parsed_kwargs, + ) + return Response(content=text, media_type="text/plain") + except RuntimeError as re: + logger.error(re, exc_info=True) + await self._report_error_event(model_uid, str(re)) + raise HTTPException(status_code=400, detail=str(re)) + except Exception as e: + logger.error(e, exc_info=True) + await self._report_error_event(model_uid, str(e)) + raise HTTPException(status_code=500, detail=str(e)) + async def create_flexible_infer(self, request: Request) -> Response: payload = await request.json() diff --git a/xinference/client/restful/restful_client.py b/xinference/client/restful/restful_client.py index 438ead7390..dd5e3f1146 100644 --- a/xinference/client/restful/restful_client.py +++ b/xinference/client/restful/restful_client.py @@ -369,6 +369,25 @@ def inpainting( response_data = response.json() return response_data + def ocr(self, image: Union[str, bytes], **kwargs): + url = f"{self._base_url}/v1/images/ocr" + params = { + "model": self._model_uid, + "kwargs": json.dumps(kwargs), + } + files: List[Any] = [] + for key, value in params.items(): + files.append((key, (None, value))) + files.append(("image", ("image", image, "application/octet-stream"))) + response = requests.post(url, files=files, headers=self.auth_headers) + if response.status_code != 200: + raise RuntimeError( + f"Failed to ocr the images, detail: {_get_error_string(response)}" + ) + + response_data = response.json() + return response_data + class RESTfulVideoModelHandle(RESTfulModelHandle): def text_to_video( diff --git a/xinference/core/model.py b/xinference/core/model.py index 206adc25d9..ff29ea941e 100644 --- a/xinference/core/model.py +++ b/xinference/core/model.py @@ -953,6 +953,25 @@ async def inpainting( f"Model {self._model.model_spec} is not for creating image." ) + @log_async( + logger=logger, + ignore_kwargs=["image"], + ) + async def ocr( + self, + image: "PIL.Image", + *args, + **kwargs, + ): + if hasattr(self._model, "ocr"): + return await self._call_wrapper_json( + self._model.ocr, + image, + *args, + **kwargs, + ) + raise AttributeError(f"Model {self._model.model_spec} is not for ocr.") + @request_limit @log_async(logger=logger, ignore_kwargs=["image"]) async def infer( diff --git a/xinference/deploy/docker/requirements.txt b/xinference/deploy/docker/requirements.txt index 89e301761a..6fc624298b 100644 --- a/xinference/deploy/docker/requirements.txt +++ b/xinference/deploy/docker/requirements.txt @@ -24,14 +24,14 @@ peft opencv-contrib-python-headless # all -transformers>=4.34.1 -accelerate>=0.27.2 +transformers>=4.43.2 +accelerate>=0.28.0 sentencepiece transformers_stream_generator bitsandbytes protobuf einops -tiktoken +tiktoken>=0.6.0 sentence-transformers>=3.1.0 diffusers>=0.30.0 controlnet_aux @@ -75,6 +75,7 @@ qwen-vl-utils # For qwen2-vl datamodel_code_generator # for minicpm-4B jsonschema # for minicpm-4B deepcache # for sd +verovio>=4.3.1 # For got_ocr2 # sglang outlines>=0.0.44 diff --git a/xinference/deploy/docker/requirements_cpu.txt b/xinference/deploy/docker/requirements_cpu.txt index c517a69717..7c15d3a9df 100644 --- a/xinference/deploy/docker/requirements_cpu.txt +++ b/xinference/deploy/docker/requirements_cpu.txt @@ -21,8 +21,8 @@ passlib[bcrypt] aioprometheus[starlette]>=23.12.0 nvidia-ml-py async-timeout -transformers>=4.34.1 -accelerate>=0.20.3 +transformers>=4.43.2 +accelerate>=0.28.0 sentencepiece transformers_stream_generator bitsandbytes @@ -69,3 +69,4 @@ ormsgpack # For Fish Speech qwen-vl-utils # For qwen2-vl datamodel_code_generator # for minicpm-4B jsonschema # for minicpm-4B +verovio>=4.3.1 # For got_ocr2 diff --git a/xinference/model/image/core.py b/xinference/model/image/core.py index 8c284c9d87..098cdaa10b 100644 --- a/xinference/model/image/core.py +++ b/xinference/model/image/core.py @@ -15,12 +15,13 @@ import logging import os from collections import defaultdict -from typing import Dict, List, Literal, Optional, Tuple +from typing import Dict, List, Literal, Optional, Tuple, Union from ...constants import XINFERENCE_CACHE_DIR from ...types import PeftModelConfig from ..core import CacheableModelSpec, ModelDescription from ..utils import valid_model_revision +from .ocr.got_ocr2 import GotOCR2Model from .stable_diffusion.core import DiffusionModel logger = logging.getLogger(__name__) @@ -180,6 +181,28 @@ def get_cache_status( return valid_model_revision(meta_path, model_spec.model_revision) +def create_ocr_model_instance( + subpool_addr: str, + devices: List[str], + model_uid: str, + model_spec: ImageModelFamilyV1, + model_path: Optional[str] = None, + **kwargs, +) -> Tuple[GotOCR2Model, ImageModelDescription]: + if not model_path: + model_path = cache(model_spec) + model = GotOCR2Model( + model_uid, + model_path, + model_spec=model_spec, + **kwargs, + ) + model_description = ImageModelDescription( + subpool_addr, devices, model_spec, model_path=model_path + ) + return model, model_description + + def create_image_model_instance( subpool_addr: str, devices: List[str], @@ -189,8 +212,18 @@ def create_image_model_instance( download_hub: Optional[Literal["huggingface", "modelscope", "csghub"]] = None, model_path: Optional[str] = None, **kwargs, -) -> Tuple[DiffusionModel, ImageModelDescription]: +) -> Tuple[Union[DiffusionModel, GotOCR2Model], ImageModelDescription]: model_spec = match_diffusion(model_name, download_hub) + if model_spec.model_ability and "ocr" in model_spec.model_ability: + return create_ocr_model_instance( + subpool_addr=subpool_addr, + devices=devices, + model_uid=model_uid, + model_name=model_name, + model_spec=model_spec, + model_path=model_path, + **kwargs, + ) controlnet = kwargs.get("controlnet") # Handle controlnet if controlnet is not None: diff --git a/xinference/model/image/model_spec.json b/xinference/model/image/model_spec.json index 04386dd2e5..43c77e0e8e 100644 --- a/xinference/model/image/model_spec.json +++ b/xinference/model/image/model_spec.json @@ -178,5 +178,14 @@ "model_ability": [ "inpainting" ] + }, + { + "model_name": "GOT-OCR2_0", + "model_family": "ocr", + "model_id": "stepfun-ai/GOT-OCR2_0", + "model_revision": "cf6b7386bc89a54f09785612ba74cb12de6fa17c", + "model_ability": [ + "ocr" + ] } ] diff --git a/xinference/model/image/model_spec_modelscope.json b/xinference/model/image/model_spec_modelscope.json index b39bfc543d..709de622b9 100644 --- a/xinference/model/image/model_spec_modelscope.json +++ b/xinference/model/image/model_spec_modelscope.json @@ -148,5 +148,15 @@ "model_revision": "62134b9d8e703b5d6f74f1534457287a8bba77ef" } ] + }, + { + "model_name": "GOT-OCR2_0", + "model_family": "ocr", + "model_id": "stepfun-ai/GOT-OCR2_0", + "model_revision": "master", + "model_hub": "modelscope", + "model_ability": [ + "ocr" + ] } ] diff --git a/xinference/model/image/ocr/__init__.py b/xinference/model/image/ocr/__init__.py new file mode 100644 index 0000000000..37f6558d95 --- /dev/null +++ b/xinference/model/image/ocr/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/xinference/model/image/ocr/got_ocr2.py b/xinference/model/image/ocr/got_ocr2.py new file mode 100644 index 0000000000..803e23ed03 --- /dev/null +++ b/xinference/model/image/ocr/got_ocr2.py @@ -0,0 +1,76 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import TYPE_CHECKING, Optional + +import PIL.Image + +if TYPE_CHECKING: + from ..core import ImageModelFamilyV1 + +logger = logging.getLogger(__name__) + + +class GotOCR2Model: + def __init__( + self, + model_uid: str, + model_path: Optional[str] = None, + device: Optional[str] = None, + model_spec: Optional["ImageModelFamilyV1"] = None, + **kwargs, + ): + self._model_uid = model_uid + self._model_path = model_path + self._device = device + # model info when loading + self._model = None + self._tokenizer = None + # info + self._model_spec = model_spec + self._abilities = model_spec.model_ability or [] # type: ignore + self._kwargs = kwargs + + @property + def model_ability(self): + return self._abilities + + def load(self): + from transformers import AutoModel, AutoTokenizer + + self._tokenizer = AutoTokenizer.from_pretrained( + self._model_path, trust_remote_code=True + ) + model = AutoModel.from_pretrained( + self._model_path, + trust_remote_code=True, + low_cpu_mem_usage=True, + device_map="cuda", + use_safetensors=True, + pad_token_id=self._tokenizer.eos_token_id, + ) + self._model = model.eval().cuda() + + def ocr( + self, + image: PIL.Image, + **kwargs, + ): + logger.info("Got OCR 2.0 kwargs: %s", kwargs) + if "ocr_type" not in kwargs: + kwargs["ocr_type"] = "ocr" + assert self._model is not None + # This chat API limits the max new tokens inside. + return self._model.chat(self._tokenizer, image, gradio_input=True, **kwargs) diff --git a/xinference/model/image/tests/test_got_ocr2.py b/xinference/model/image/tests/test_got_ocr2.py new file mode 100644 index 0000000000..385fb375f5 --- /dev/null +++ b/xinference/model/image/tests/test_got_ocr2.py @@ -0,0 +1,41 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io + +from diffusers.utils import load_image + + +def test_got_ocr2(setup): + endpoint, _ = setup + from ....client import Client + + client = Client(endpoint) + + model_uid = client.launch_model( + model_uid="ocr_test", + model_name="GOT-OCR2_0", + model_type="image", + ) + model = client.get_model(model_uid) + + url = "https://huggingface.co/stepfun-ai/GOT-OCR2_0/resolve/main/assets/train_sample.jpg" + image = load_image(url) + bio = io.BytesIO() + image.save(bio, format="JPEG") + r = model.ocr( + image=bio.getvalue(), + ocr_type="ocr", + ) + assert "Jesuits Estate" in r From 6fd879cea94003c54ee35ea4b67c178134381b60 Mon Sep 17 00:00:00 2001 From: JinCheng666 <48248936+JinCheng666@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:42:29 +0800 Subject: [PATCH 6/9] BUG: fix embedding model gte-Qwen2 dimensions (#2479) --- doc/source/models/builtin/embedding/gte-qwen2.rst | 4 ++-- xinference/model/embedding/model_spec.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/source/models/builtin/embedding/gte-qwen2.rst b/doc/source/models/builtin/embedding/gte-qwen2.rst index a88fdece9d..85eeeac39a 100644 --- a/doc/source/models/builtin/embedding/gte-qwen2.rst +++ b/doc/source/models/builtin/embedding/gte-qwen2.rst @@ -11,11 +11,11 @@ gte-Qwen2 Specifications ^^^^^^^^^^^^^^ -- **Dimensions:** 3584 +- **Dimensions:** 4096 - **Max Tokens:** 32000 - **Model ID:** Alibaba-NLP/gte-Qwen2-7B-instruct - **Model Hubs**: `Hugging Face `__, `ModelScope `__ Execute the following command to launch the model:: - xinference launch --model-name gte-Qwen2 --model-type embedding \ No newline at end of file + xinference launch --model-name gte-Qwen2 --model-type embedding diff --git a/xinference/model/embedding/model_spec.json b/xinference/model/embedding/model_spec.json index 14d4ced519..dc8d851b85 100644 --- a/xinference/model/embedding/model_spec.json +++ b/xinference/model/embedding/model_spec.json @@ -233,7 +233,7 @@ }, { "model_name": "gte-Qwen2", - "dimensions": 3584, + "dimensions": 4096, "max_tokens": 32000, "language": ["zh", "en"], "model_id": "Alibaba-NLP/gte-Qwen2-7B-instruct", From 804297fd6f9e5b819c2b8c58d0b0d44ea1e5a3d2 Mon Sep 17 00:00:00 2001 From: codingl2k1 <138426806+codingl2k1@users.noreply.github.com> Date: Fri, 25 Oct 2024 04:45:40 +0200 Subject: [PATCH 7/9] ENH: Pending queue for concurrent requests (#2473) --- xinference/client/tests/test_client.py | 18 +---- xinference/core/model.py | 103 +++++++++++++++++++---- xinference/core/tests/test_model.py | 108 +++++++++++++++++++++++++ 3 files changed, 198 insertions(+), 31 deletions(-) create mode 100644 xinference/core/tests/test_model.py diff --git a/xinference/client/tests/test_client.py b/xinference/client/tests/test_client.py index 5e85556d05..df28e8260a 100644 --- a/xinference/client/tests/test_client.py +++ b/xinference/client/tests/test_client.py @@ -99,15 +99,9 @@ def _check_stream(): for _ in range(2): r = executor.submit(_check_stream) results.append(r) - # Parallel generation is not supported by llama-cpp-python. - error_count = 0 + for r in results: - try: - r.result() - except Exception as ex: - assert "Parallel generation" in str(ex) - error_count += 1 - assert error_count == 1 + r.result() # After iteration finish, we can iterate again. _check_stream() @@ -143,18 +137,12 @@ def _check(stream=False): for stream in [True, False]: results = [] - error_count = 0 with ThreadPoolExecutor() as executor: for _ in range(3): r = executor.submit(_check, stream=stream) results.append(r) for r in results: - try: - r.result() - except Exception as ex: - assert "Parallel generation" in str(ex) - error_count += 1 - assert error_count == (2 if stream else 0) + r.result() client.terminate_model(model_uid=model_uid) assert len(client.list_models()) == 0 diff --git a/xinference/core/model.py b/xinference/core/model.py index ff29ea941e..567ef81769 100644 --- a/xinference/core/model.py +++ b/xinference/core/model.py @@ -17,10 +17,10 @@ import inspect import json import os +import queue import time import types import uuid -import weakref from asyncio.queues import Queue from asyncio.tasks import wait_for from concurrent.futures import Future as ConcurrentFuture @@ -32,7 +32,6 @@ Callable, Dict, Generator, - Iterator, List, Optional, Union, @@ -209,9 +208,8 @@ def __init__( model_description.to_dict() if model_description else {} ) self._request_limits = request_limits - - self._generators: Dict[str, Union[Iterator, AsyncGenerator]] = {} - self._current_generator = lambda: None + self._pending_requests: asyncio.Queue = asyncio.Queue() + self._handle_pending_requests_task = None self._lock = ( None if isinstance( @@ -237,6 +235,10 @@ def __init__( async def __post_create__(self): self._loop = asyncio.get_running_loop() + self._handle_pending_requests_task = asyncio.create_task( + self._handle_pending_requests() + ) + if self.allow_batching(): from .scheduler import SchedulerActor @@ -474,6 +476,43 @@ async def _to_async_gen(self, output_type: str, gen: types.AsyncGeneratorType): ) await asyncio.gather(*coros) + async def _handle_pending_requests(self): + logger.info("Start requests handler.") + while True: + gen, stream_out, stop = await self._pending_requests.get() + + async def _async_wrapper(_gen): + try: + # anext is only available for Python >= 3.10 + return await _gen.__anext__() # noqa: F821 + except StopAsyncIteration: + return stop + + def _wrapper(_gen): + # Avoid issue: https://github.com/python/cpython/issues/112182 + try: + return next(_gen) + except StopIteration: + return stop + + while True: + try: + if inspect.isgenerator(gen): + r = await asyncio.to_thread(_wrapper, gen) + elif inspect.isasyncgen(gen): + r = await _async_wrapper(gen) + else: + raise Exception( + f"The generator {gen} should be a generator or an async generator, " + f"but a {type(gen)} is got." + ) + stream_out.put_nowait(r) + if r is not stop: + continue + except Exception: + logger.exception("stream encountered an error.") + break + async def _call_wrapper_json(self, fn: Callable, *args, **kwargs): return await self._call_wrapper("json", fn, *args, **kwargs) @@ -487,6 +526,13 @@ async def _call_wrapper(self, output_type: str, fn: Callable, *args, **kwargs): ret = await fn(*args, **kwargs) else: ret = await asyncio.to_thread(fn, *args, **kwargs) + + if inspect.isgenerator(ret): + gen = self._to_generator(output_type, ret) + return gen + if inspect.isasyncgen(ret): + gen = self._to_async_gen(output_type, ret) + return gen else: async with self._lock: if inspect.iscoroutinefunction(fn): @@ -494,17 +540,40 @@ async def _call_wrapper(self, output_type: str, fn: Callable, *args, **kwargs): else: ret = await asyncio.to_thread(fn, *args, **kwargs) - if self._lock is not None and self._current_generator(): - raise Exception("Parallel generation is not supported by llama-cpp-python.") + stream_out: Union[queue.Queue, asyncio.Queue] + + if inspect.isgenerator(ret): + gen = self._to_generator(output_type, ret) + stream_out = queue.Queue() + stop = object() + self._pending_requests.put_nowait((gen, stream_out, stop)) + + def _stream_out_generator(): + while True: + o = stream_out.get() + if o is stop: + break + else: + yield o + + return _stream_out_generator() + + if inspect.isasyncgen(ret): + gen = self._to_async_gen(output_type, ret) + stream_out = asyncio.Queue() + stop = object() + self._pending_requests.put_nowait((gen, stream_out, stop)) + + async def _stream_out_async_gen(): + while True: + o = await stream_out.get() + if o is stop: + break + else: + yield o + + return _stream_out_async_gen() - if inspect.isgenerator(ret): - gen = self._to_generator(output_type, ret) - self._current_generator = weakref.ref(gen) - return gen - if inspect.isasyncgen(ret): - gen = self._to_async_gen(output_type, ret) - self._current_generator = weakref.ref(gen) - return gen if output_type == "json": return await asyncio.to_thread(json_dumps, ret) else: @@ -592,7 +661,6 @@ async def handle_batching_request( prompt_or_messages, queue, call_ability, *args, **kwargs ) gen = self._to_async_gen("json", ret) - self._current_generator = weakref.ref(gen) return gen else: from .scheduler import XINFERENCE_NON_STREAMING_ABORT_FLAG @@ -1013,3 +1081,6 @@ async def text_to_video( async def record_metrics(self, name, op, kwargs): worker_ref = await self._get_worker_ref() await worker_ref.record_metrics(name, op, kwargs) + + async def get_pending_requests_count(self): + return self._pending_requests.qsize() diff --git a/xinference/core/tests/test_model.py b/xinference/core/tests/test_model.py new file mode 100644 index 0000000000..655debf799 --- /dev/null +++ b/xinference/core/tests/test_model.py @@ -0,0 +1,108 @@ +# Copyright 2022-2023 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import pytest +import pytest_asyncio +import xoscar as xo +from xoscar import create_actor_pool + +from ..model import ModelActor + +TEST_EVENT = None +TEST_VALUE = None + + +class MockModel: + async def generate(self, prompt, **kwargs): + global TEST_VALUE + TEST_VALUE = True + assert isinstance(TEST_EVENT, asyncio.Event) + await TEST_EVENT.wait() + yield {"test1": prompt} + yield {"test2": prompt} + + +class MockModelActor(ModelActor): + def __init__( + self, + supervisor_address: str, + worker_address: str, + ): + super().__init__(supervisor_address, worker_address, MockModel()) # type: ignore + self._lock = asyncio.locks.Lock() + + async def __pre_destroy__(self): + pass + + async def record_metrics(self, name, op, kwargs): + pass + + +@pytest_asyncio.fixture +async def setup_pool(): + pool = await create_actor_pool( + f"test://127.0.0.1:{xo.utils.get_next_port()}", n_process=0 + ) + async with pool: + yield pool + + +@pytest.mark.asyncio +async def test_concurrent_call(setup_pool): + pool = setup_pool + addr = pool.external_address + + global TEST_EVENT + TEST_EVENT = asyncio.Event() + + worker: xo.ActorRefType[MockModelActor] = await xo.create_actor( # type: ignore + MockModelActor, + address=addr, + uid=MockModelActor.default_uid(), + supervisor_address="test:123", + worker_address="test:345", + ) + + await worker.generate("test_prompt1") + assert TEST_VALUE is not None + # This request is waiting for the TEST_EVENT, so the queue is empty. + pending_count = await worker.get_pending_requests_count() + assert pending_count == 0 + await worker.generate("test_prompt3") + # This request is waiting in the queue because the previous request is waiting for TEST_EVENT. + pending_count = await worker.get_pending_requests_count() + assert pending_count == 1 + + async def _check(): + gen = await worker.generate("test_prompt2") + result = [] + async for g in gen: + result.append(g) + assert result == [ + b'data: {"test1": "test_prompt2"}\r\n\r\n', + b'data: {"test2": "test_prompt2"}\r\n\r\n', + ] + + check_task = asyncio.create_task(_check()) + await asyncio.sleep(2) + assert not check_task.done() + # Pending 2 requests: test_prompt3 and test_prompt2 + pending_count = await worker.get_pending_requests_count() + assert pending_count == 2 + TEST_EVENT.set() + await check_task + pending_count = await worker.get_pending_requests_count() + assert pending_count == 0 From f7f873f188a9a2ec3a404455622332407d115bb7 Mon Sep 17 00:00:00 2001 From: yiboyasss <143868051+yiboyasss@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:09:40 +0800 Subject: [PATCH 8/9] FEAT: [UI] Image model with the lora_config. (#2482) --- .../ui/src/scenes/launch_model/modelCard.js | 424 ++++++++++-------- 1 file changed, 239 insertions(+), 185 deletions(-) diff --git a/xinference/web/ui/src/scenes/launch_model/modelCard.js b/xinference/web/ui/src/scenes/launch_model/modelCard.js index a4391f830a..bcffbace11 100644 --- a/xinference/web/ui/src/scenes/launch_model/modelCard.js +++ b/xinference/web/ui/src/scenes/launch_model/modelCard.js @@ -327,6 +327,9 @@ const ModelCard = ({ modelDataWithID_LLM.n_gpu_layers = nGPULayers } + const modelDataWithID = + modelType === 'LLM' ? modelDataWithID_LLM : modelDataWithID_other + if ( loraListArr.length || imageLoraLoadKwargsArr.length || @@ -354,12 +357,9 @@ const ModelCard = ({ }) peft_model_config['lora_list'] = lora_list } - modelDataWithID_LLM['peft_model_config'] = peft_model_config + modelDataWithID['peft_model_config'] = peft_model_config } - const modelDataWithID = - modelType === 'LLM' ? modelDataWithID_LLM : modelDataWithID_other - if (customParametersArr.length) { customParametersArr.forEach((item) => { modelDataWithID[item.key] = handleValueType(item.value) @@ -376,10 +376,15 @@ const ModelCard = ({ `/running_models/${modelType}` ) let historyArr = JSON.parse(localStorage.getItem('historyArr')) || [] - if (!historyArr.some((item) => deepEqual(item, modelDataWithID))) { - historyArr = historyArr.filter( - (item) => item.model_name !== modelDataWithID.model_name - ) + const historyModelNameArr = historyArr.map((item) => item.model_name) + if (historyModelNameArr.includes(modelDataWithID.model_name)) { + historyArr = historyArr.map((item) => { + if (item.model_name === modelDataWithID.model_name) { + return modelDataWithID + } + return item + }) + } else { historyArr.push(modelDataWithID) } localStorage.setItem('historyArr', JSON.stringify(historyArr)) @@ -600,28 +605,10 @@ const ModelCard = ({ }) setLoraArr(loraData) - let ImageLoraLoadData = [] - for (let key in peft_model_config?.image_lora_load_kwargs) { - ImageLoraLoadData.push({ - key: key, - value: peft_model_config?.image_lora_load_kwargs[key], - }) - } - setImageLoraLoadArr(ImageLoraLoadData) - - let ImageLoraFuseData = [] - for (let key in peft_model_config?.image_lora_fuse_kwargs) { - ImageLoraFuseData.push({ - key: key, - value: peft_model_config?.image_lora_fuse_kwargs[key], - }) - } - setImageLoraFuseArr(ImageLoraFuseData) - let customData = [] for (let key in arr[0]) { !llmAllDataKey.includes(key) && - customData.push({ key: key, value: arr[0][key] }) + customData.push({ key: key, value: arr[0][key] || 'none' }) } setCustomArr(customData) @@ -635,11 +622,7 @@ const ModelCard = ({ ) setIsOther(true) - if ( - loraData.length || - ImageLoraLoadData.length || - ImageLoraFuseData.length - ) { + if (loraData.length) { setIsOther(true) setIsPeftModelConfig(true) } @@ -657,37 +640,54 @@ const ModelCard = ({ setDownloadHub(arr[0].download_hub || '') setModelPath(arr[0].model_path || '') + if (arr[0].model_type === 'image') { + let loraData = [] + arr[0].peft_model_config?.lora_list?.forEach((item) => { + loraData.push({ + lora_name: item.lora_name, + local_path: item.local_path, + }) + }) + setLoraArr(loraData) + + let ImageLoraLoadData = [] + for (let key in arr[0].peft_model_config?.image_lora_load_kwargs) { + ImageLoraLoadData.push({ + key: key, + value: + arr[0].peft_model_config?.image_lora_load_kwargs[key] || 'none', + }) + } + setImageLoraLoadArr(ImageLoraLoadData) + + let ImageLoraFuseData = [] + for (let key in arr[0].peft_model_config?.image_lora_fuse_kwargs) { + ImageLoraFuseData.push({ + key: key, + value: + arr[0].peft_model_config?.image_lora_fuse_kwargs[key] || 'none', + }) + } + setImageLoraFuseArr(ImageLoraFuseData) + + if ( + loraData.length || + ImageLoraLoadData.length || + ImageLoraFuseData.length + ) { + setIsPeftModelConfig(true) + } + } + let customData = [] for (let key in arr[0]) { !llmAllDataKey.includes(key) && - customData.push({ key: key, value: arr[0][key] }) + customData.push({ key: key, value: arr[0][key] || 'none' }) } setCustomArr(customData) } } - const deepEqual = (obj1, obj2) => { - if (obj1 === obj2) return true - if ( - typeof obj1 !== 'object' || - typeof obj2 !== 'object' || - obj1 == null || - obj2 == null - ) { - return false - } - - let keysA = Object.keys(obj1) - let keysB = Object.keys(obj2) - if (keysA.length !== keysB.length) return false - for (let key of keysA) { - if (!keysB.includes(key) || !deepEqual(obj1[key], obj2[key])) { - return false - } - } - return true - } - const handleCollection = (bool) => { setHover(false) @@ -725,8 +725,6 @@ const ModelCard = ({ setDownloadHub('') setModelPath('') setLoraArr([]) - setImageLoraLoadArr([]) - setImageLoraFuseArr([]) setCustomArr([]) setIsOther(false) setIsPeftModelConfig(false) @@ -738,6 +736,11 @@ const ModelCard = ({ setWorkerIp('') setDownloadHub('') setModelPath('') + setLoraArr([]) + setImageLoraLoadArr([]) + setImageLoraFuseArr([]) + setCustomArr([]) + setIsPeftModelConfig(false) } } @@ -991,7 +994,14 @@ const ModelCard = ({ {(() => { if (modelData.language) { return modelData.language.map((v) => { - return + return ( + + ) }) } else if (modelData.model_family) { return ( @@ -1446,30 +1456,6 @@ const ModelCard = ({ onJudgeArr={judgeArr} pairData={loraArr} /> - { - setImageLoraLoadKwargsArr(arr) - }} - onJudgeArr={judgeArr} - pairData={imageLoraLoadArr} - /> - { - setImageLoraFuseKwargsArr(arr) - }} - onJudgeArr={judgeArr} - pairData={imageLoraFuseArr} - /> ) : ( - - setModelUID(e.target.value)} - /> - setReplica(parseInt(e.target.value, 10))} - /> + - Device - - - {nGpu === 'GPU' && ( + setModelUID(e.target.value)} + /> + setReplica(parseInt(e.target.value, 10))} + /> + + Device + + + {nGpu === 'GPU' && ( + + { + setGPUIdxAlert(false) + setGPUIdx(e.target.value) + const regular = /^\d+(?:,\d+)*$/ + if ( + e.target.value !== '' && + !regular.test(e.target.value) + ) { + setGPUIdxAlert(true) + } + }} + /> + {GPUIdxAlert && ( + + Please enter numeric data separated by commas, for + example: 0,1,2 + + )} + + )} setWorkerIp(e.target.value)} + /> + + + + (Optional) Download_hub + + + + + setModelPath(e.target.value)} /> - {GPUIdxAlert && ( - - Please enter numeric data separated by commas, for - example: 0,1,2 - - )} - )} - - setWorkerIp(e.target.value)} - /> - - - - (Optional) Download_hub - - - - - setModelPath(e.target.value)} + onGetArr={(arr) => { + setCustomParametersArr(arr) + }} + onJudgeArr={judgeArr} + pairData={customArr} /> - { - setCustomParametersArr(arr) - }} - onJudgeArr={judgeArr} - pairData={customArr} - /> - + )}