refactor: refactor package import usage

wemysschen · Nov 9, 2023 · dc928c8 · dc928c8
1 parent 3ad55a0
commit dc928c8
Show file tree

Hide file tree

Showing 12 changed files with 101 additions and 41 deletions.
diff --git a/Makefile b/Makefile
@@ -8,7 +8,7 @@ uninstall:
 	pip uninstall -y qianfan
 
 clean:
-	rm -rf build output dist qianfan.egg-info docs
+	rm -rf build output dist qianfan.egg-info
 
 doc: install
 	poetry run bash src/scripts/build_doc.sh

diff --git a/README.md b/README.md
@@ -97,7 +97,7 @@ print(resp["result"])
 
 ### 大模型调优
 
-目前千帆平台支持如下训练调优能力，详见[训练调优](./docs/console.md)
+目前千帆平台支持如下训练调优能力，详见[训练调优](./docs/train_service.md)
 - 创建训练任务
 - 创建任务运行
 - 获取任务运行详情

diff --git a/cookbook/console-finetune/console-finetune.ipynb b/cookbook/console-finetune/console-finetune.ipynb
@@ -21,6 +21,17 @@
     "bce_sk = \"your_iam_sk\""
    ]
   },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "本文使用的千帆版本：\n",
+    "```\n",
+    "pip install qianfan==0.1.3\n",
+    "```"
+   ]
+  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -131,7 +142,7 @@
     }
    ],
    "source": [
-    "from qianfan.resources.console.data import Data\n",
+    "from qianfan.resources import Data\n",
     "from qianfan.resources.console.consts import DataSetType, DataProjectType, DataTemplateType, DataStorageType\n",
     "\n",
     "# 创建数据集\n",
@@ -288,8 +299,9 @@
     }
    ],
    "source": [
+    "from qianfan.resources import FineTune\n",
     "# 创建任务\n",
-    "resp = qianfan.FineTune.create_task(name=\"test_sdk_taskqf02\", description=\"for_eb_turbo1\")\n",
+    "resp = FineTune.create_task(name=\"test_sdk_taskqf02\", description=\"for_eb_turbo1\")\n",
     "\n",
     "# 获取任务ID\n",
     "task_id = resp[\"result\"][\"id\"]\n",
@@ -331,7 +343,7 @@
    ],
    "source": [
     "# 创建任务运行，具体参数可以参见 API 文档\n",
-    "create_job_resp = qianfan.FineTune.create_job({\n",
+    "create_job_resp = FineTune.create_job({\n",
     "    \"taskId\": task_id,\n",
     "    \"baseTrainType\": \"ERNIE-Bot-turbo\",\n",
     "    \"trainType\": \"ERNIE-Bot-turbo-0725\",\n",
@@ -425,7 +437,7 @@
     "\n",
     "job_id = create_job_resp[\"result\"][\"id\"]\n",
     "while True:\n",
-    "    job_status_resp = qianfan.FineTune.get_job(task_id=task_id, job_id=job_id)\n",
+    "    job_status_resp = FineTune.get_job(task_id=task_id, job_id=job_id)\n",
     "    job_status = job_status_resp[\"result\"][\"trainStatus\"]\n",
     "    print(\"job status:\", job_status)\n",
     "    if job_status != 'RUNNING':\n",
@@ -459,7 +471,9 @@
     }
    ],
    "source": [
-    "sft_task_publish_resp = qianfan.Model.publish(is_new=True, model_name=\"test_sdk_ebt1\", version_meta={\n",
+    "from qianfan.resources import Model\n",
+    "\n",
+    "sft_task_publish_resp = Model.publish(is_new=True, model_name=\"test_sdk_ebt1\", version_meta={\n",
     "    \"taskId\": task_id, \"iterationId\": job_id\n",
     "})\n",
     "# 获取model_id and version\n",
@@ -484,7 +498,7 @@
    ],
    "source": [
     "# 获取模型版本信息：\n",
-    "model_version_list = qianfan.Model.list(model_id=model_id)\n",
+    "model_version_list = Model.list(model_id=model_id)\n",
     "model_version_id: int = 0 \n",
     "for m in model_version_list[\"result\"][\"modelVersionList\"]:\n",
     "    if m[\"modelId\"] == model_id and m[\"version\"] == model_version:\n",
@@ -522,7 +536,7 @@
     "# 获取模型版本详情\n",
     "# 模型版本状态有三种：Creating, Ready, Failed\n",
     "while True:\n",
-    "    model_detail_info = qianfan.Model.detail(model_version_id=model_version_id)\n",
+    "    model_detail_info = Model.detail(model_version_id=model_version_id)\n",
     "    model_version_state = model_detail_info[\"result\"][\"state\"]\n",
     "    print(\"current model_version_state:\", model_version_state)\n",
     "    if model_version_state != \"Creating\":\n",
@@ -554,8 +568,8 @@
     }
    ],
    "source": [
-    "\n",
-    "g = qianfan.Service.create(\n",
+    "from qianfan.resources import Service\n",
+    "g = Service.create(\n",
     "    model_id = model_id, \n",
     "    iteration_id = model_version_id, \n",
     "    name=\"testsebqf1\", \n",
@@ -651,7 +665,7 @@
     "# 资源付费完成后，serviceStatus会变成Deploying，查看模型服务状态, 直到serviceStatus变成部署完成，得到model_endpoint\n",
     "# 这一步涉及到资源调度，需要等待5-20分钟不等\n",
     "while True:\n",
-    "    resp = qianfan.Service.get(id = svc_id)\n",
+    "    resp = Service.get(id = svc_id)\n",
     "    svc_status = resp[\"result\"][\"serviceStatus\"]\n",
     "    print(\"svc deploy status:\", svc_status)\n",
     "    if svc_status in [\"Finish\", \"\"]:\n",
@@ -701,6 +715,7 @@
     }
    ],
    "source": [
+    "import qianfan\n",
     "chat_comp = qianfan.ChatCompletion(endpoint=sft_model_endpoint)\n",
     "msgs = qianfan.Messages()\n",
     "msgs.append(message=\"你好，你是谁？\", role=qianfan.QfRole.User)\n",

diff --git a/docs/dataset.md b/docs/dataset.md
@@ -12,7 +12,7 @@
 
 使用前需要引用入Data类
 ```python
-from qianfan.resources.console.data import Data
+from qianfan.resources import Data
 ```
 
 #### **创建数据集**

diff --git a/docs/console.md → docs/train_service.md b/docs/console.md → docs/train_service.md
@@ -7,13 +7,18 @@ SFT 相关操作：
 - [获取任务运行详情](#获取任务运行详情)
 - [停止任务运行](#停止任务运行)
 
+使用前需要引用入FineTune类
+```python
+from qianfan.resources import FineTune
+```
+
 #### **创建训练任务**
 
 需要提供任务名称 `name` 和任务描述 `description`，返回结果在 `result` 字段中，具体字段与 [API 文档](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/almrgn397#body%E5%8F%82%E6%95%B0) 一致。
 
 ```python
 # 创建任务
-resp = qianfan.FineTune.create_task(name="task_name", description="task_desc")
+resp = FineTune.create_task(name="task_name", description="task_desc")
 # 获取返回结果
 task_id = resp['result']['id']
 print(task_id)
@@ -25,7 +30,7 @@ print(task_id)
 
 ```python
 # 创建任务运行，具体参数可以参见 API 文档
-create_job_resp = qianfan.FineTune.create_job({
+create_job_resp = FineTune.create_job({
     "taskId": task_id,
     "baseTrainType": "ERNIE-Bot-turbo",
     "trainType": "ERNIE-Bot-turbo-0725",
@@ -56,7 +61,7 @@ print(create_job_resp['result']['id'])
 
 ```python
 # 根据任务和运行id，查询任务运行的具体状态
-job= qianfan.FineTune.get_job(task_id, job_id)
+job= FineTune.get_job(task_id, job_id)
 # 获取任务详情
 print(job['result'])
 ```
@@ -67,7 +72,7 @@ print(job['result'])
 
 ```python
 # 提供任务和运行 id，停止运行
-stop = qianfan.FineTune.stop_job(task_id, job_id)
+stop = FineTune.stop_job(task_id, job_id)
 # 获取停止结果
 print(stop['result']) # => True
 ```
@@ -82,12 +87,17 @@ print(stop['result']) # => True
 - [获取模型版本详情](#获取模型版本详情)
 - [训练任务发布为模型](#训练任务发布为模型)
 
+使用前需要引用入Model类
+```python
+from qianfan.resources import Model
+```
+
 #### **获取模型详情**
 
 可以获得该模型的所有版本信息，需要提供模型的 id，可以从 [智能云千帆控制台-模型仓库列表](https://console.bce.baidu.com/qianfan/modelcenter/model/user/list) 获得，详细方法和返回参数字段参见 [API 文档](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/clnlizwcs)。
 
 ```python
-model_list = qianfan.Model.list(model_id = 5862)
+model_list = Model.list(model_id = 5862)
 print(model_list['result']['modelVersionList'][0]['modelName'])
 ```
 
@@ -96,7 +106,7 @@ print(model_list['result']['modelVersionList'][0]['modelName'])
 可以获取某个模型版本的具体信息，需要提供模型版本 id，可以从 [智能云千帆控制台-模型仓库列表](https://console.bce.baidu.com/qianfan/modelcenter/model/user/list) 的某个模型详情中获得，详细方法和返回参数字段参见 [API 文档](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/ylnljj3ku)。
 
 ```python
-model = qianfan.Model.detail(model_version_id = 5659)
+model = Model.detail(model_version_id = 5659)
 print(model['result']['modelName'])
 ```
 
@@ -105,7 +115,7 @@ print(model['result']['modelName'])
 可以将某个已完成的训练任务得到的模型发布至模型仓库中，需要提供任务 id 等信息，字段定义与返回参数字段参见 [API 文档](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/Jlnlm0rdx)。
 
 ```python
-g = qianfan.Model.publish(
+g = Model.publish(
     is_new=True,
     model_name="sdk_test_1",
     version_meta={"taskId": 9220, "iterationId": 5234},
@@ -122,12 +132,17 @@ print(g['result']['modelId'])
 - [创建服务](#创建服务)
 - [查询服务详情](#查询服务详情)
 
+使用前需要引用入Service类
+```python
+from qianfan.resources import Service
+```
+
 #### **创建服务**
 
 可以将某个模型发布成可对外访问的服务，需要提供模型的 id、服务名称等信息，详细字段和返回参数字段参见 [API 文档](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/Plnlmxdgy)。
 
 ```python
-g = qianfan.Service.create(
+g = Service.create(
     model_id=123,
     model_version_id=456,
     name="sdk_test",
@@ -142,6 +157,6 @@ print(g['result'])
 可以获取服务的具体信息，需要提供服务的 id，可以从 [百度智能云千帆控制台-服务管理](https://console.bce.baidu.com/qianfan/ais/console/onlineService) 的某个服务详情中获得，详细方法和返回参数字段参见 [API 文档](https://cloud.baidu.com/doc/WENXINWORKSHOP/s/llnlmyp8o)。
 
 ```python
-svc = qianfan.Service.get(id = 2047)
+svc = Service.get(id = 2047)
 print(svc['result']['id'])
 ```
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qianfan"
-version = "0.1.2"
+version = "0.1.3"
 description = "文心千帆大模型平台 Python SDK"
 authors = []
 license = "Apache-2.0"

diff --git a/src/qianfan/__init__.py b/src/qianfan/__init__.py
@@ -13,21 +13,22 @@
 # limitations under the License.
 
 """
-Library that wraps the qianfan API.
+Library aimed to helping developer to interactive with LLM.
 """
 
 
 from qianfan.config import AK, SK, AccessKey, AccessToken, SecretKey, get_config
-from qianfan.resources.console.finetune import FineTune
-from qianfan.resources.console.model import Model
-from qianfan.resources.console.service import Service
-from qianfan.resources.images.text2image import Text2Image
-from qianfan.resources.llm.chat_completion import ChatCompletion
-from qianfan.resources.llm.completion import Completion
-from qianfan.resources.llm.embedding import Embedding
-from qianfan.resources.llm.plugin import Plugin
-from qianfan.resources.tools.tokenizer import Tokenizer
-from qianfan.resources.typing import QfMessages, QfResponse, QfRole
+from qianfan.resources import (
+    ChatCompletion,
+    Completion,
+    Embedding,
+    Plugin,
+    QfMessages,
+    QfResponse,
+    QfRole,
+    Text2Image,
+    Tokenizer,
+)
 from qianfan.utils import disable_log, enable_log
 from qianfan.version import VERSION
 
@@ -40,9 +41,6 @@
     "Embedding",
     "Completion",
     "Plugin",
-    "FineTune",
-    "Model",
-    "Service",
     "Text2Image",
     "Tokenizer",
     "AK",

diff --git a/src/qianfan/resources/__init__.py b/src/qianfan/resources/__init__.py
@@ -0,0 +1,32 @@
+from qianfan.resources.console.data import Data
+from qianfan.resources.console.finetune import FineTune
+from qianfan.resources.console.model import Model
+from qianfan.resources.console.service import Service
+from qianfan.resources.images.text2image import Text2Image
+from qianfan.resources.llm.chat_completion import ChatCompletion
+from qianfan.resources.llm.completion import Completion
+from qianfan.resources.llm.embedding import Embedding
+from qianfan.resources.llm.plugin import Plugin
+from qianfan.resources.tools.tokenizer import Tokenizer
+from qianfan.resources.typing import QfMessages, QfResponse, QfRole
+
+__all__ = [
+    "Data",
+    "Model",
+    "Service",
+    "FineTune",
+    "ChatCompletion",
+    "Embedding",
+    "Completion",
+    "Plugin",
+    "Text2Image",
+    "Tokenizer",
+    "AK",
+    "SK",
+    "Role",
+    "Messages",
+    "Response",
+    "QfRole",
+    "QfMessages",
+    "QfResponse",
+]
diff --git a/src/qianfan/tests/data_api_test.py b/src/qianfan/tests/data_api_test.py
@@ -17,6 +17,7 @@
 """
 
 
+from qianfan.resources import Data
 from qianfan.resources.console.consts import (
     DataExportDestinationType,
     DataProjectType,
@@ -25,7 +26,6 @@
     DataStorageType,
     DataTemplateType,
 )
-from qianfan.resources.console.data import Data
 
 enter_except_flag = True
 

diff --git a/src/qianfan/tests/finetune_test.py b/src/qianfan/tests/finetune_test.py
@@ -17,7 +17,7 @@
 """
 
 
-from qianfan import FineTune
+from qianfan.resources import FineTune
 
 
 def test_create_finetune_task():

diff --git a/src/qianfan/tests/model_test.py b/src/qianfan/tests/model_test.py
@@ -17,7 +17,7 @@
 """
 
 
-from qianfan import Model
+from qianfan.resources import Model
 
 
 def test_list_model():

diff --git a/src/qianfan/tests/service_test.py b/src/qianfan/tests/service_test.py
@@ -17,7 +17,7 @@
 """
 
 
-from qianfan import Service
+from qianfan.resources import Service
 
 
 def test_create_service():
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,7 +17,7 @@ @@
     """
-    from qianfan import FineTune
+    from qianfan.resources import FineTune
     def test_create_finetune_task():
@@ Expand Down @@