Shard key create confirmation (qdrant#3027)

* create dedicated API for creating shards with explicit avait on the consensus * fmt * update api definitions
AKJUS · Dec 6, 2023 · 1fcadcd · 1fcadcd
1 parent d699f66
commit 1fcadcd
Show file tree

Hide file tree

Showing 19 changed files with 924 additions and 20 deletions.
diff --git a/docs/grpc/docs.md b/docs/grpc/docs.md
@@ -19,8 +19,14 @@
     - [CollectionParamsDiff](#qdrant-CollectionParamsDiff)
     - [CreateAlias](#qdrant-CreateAlias)
     - [CreateCollection](#qdrant-CreateCollection)
+    - [CreateShardKey](#qdrant-CreateShardKey)
+    - [CreateShardKeyRequest](#qdrant-CreateShardKeyRequest)
+    - [CreateShardKeyResponse](#qdrant-CreateShardKeyResponse)
     - [DeleteAlias](#qdrant-DeleteAlias)
     - [DeleteCollection](#qdrant-DeleteCollection)
+    - [DeleteShardKey](#qdrant-DeleteShardKey)
+    - [DeleteShardKeyRequest](#qdrant-DeleteShardKeyRequest)
+    - [DeleteShardKeyResponse](#qdrant-DeleteShardKeyResponse)
     - [Disabled](#qdrant-Disabled)
     - [GetCollectionInfoRequest](#qdrant-GetCollectionInfoRequest)
     - [GetCollectionInfoResponse](#qdrant-GetCollectionInfoResponse)
@@ -499,6 +505,56 @@
 
 
 
+<a name="qdrant-CreateShardKey"></a>
+
+### CreateShardKey
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| shard_key | [ShardKey](#qdrant-ShardKey) |  | User-defined shard key |
+| shards_number | [uint32](#uint32) | optional | Number of shards to create per shard key |
+| replication_factor | [uint32](#uint32) | optional | Number of replicas of each shard to create |
+| placement | [uint64](#uint64) | repeated | List of peer ids, allowed to create shards. If empty - all peers are allowed |
+
+
+
+
+
+
+<a name="qdrant-CreateShardKeyRequest"></a>
+
+### CreateShardKeyRequest
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| collection_name | [string](#string) |  | Name of the collection |
+| request | [CreateShardKey](#qdrant-CreateShardKey) |  | Request to create shard key |
+| timeout | [uint64](#uint64) | optional | Wait timeout for operation commit in seconds, if not specified - default value will be supplied |
+
+
+
+
+
+
+<a name="qdrant-CreateShardKeyResponse"></a>
+
+### CreateShardKeyResponse
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| result | [bool](#bool) |  |  |
+
+
+
+
+
+
 <a name="qdrant-DeleteAlias"></a>
 
 ### DeleteAlias
@@ -530,6 +586,53 @@
 
 
 
+<a name="qdrant-DeleteShardKey"></a>
+
+### DeleteShardKey
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| shard_key | [ShardKey](#qdrant-ShardKey) |  | Shard key to delete |
+
+
+
+
+
+
+<a name="qdrant-DeleteShardKeyRequest"></a>
+
+### DeleteShardKeyRequest
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| collection_name | [string](#string) |  | Name of the collection |
+| request | [DeleteShardKey](#qdrant-DeleteShardKey) |  | Request to delete shard key |
+| timeout | [uint64](#uint64) | optional | Wait timeout for operation commit in seconds, if not specified - default value will be supplied |
+
+
+
+
+
+
+<a name="qdrant-DeleteShardKeyResponse"></a>
+
+### DeleteShardKeyResponse
+
+
+
+| Field | Type | Label | Description |
+| ----- | ---- | ----- | ----------- |
+| result | [bool](#bool) |  |  |
+
+
+
+
+
+
 <a name="qdrant-Disabled"></a>
 
 ### Disabled
@@ -986,6 +1089,8 @@ Note: 1kB = 1 vector of size 256. |
 | replicate_shard | [MoveShard](#qdrant-MoveShard) |  |  |
 | abort_transfer | [MoveShard](#qdrant-MoveShard) |  |  |
 | drop_replica | [Replica](#qdrant-Replica) |  |  |
+| create_shard_key | [CreateShardKey](#qdrant-CreateShardKey) |  |  |
+| delete_shard_key | [DeleteShardKey](#qdrant-DeleteShardKey) |  |  |
 | timeout | [uint64](#uint64) | optional | Wait timeout for operation commit in seconds, if not specified - default value will be supplied |
 
 
@@ -1320,6 +1425,8 @@ Note: 1kB = 1 vector of size 256. |
 | ListAliases | [ListAliasesRequest](#qdrant-ListAliasesRequest) | [ListAliasesResponse](#qdrant-ListAliasesResponse) | Get list of all aliases for all existing collections |
 | CollectionClusterInfo | [CollectionClusterInfoRequest](#qdrant-CollectionClusterInfoRequest) | [CollectionClusterInfoResponse](#qdrant-CollectionClusterInfoResponse) | Get cluster information for a collection |
 | UpdateCollectionClusterSetup | [UpdateCollectionClusterSetupRequest](#qdrant-UpdateCollectionClusterSetupRequest) | [UpdateCollectionClusterSetupResponse](#qdrant-UpdateCollectionClusterSetupResponse) | Update cluster setup for a collection |
+| CreateShardKey | [CreateShardKeyRequest](#qdrant-CreateShardKeyRequest) | [CreateShardKeyResponse](#qdrant-CreateShardKeyResponse) | Create shard key |
+| DeleteShardKey | [DeleteShardKeyRequest](#qdrant-DeleteShardKeyRequest) | [DeleteShardKeyResponse](#qdrant-DeleteShardKeyResponse) | Delete shard key |
 
 
 

diff --git a/docs/redoc/master/openapi.json b/docs/redoc/master/openapi.json
@@ -1,5 +1,179 @@
 {
   "paths": {
+    "/collections/{collection_name}/shards": {
+      "put": {
+        "tags": [
+          "collections",
+          "cluster"
+        ],
+        "summary": "Create shard key",
+        "operationId": "create_shard_key",
+        "requestBody": {
+          "description": "Shard key configuration",
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/CreateShardingKey"
+              }
+            }
+          }
+        },
+        "parameters": [
+          {
+            "name": "collection_name",
+            "in": "path",
+            "description": "Name of the collection to create shards for",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "timeout",
+            "in": "query",
+            "description": "Wait for operation commit timeout in seconds. \nIf timeout is reached - request will return with service error.\n",
+            "schema": {
+              "type": "integer"
+            }
+          }
+        ],
+        "responses": {
+          "default": {
+            "description": "error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "description": "error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          },
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "time": {
+                      "type": "number",
+                      "format": "float",
+                      "description": "Time spent to process this request"
+                    },
+                    "status": {
+                      "type": "string",
+                      "enum": [
+                        "ok"
+                      ]
+                    },
+                    "result": {
+                      "type": "boolean"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "/collections/{collection_name}/shards/delete": {
+      "post": {
+        "tags": [
+          "collections",
+          "cluster"
+        ],
+        "summary": "Delete shard key",
+        "operationId": "delete_shard_key",
+        "requestBody": {
+          "description": "Select shard key to delete",
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/DropShardingKey"
+              }
+            }
+          }
+        },
+        "parameters": [
+          {
+            "name": "collection_name",
+            "in": "path",
+            "description": "Name of the collection to create shards for",
+            "required": true,
+            "schema": {
+              "type": "string"
+            }
+          },
+          {
+            "name": "timeout",
+            "in": "query",
+            "description": "Wait for operation commit timeout in seconds. \nIf timeout is reached - request will return with service error.\n",
+            "schema": {
+              "type": "integer"
+            }
+          }
+        ],
+        "responses": {
+          "default": {
+            "description": "error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          },
+          "4XX": {
+            "description": "error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          },
+          "200": {
+            "description": "successful operation",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "time": {
+                      "type": "number",
+                      "format": "float",
+                      "description": "Time spent to process this request"
+                    },
+                    "status": {
+                      "type": "string",
+                      "enum": [
+                        "ok"
+                      ]
+                    },
+                    "result": {
+                      "type": "boolean"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "/telemetry": {
       "get": {
         "summary": "Collect telemetry data",

diff --git a/lib/api/src/grpc/proto/collections.proto b/lib/api/src/grpc/proto/collections.proto
@@ -454,17 +454,50 @@ message Replica {
   uint64 peer_id = 2;
 }
 
+message CreateShardKey {
+    ShardKey shard_key = 1; // User-defined shard key
+    optional uint32 shards_number = 2; // Number of shards to create per shard key
+    optional uint32 replication_factor = 3; // Number of replicas of each shard to create
+    repeated uint64 placement = 4; // List of peer ids, allowed to create shards. If empty - all peers are allowed
+}
+
+message DeleteShardKey {
+    ShardKey shard_key = 1; // Shard key to delete
+}
+
 message UpdateCollectionClusterSetupRequest {
   string collection_name = 1; // Name of the collection
   oneof operation {
     MoveShard move_shard = 2;
     MoveShard replicate_shard = 3;
     MoveShard abort_transfer = 4;
     Replica drop_replica = 5;
+    CreateShardKey create_shard_key = 7;
+    DeleteShardKey delete_shard_key = 8;
   }
   optional uint64 timeout = 6; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
 }
 
 message UpdateCollectionClusterSetupResponse {
   bool result = 1;
 }
+
+message CreateShardKeyRequest {
+    string collection_name = 1; // Name of the collection
+    CreateShardKey request = 2; // Request to create shard key
+    optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
+}
+
+message DeleteShardKeyRequest {
+    string collection_name = 1; // Name of the collection
+    DeleteShardKey request = 2; // Request to delete shard key
+    optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
+}
+
+message CreateShardKeyResponse {
+    bool result = 1;
+}
+
+message DeleteShardKeyResponse {
+    bool result = 1;
+}
diff --git a/lib/api/src/grpc/proto/collections_service.proto b/lib/api/src/grpc/proto/collections_service.proto
@@ -46,4 +46,12 @@ service Collections {
   Update cluster setup for a collection
   */
   rpc UpdateCollectionClusterSetup (UpdateCollectionClusterSetupRequest) returns (UpdateCollectionClusterSetupResponse) {}
+  /*
+  Create shard key
+  */
+  rpc CreateShardKey (CreateShardKeyRequest) returns (CreateShardKeyResponse) {}
+  /*
+  Delete shard key
+  */
+  rpc DeleteShardKey (DeleteShardKeyRequest) returns (DeleteShardKeyResponse) {}
 }