feat: consul deduplicate and sort (apache#10941)

dabue · Feb 27, 2024 · 18bf7ae · 18bf7ae
1 parent 44c1180
commit 18bf7ae
Show file tree

Hide file tree

Showing 5 changed files with 393 additions and 6 deletions.
diff --git a/apisix/discovery/consul/init.lua b/apisix/discovery/consul/init.lua
@@ -45,6 +45,7 @@ local next               = next
 local all_services = core.table.new(0, 5)
 local default_service
 local default_weight
+local sort_type
 local skip_service_map = core.table.new(0, 1)
 local dump_params
 
@@ -339,6 +340,25 @@ local function watch_result_is_valid(watch_type, index, catalog_index, health_in
 end
 
 
+local function combine_sort_nodes_cmp(left, right)
+    if left.host ~= right.host then
+        return left.host < right.host
+    end
+
+    return left.port < right.port
+end
+
+
+local function port_sort_nodes_cmp(left, right)
+    return left.port < right.port
+end
+
+
+local function host_sort_nodes_cmp(left, right)
+    return left.host < right.host
+end
+
+
 function _M.connect(premature, consul_server, retry_delay)
     if premature then
         return
@@ -489,6 +509,7 @@ function _M.connect(premature, consul_server, retry_delay)
             if is_not_empty(result.body) then
                 -- add services to table
                 local nodes = up_services[service_name]
+                local nodes_uniq = {}
                 for _, node in ipairs(result.body) do
                     if not node.Service then
                         goto CONTINUE
@@ -500,12 +521,29 @@ function _M.connect(premature, consul_server, retry_delay)
                         nodes = core.table.new(1, 0)
                         up_services[service_name] = nodes
                     end
-                    -- add node to nodes table
-                    core.table.insert(nodes, {
-                        host = svc_address,
-                        port = tonumber(svc_port),
-                        weight = default_weight,
-                    })
+                    -- not store duplicate service IDs.
+                    local service_id = svc_address .. ":" .. svc_port
+                    if not nodes_uniq[service_id] then
+                        -- add node to nodes table
+                        core.table.insert(nodes, {
+                            host = svc_address,
+                            port = tonumber(svc_port),
+                            weight = default_weight,
+                        })
+                        nodes_uniq[service_id] = true
+                    end
+                end
+                if nodes then
+                    if sort_type == "port_sort" then
+                        core.table.sort(nodes, port_sort_nodes_cmp)
+
+                    elseif sort_type == "host_sort" then
+                        core.table.sort(nodes, host_sort_nodes_cmp)
+
+                    elseif sort_type == "combine_sort" then
+                        core.table.sort(nodes, combine_sort_nodes_cmp)
+
+                    end
                 end
                 up_services[service_name] = nodes
             end
@@ -592,6 +630,7 @@ function _M.init_worker()
 
     log.notice("consul_conf: ", json_delay_encode(consul_conf, true))
     default_weight = consul_conf.weight
+    sort_type = consul_conf.sort_type
     -- set default service, used when the server node cannot be found
     if consul_conf.default_service then
         default_service = consul_conf.default_service

diff --git a/apisix/discovery/consul/schema.lua b/apisix/discovery/consul/schema.lua
@@ -44,6 +44,11 @@ return {
                 wait = 60,
             }
         },
+        sort_type = {
+            type = "string",
+            enum = {"origin", "host_sort", "port_sort", "combine_sort"},
+            default = "origin",
+        },
         skip_services = {
             type = "array",
             minItems = 1,

diff --git a/ci/pod/docker-compose.first.yml b/ci/pod/docker-compose.first.yml
@@ -55,6 +55,48 @@ services:
     networks:
       consul_net:
 
+  ## Consul cluster
+  consul_node_1:
+    image: consul:1.7
+    restart: unless-stopped
+    ports:
+      - "9500:8500"
+      - "8300:8300"
+      - "8301:8301"
+      - "8302:8302"
+      - "9600:8600"
+    command: [ "consul", "agent", "-server", "-bootstrap-expect=1", "-bind", "0.0.0.0", "-client", "0.0.0.0", "-node", "node-1", "-log-level", "info", "-data-dir=/consul/data", "-enable-script-checks" ]
+    networks:
+      consul_cluster_net:
+        aliases:
+          - consul.cluster
+
+  consul_node_2:
+    image: consul:1.7
+    restart: unless-stopped
+    environment:
+      - CONSUL_BIND_INTERFACE=eth0
+    ports:
+      - "9501:8500"
+    command: [ "consul", "agent", "-server", "-bind", "0.0.0.0", "-client", "0.0.0.0", "-retry-join", "consul.cluster", "-node", "node-2", "-log-level", "info", "-data-dir=/consul/data", "-enable-script-checks" ]
+    networks:
+      consul_cluster_net:
+        aliases:
+          - consul.cluster
+
+  consul_node_3:
+    image: consul:1.7
+    restart: unless-stopped
+    environment:
+      - CONSUL_BIND_INTERFACE=eth0
+    ports:
+      - "9502:8500"
+    command: [ "consul", "agent", "-server", "-bind", "0.0.0.0", "-client", "0.0.0.0", "-retry-join", "consul.cluster", "-node", "node-3", "-log-level", "info", "-data-dir=/consul/data", "-enable-script-checks" ]
+    networks:
+      consul_cluster_net:
+        aliases:
+          - consul.cluster
+
   ## Nacos cluster
   nacos_auth:
     hostname: nacos1
@@ -246,5 +288,6 @@ services:
 
 
 networks:
+  consul_cluster_net:
   consul_net:
   nacos_net:
diff --git a/docs/en/latest/discovery/consul.md b/docs/en/latest/discovery/consul.md
@@ -47,6 +47,7 @@ discovery:
     weight: 1                     # default 1
     fetch_interval: 5             # default 3 sec, only take effect for keepalive: false way
     keepalive: true               # default true, use the long pull way to query consul servers
+    sort_type: "origin"           # default origin
     default_service:              # you can define default service when missing hit
       host: "127.0.0.1"
       port: 20999
@@ -73,6 +74,13 @@ The `keepalive` has two optional values:
 - `true`, default and recommend value, use the long pull way to query consul servers
 - `false`, not recommend, it would use the short pull way to query consul servers, then you can set the `fetch_interval` for fetch interval
 
+The `sort_type` has four optional values:
+
+- `origin`, not sorting
+- `host_sort`, sort by host
+- `port_sort`, sort by port
+- `combine_sort`, with the precondition that hosts are ordered, ports are also ordered.
+
 #### Dump Data
 
 When we need reload `apisix` online, as the `consul` module maybe loads data from CONSUL slower than load routes from ETCD, and would get the log at the moment before load successfully from consul: