merge from upstream/main

Vasanth-96 · Jun 29, 2022 · e908159 · e908159
2 parents e056025 + 5a9e503
commit e908159
Show file tree

Hide file tree

Showing 64 changed files with 1,235 additions and 955 deletions.
diff --git a/.github/actions/test_sqllogic_standalone_linux/action.yml b/.github/actions/test_sqllogic_standalone_linux/action.yml
@@ -19,10 +19,10 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Install python dependences
+    - name: Test setup
       shell: bash
       run: |
-        pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql six PyHamcrest requests environs fire
+        bash ./scripts/setup/dev_setup.sh -yd
 
     - name: Run sqllogic Tests with Standalone mode with embedded meta-store
       shell: bash

diff --git a/.github/actions/test_sqllogic_standalone_macos/action.yml b/.github/actions/test_sqllogic_standalone_macos/action.yml
@@ -18,17 +18,11 @@ runs:
         profile: ${{ inputs.profile }}
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
-        source: github
 
-    - name: Install mysql client for macos-11
+    - name: Test setup
       shell: bash
       run: |
-        brew install mysql
-
-    - name: Install python dependences
-      shell: bash
-      run: |
-        pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql six PyHamcrest requests environs fire
+        bash ./scripts/setup/dev_setup.sh -yd
 
     - name: Run sqllogic Tests with Standalone mode with embedded meta-store
       shell: bash

diff --git a/.github/actions/test_stateless_cluster_macos/action.yml b/.github/actions/test_stateless_cluster_macos/action.yml
@@ -19,15 +19,10 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Install mysql client for macos-11
+    - name: Test setup
       shell: bash
       run: |
-        brew install mysql
-
-    - name: Install python dependences
-      shell: bash
-      run: |
-        pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy clickhouse_driver
+        bash ./scripts/setup/dev_setup.sh -yd
 
     - name: Run Stateless Tests with Cluster mode
       shell: bash

diff --git a/.github/actions/test_stateless_standalone_macos/action.yml b/.github/actions/test_stateless_standalone_macos/action.yml
@@ -19,15 +19,10 @@ runs:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
 
-    - name: Install mysql client for macos-11
+    - name: Test setup
       shell: bash
       run: |
-        brew install mysql
-
-    - name: Install python dependences
-      shell: bash
-      run: |
-        pip3 install --user boto3 "moto[all]" yapf shfmt-py mysql-connector pymysql sqlalchemy clickhouse_driver
+        bash ./scripts/setup/dev_setup.sh -yd
 
     - name: Run Stateless Tests with Standalone mode
       shell: bash

diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -13,7 +13,7 @@ queue_rules:
       # Reference: https://docs.mergify.com/conditions/#validating-all-status-checks
       # We only require linux checks to pass
       - check-success=check
-      - check-success=build_musl
+      - check-success~=^build_(aarch64|x86_64)_musl$
       - check-success=test_unit
       - check-success=test_metactl
       - check-success=test_stateless_standalone_linux
@@ -23,10 +23,12 @@ queue_rules:
 
 pull_request_rules:
   # Push PR into queue when it passes all checks
-  - name: put bug fix pr to queue
+  - name: put approved pr to queue
     conditions:
       - "#approved-reviews-by>=2"
+      - -draft
       - check-success=check
+      - check-success~=^build_(aarch64|x86_64)_musl$
       - check-success=test_unit
       - check-success=test_metactl
       - check-success=test_stateless_standalone_linux
@@ -41,6 +43,7 @@ pull_request_rules:
   - name: ping author on conflicts
     conditions:
       - conflict
+      - -draft
       - "#approved-reviews-by >= 2"
     actions:
       comment:
@@ -63,6 +66,7 @@ pull_request_rules:
   - name: Check PR description
     conditions:
       - author!=Mergify
+      - -draft
       - '-body~=I hereby agree to the terms of the CLA available at: https:\/\/databend\.rs\/dev\/policies\/cla\/'
       - "-body~=Summary"
       - "-body~=Changelog"

diff --git a/.github/workflows/developing.yml → .github/workflows/dev-linux.yml b/.github/workflows/developing.yml → .github/workflows/dev-linux.yml
@@ -1,4 +1,4 @@
-name: Developing
+name: Dev Linux
 
 on:
   pull_request:
@@ -18,7 +18,7 @@ env:
 
 jobs:
   check:
-    runs-on: [self-hosted, x64, Linux]
+    runs-on: [self-hosted, X64, Linux]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -28,61 +28,44 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
-  build_macos:
-    name: "build_macos(optional)"
-    runs-on: macos-11
+  build_gnu:
+    name: build_${{ matrix.arch }}_gnu
+    runs-on: [self-hosted, X64, Linux, development]
     strategy:
       matrix:
-        target:
-          - x86_64-apple-darwin
-          - aarch64-apple-darwin
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          # fetch all tags,  metasrv and metaclient need tag as its version.
-          fetch-depth: 0
-      - uses: ./.github/actions/build_macos
-        with:
-          target: ${{ matrix.target }}
-          profile: debug
-
-  build_linux:
-    runs-on: [self-hosted, x64, Linux, development]
-    strategy:
-      matrix:
-        target:
-          - x86_64-unknown-linux-gnu
-          - aarch64-unknown-linux-gnu
+        arch:
+          - x86_64
+          - aarch64
     steps:
       - uses: actions/checkout@v3
         with:
           # fetch all tags,  metasrv and metaclient need tag as its version.
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-unknown-linux-gnu
           profile: debug
 
-
   build_musl:
-    runs-on: [self-hosted, x64, Linux, development]
+    name: build_${{ matrix.arch }}_musl
+    runs-on: [self-hosted, X64, Linux, development]
     strategy:
       matrix:
-        target:
-          - x86_64-unknown-linux-musl
-          - aarch64-unknown-linux-musl
+        arch:
+          - x86_64
+          - aarch64
     steps:
       - uses: actions/checkout@v3
         with:
           # fetch all tags,  metasrv and metaclient need tag as its version.
           fetch-depth: 0
       - uses: ./.github/actions/build_linux
         with:
-          target: ${{ matrix.target }}
+          target: ${{ matrix.arch }}-unknown-linux-musl
           profile: debug
 
   test_unit:
-    runs-on: [self-hosted, x64, Linux]
+    runs-on: [self-hosted, X64, Linux]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -93,94 +76,69 @@ jobs:
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
 
   test_metactl:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_metactl
 
   test_compat:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_compat
 
   test_meta_cluster:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_meta_cluster
 
   test_stateless_standalone_linux:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateless_standalone_linux
 
-  # https://github.com/datafuselabs/databend/issues/5287
-  # test_stateless_standalone_macos:
-  #  runs-on: macos-11
-  #  needs: build_macos
-  #  steps:
-  #    - uses: actions/checkout@v3
-  #    - uses: ./.github/actions/test_stateless_standalone_macos
-
   test_stateless_cluster_linux:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateless_cluster_linux
 
-  test_stateless_cluster_macos:
-    name: "test_stateless_cluster_macos(optional)"
-    runs-on: macos-11
-    needs: build_macos
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/test_stateless_cluster_macos
-
-  test_sqllogic_standalone_macos:
-    name: "test_sqllogic_standalone_macos(optional)"
-    runs-on: macos-11
-    needs: build_macos
-    steps:
-      - uses: actions/checkout@v2
-      - uses: ./.github/actions/test_sqllogic_standalone_macos
-
   test_sqllogic_standalone_linux:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/test_sqllogic_standalone_linux
 
   test_stateful_standalone_linux:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_gnu
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateful_standalone_linux
 
-  build_linux_hive:
-    runs-on: [self-hosted, x64, Linux, development]
-    strategy:
-      matrix:
-        target:
-          - x86_64-unknown-linux-gnu
+  build_hive:
+    runs-on: [self-hosted, X64, Linux, development]
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/build_linux_hive
         with:
-          target: ${{ matrix.target }}
+          target: x86_64-unknown-linux-gnu
           profile: debug
 
   test_stateful_hive_standalone:
-    runs-on: [self-hosted, x64, Linux, development]
-    needs: build_linux_hive
+    runs-on: [self-hosted, X64, Linux, development]
+    needs: build_hive
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/test_stateful_hive_standalone
+        with:
+          target: x86_64-unknown-linux-gnu
+          profile: debug
diff --git a/.github/workflows/dev-macos.yml b/.github/workflows/dev-macos.yml
@@ -0,0 +1,60 @@
+name: Dev MacOS
+
+on:
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "website/**"
+      - "**.md"
+      - "scripts/setup/**"
+      - ".devcontainer/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  BUILD_PROFILE: debug
+
+jobs:
+  build_macos:
+    name: "build_${{ matrix.arch }}_macos(optional)"
+    runs-on: macos-11
+    strategy:
+      matrix:
+        arch:
+          - x86_64
+          - aarch64
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # fetch all tags,  metasrv and metaclient need tag as its version.
+          fetch-depth: 0
+      - uses: ./.github/actions/build_macos
+        with:
+          target: ${{ matrix.arch }}-apple-darwin
+          profile: debug
+
+  # https://github.com/datafuselabs/databend/issues/5287
+  # test_stateless_standalone_macos:
+  #  runs-on: macos-11
+  #  needs: build_macos
+  #  steps:
+  #    - uses: actions/checkout@v3
+  #    - uses: ./.github/actions/test_stateless_standalone_macos
+
+  test_stateless_cluster_macos:
+    name: "test_stateless_cluster_macos(optional)"
+    runs-on: macos-11
+    needs: build_macos
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/test_stateless_cluster_macos
+
+  test_sqllogic_standalone_macos:
+    name: "test_sqllogic_standalone_macos(optional)"
+    runs-on: macos-11
+    needs: build_macos
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/test_sqllogic_standalone_macos