[ci] added wheel build scripts

hpcaitech · FrankLeeeee · May 5, 2022 · Apr 29, 2022 · May 1, 2022 · May 1, 2022
commit 524b88a15edb41d087e968a80cb055c3ed344be3
@@ -3,18 +3,21 @@ name: Release bdist wheel
 on:
   workflow_dispatch:
     inputs:
-      version:
+      cuda_version:
         type: choice
-        description: version for testing
+        description: CUDA Version
         default: 'all'
         required: true
         options:
         - all
-        - pytorch-cuda:1.9.0-11.1.1 # python 3.8
-        - pytorch-cuda:1.8.1-11.1.1 # python 3.8
-        - pytorch-cuda:1.7.1-11.0.3 # python 3.8
-        - pytorch-cuda:1.6.0-10.2 # python 3.6
-
+        - "11.3"
+        - "11.1"
+        - "10.2"
+      github_ref:
+        type: string
+        description: Branch or Tag
+        default: 'main'
+        required: true
 
 jobs:
   matrix_preparation:
@@ -25,9 +28,9 @@ jobs:
     steps:
     - id: set-matrix
       run: |
-        [ "${{github.event.inputs.version}}" != "" ] && matrix="[\"frankleeeee/${{github.event.inputs.version}}\"]"
-        [ "${{github.event.inputs.version}}" == "" ] || [ "${{github.event.inputs.version}}" == "all" ] && \
-          matrix="[\"frankleeeee/pytorch-cuda:1.9.0-11.1.1\", \"frankleeeee/pytorch-cuda:1.8.1-11.1.1\", \"frankleeeee/pytorch-cuda:1.7.1-11.0.3\", \"frankleeeee/pytorch-cuda:1.6.0-10.2\"]"
+        [ "${{github.event.inputs.cuda_version}}" != "" ] && matrix="[\"frankleeeee/cuda-conda:${{github.event.inputs.cuda_version}}\"]"
+        [ "${{github.event.inputs.cuda_version}}" == "" ] || [ "${{github.event.inputs.version}}" == "all" ] && \
+          matrix="[\"frankleeeee/cuda-conda:11.3\", \"frankleeeee/cuda-conda:11.1\", \"frankleeeee/cuda-conda:10.2\"]"
         echo $matrix
         echo "::set-output name=matrix::{\"container\":$(echo $matrix)}"
 
@@ -42,20 +45,24 @@ jobs:
     container:
       image: ${{ matrix.container }}
       options: --gpus all --rm
-    timeout-minutes: 120
     steps:
-      - name: Install dependencies
-        run: |
-          pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
-          pip install -U pip setuptools wheel --user
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Copy scripts and checkout
+        run: |
+          cp -r ./.github/workflows/scripts/* ./
+          git checkout $git_ref
+        env:
+          git_ref: ${{ github.event.inputs.github_ref }}
       - name: Build bdist wheel
         run: |
-          python setup.py bdist_wheel
+          pip install beautifulsoup4 requests
+          python ./build_colossalai_wheel.py
       - name: 🚀 Deploy
         uses: garygrossgarten/github-action-scp@release
         with:
-          local: dist
+          local: all_dist
           remote: ${{ secrets.PRIVATE_PYPI_DIR }}
           host: ${{ secrets.PRIVATE_PYPI_HOST }}
           username: ${{ secrets.PRIVATE_PYPI_USER }}

@@ -0,0 +1,74 @@
+import requests
+from bs4 import BeautifulSoup
+import re
+import os
+import subprocess
+
+
+WHEEL_TEXT_ROOT_URL = 'https://github.com/hpcaitech/public_assets/tree/main/colossalai/torch_wheel/wheel_urls'
+RAW_TEXT_FILE_PREFIX = 'https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/torch_wheel/wheel_urls'
+CUDA_HOME = os.environ['CUDA_HOME']
+
+def get_cuda_bare_metal_version():
+
+    raw_output = subprocess.check_output([CUDA_HOME + "/bin/nvcc", "-V"], universal_newlines=True)
+    output = raw_output.split()
+    release_idx = output.index("release") + 1
+    release = output[release_idx].split(".")
+    bare_metal_major = release[0]
+    bare_metal_minor = release[1][0]
+
+    return bare_metal_major, bare_metal_minor
+
+def all_wheel_info():
+    page_text = requests.get(WHEEL_TEXT_ROOT_URL).text
+    soup = BeautifulSoup(page_text)
+
+    all_a_links = soup.find_all('a')
+
+    wheel_info = dict()
+
+    for a_link in all_a_links:
+        if 'cu' in a_link.text and '.txt' in a_link.text:
+            filename = a_link.text
+            torch_version, cuda_version = filename.rstrip('.txt').split('-')
+
+            wheel_info[torch_version] = dict()
+            wheel_info[torch_version][cuda_version] = dict()
+
+            file_text = requests.get(f'{RAW_TEXT_FILE_PREFIX}/{filename}').text
+            wheel_urls = file_text.strip().split('\n')
+
+            for url in wheel_urls:
+                for part in url.split('-'):
+                    if re.search('cp\d+', part):
+                        python_version = f"3.{part.lstrip('cp3')}"
+                        wheel_info[torch_version][cuda_version][python_version] = url
+                        break
+    return wheel_info
+
+
+def build_colossalai(wheel_info):
+    cuda_version_major, cuda_version_minor = get_cuda_bare_metal_version()
+    cuda_version_on_host = f'cu{cuda_version_major}{cuda_version_minor}'
+
+    for torch_version, cuda_versioned_wheel_info in wheel_info.items():
+        for cuda_version, python_versioned_wheel_info in cuda_versioned_wheel_info.items():
+            if cuda_version_on_host == cuda_version:
+                for python_version, torch_wheel_url in python_versioned_wheel_info.items():
+                    filename = torch_wheel_url.split('/')[-1]
+                    cmd = f'bash ./build_colossalai_wheel.sh {python_version} {torch_wheel_url} {filename}'
+                    os.system(cmd)
+
+def main():
+    wheel_info = all_wheel_info()
+    build_colossalai(wheel_info)
+
+if __name__ == '__main__':
+    main()
+
+
+
+
+
+
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+git reset --hard HEAD
+mkdir -p ./all_dist
+source activate base
+conda create -n $1 -y python=$1
+source activate $1
+wget -q $2
+ls | grep torch | xargs pip install
+python setup.py bdist_wheel
+mv ./dist/* ./all_dist
+ls | grep torch | xargs rm
+python setup.py clean
+conda env remove -n $1
+
+