Skip Building on incompatible Python Versions (Azure#7160)

* removing --universal argument in build_packages.py * extending common_tasks.py. now we filter package set ALSO by CI compatibility * adding additional dependencies during build to allow targeting to successfully work
florentx · Sep 11, 2019 · 7354bbd · 7354bbd
1 parent b3790cd
commit 7354bbd
Show file tree

Hide file tree

Showing 9 changed files with 88 additions and 16 deletions.
diff --git a/build_package.py b/build_package.py
@@ -19,7 +19,7 @@ def create_package(name, dest_folder=DEFAULT_DEST_FOLDER):
     absdirs = [os.path.dirname(package) for package in (glob.glob('{}/setup.py'.format(name)) + glob.glob('sdk/*/{}/setup.py'.format(name)))]
 
     absdirpath = os.path.abspath(absdirs[0])
-    check_call(['python', 'setup.py', 'bdist_wheel', '--universal', '-d', dest_folder], cwd=absdirpath)
+    check_call(['python', 'setup.py', 'bdist_wheel', '-d', dest_folder], cwd=absdirpath)
     check_call(['python', 'setup.py', "sdist", "--format", "zip", '-d', dest_folder], cwd=absdirpath)
 
 

diff --git a/doc/dev/engineering_assumptions.md b/doc/dev/engineering_assumptions.md
@@ -1,6 +1,12 @@
 ## Engineering System Assumptions, Gotchas, and Minutiae
 
-1. All wheels are generated with `--universal` flag passed to `bdist_wheel`. We do not currently allow non-universal packages to be shipped out of this repository.
+1. All wheels are generated with influence from `setup.cfg` at the built package root. This means that for most of our packages, the `universal` flag is set within the `setup.cfg`.
+
+```
+[bdist_wheel]
+universal=1
+```
+2. If a package does _not_ have `universal` set, then the default build will produce a wheel for `py3`. This is due to the fact that wheels are currently generated by a CI step running `Python 3.6`.
 
 ## Build
 
@@ -15,3 +21,7 @@ Build CI for `azure-sdk-for-python` essentially builds and tests packages in one
 1. Install on packages (and their dev_requirements!) in one go.
 2. Run `pytest <folder1>, pytest <folder2>` where folders correspond to package folders
     1. While all packages are installed alongside each other, each test run is individual to the package. This has the benefit of not allowing `packageA`'s `conftest.py` to mess with `packageB`'s environment.'
+
+### So when do they run?
+
+A standard pull request will option target the `Individual Packages` option. On a nightly cadence, the `python - client` build installs the entire world and tests in _both_ methodologies outlined above.
diff --git a/eng/pipelines/templates/steps/build-artifacts.yml b/eng/pipelines/templates/steps/build-artifacts.yml
@@ -15,7 +15,7 @@ steps:
       versionSpec: $(PythonVersion)
 
   - script: |
-      pip install wheel setuptools pathlib twine readme-renderer[md]
+      pip install wheel setuptools pathlib twine readme-renderer[md] packaging
     displayName: 'Prep Environment'
 
   - task: PythonScript@0

diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml
@@ -35,7 +35,7 @@ steps:
       arguments: ${{ parameters.OSName }}
 
   - script: |
-      pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo
+      pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo packaging
     displayName: 'Prep Environment'
     
   - ${{ parameters.BeforeTestSteps }}

diff --git a/eng/pipelines/templates/steps/test-nightly.yml b/eng/pipelines/templates/steps/test-nightly.yml
@@ -34,7 +34,7 @@ steps:
       arguments: ${{ parameters.OSName }}
 
   - script: |
-      pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo
+      pip install pathlib twine codecov beautifulsoup4 tox tox-monorepo packaging
     displayName: 'Prep Environment'
 
   - ${{ parameters.BeforeTestSteps }}

diff --git a/eng/tox/create_wheel_and_install.py b/eng/tox/create_wheel_and_install.py
@@ -59,7 +59,6 @@ def cleanup_build_artifacts(build_folder):
             "python",
             os.path.join(args.target_setup, "setup.py"),
             "bdist_wheel",
-            "--universal",
             "-d",
             args.distribution_directory,
         ]

diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py
@@ -9,36 +9,44 @@
 # package targeting during release.
 
 import glob
-from pathlib import Path
 from subprocess import check_call, CalledProcessError
 import os
 import errno
 import shutil
 import sys
 import logging
+import ast
+import textwrap
+import io
+import re
+import pdb
+
+# ssumes the presence of setuptools
+from pkg_resources import parse_version
+
+# this assumes the presence of "packaging"
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
+
 
 logging.getLogger().setLevel(logging.INFO)
 
 OMITTED_CI_PACKAGES = ["azure-mgmt-documentdb", "azure-servicemanagement-legacy"]
-
 MANAGEMENT_PACKAGE_IDENTIFIERS = [
     "mgmt",
     "azure-cognitiveservices",
     "azure-servicefabric",
     "nspkg"
 ]
 
-
 def log_file(file_location, is_error=False):
     with open(file_location, "r") as file:
         for line in file:
             sys.stdout.write(line)
         sys.stdout.write("\n")
-        # CI consistently sees outputs in the wrong location. Trying this to see if it helps
         sys.stdout.flush()
 
 
-
 def read_file(file_location):
     str_buffer = ""
     with open(file_location, "r") as file:
@@ -68,6 +76,57 @@ def clean_coverage(coverage_dir):
         else:
             raise
 
+def parse_setup_requires(setup_path):
+    setup_filename = os.path.join(setup_path, 'setup.py')
+    mock_setup = textwrap.dedent('''\
+    def setup(*args, **kwargs):
+        __setup_calls__.append((args, kwargs))
+    ''')
+    parsed_mock_setup = ast.parse(mock_setup, filename=setup_filename)
+    with io.open(setup_filename, 'r', encoding='utf-8-sig') as setup_file:
+        parsed = ast.parse(setup_file.read())
+        for index, node in enumerate(parsed.body[:]):
+            if (
+                not isinstance(node, ast.Expr) or
+                not isinstance(node.value, ast.Call) or
+                not hasattr(node.value.func, 'id') or
+                node.value.func.id != 'setup'
+            ):
+                continue
+            parsed.body[index:index] = parsed_mock_setup.body
+            break
+
+    fixed = ast.fix_missing_locations(parsed)
+    codeobj = compile(fixed, setup_filename, 'exec')
+    local_vars = {}
+    global_vars = {'__setup_calls__': []}
+    current_dir = os.getcwd()
+    working_dir = os.path.dirname(setup_filename)
+    os.chdir(working_dir)
+    exec(codeobj, global_vars, local_vars)
+    os.chdir(current_dir)
+    _, kwargs = global_vars['__setup_calls__'][0]
+
+    try:
+        python_requires = kwargs['python_requires']
+    # most do not define this, fall back to what we define as universal
+    except KeyError as e:
+        python_requires = ">=2.7"
+
+    return python_requires
+
+def filter_for_compatibility(package_set):
+    collected_packages = []
+    v = sys.version_info
+    running_major_version = Version(".".join([str(v[0]), str(v[1]), str(v[2])]))
+
+    for pkg in package_set:
+        spec_set = SpecifierSet(parse_setup_requires(pkg))
+
+        if running_major_version in spec_set:
+            collected_packages.append(pkg)
+
+    return collected_packages
 
 # this function is where a glob string gets translated to a list of packages
 # It is called by both BUILD (package) and TEST. In the future, this function will be the central location
@@ -91,12 +150,17 @@ def process_glob_string(glob_string, target_root_dir):
     # if we have individually queued this specific package, it's obvious that we want to build it specifically
     # in this case, do not honor the omission list
     if len(collected_directories) == 1:
-        return collected_directories
+        return filter_for_compatibility(collected_directories)
     # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted
     # packages
     else:
-        return sorted(remove_omitted_packages(collected_directories))
+        allowed_package_set = remove_omitted_packages(collected_directories)
+        logging.info("Target packages after filtering by omission list: {}".format(allowed_package_set))
 
+        pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set)
+        logging.info("Package(s) omitted by CI filter: {}".format(list(set(allowed_package_set) - set(pkg_set_ci_filtered))))
+
+        return sorted(pkg_set_ci_filtered)
 
 def remove_omitted_packages(collected_directories):
     return [
@@ -105,7 +169,6 @@ def remove_omitted_packages(collected_directories):
         if os.path.basename(package_dir) not in OMITTED_CI_PACKAGES
     ]
 
-
 def run_check_call(
     command_array,
     working_directory,

diff --git a/sdk/redis/azure-mgmt-redis/dev_requirements.txt b/sdk/redis/azure-mgmt-redis/dev_requirements.txt
@@ -1 +1 @@
--e ../../../tools/azure-sdk-tools
+-e ../../../tools/azure-sdk-tools
diff --git a/sdk/storage/azure-storage-file/dev_requirements.txt b/sdk/storage/azure-storage-file/dev_requirements.txt
@@ -2,4 +2,4 @@
 -e ../../../tools/azure-sdk-tools
 -e ../../core/azure-core
 aiohttp>=3.0; python_version >= '3.5'
-pytest
+pytest
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		-e ../../../tools/azure-sdk-tools
		-e ../../../tools/azure-sdk-tools