Adding nightly-publish task

This task publishes artifacts needed by consumers to use the nightly, including: - maven.json: where were the maven packages published? - swift-components.zip: generated swift files + xcframework - nimbus-fml.zip / nimbus-fml.sha256: nimbus-fml binary Currently, nightly builds use a different maven group id (org.mozilla.appservices.nightly) to separate them from "real" builds, although this may change based on feedback from the Android team Don't try to publish maven packages using the nightly repos. The Android team prefers them in the regular repos. Small fix to make exceptions print out correctly for NimbusGradlePlugin. I was seeing a lot of download errors when testing this. Building swift-components.zip required building on a mac worker, which required some extra work/hacking - Added a `run-commands` job implementation that works on `generic-worker`. - The ios tests are not working yet, so I just commented them out. - For now the entire build happens in one task. Eventually we should split it up, maybe into toolchain task and a separate build/test task for Fenix/Focus. - Updated several of our scripts to use env vars rather than hardcoding things based on the docker-worker.
Sajjon · Apr 21, 2023 · 7914164 · 7914164
1 parent 25685f8
commit 7914164
Show file tree

Hide file tree

Showing 28 changed files with 605 additions and 48 deletions.
diff --git a/build-scripts/substitute-local-appservices.gradle b/build-scripts/substitute-local-appservices.gradle
@@ -37,7 +37,7 @@ configurations.all { config ->
                     }
 
                     def group = dependency.requested.group
-                    if (group == 'org.mozilla.appservices') {
+                    if (group == 'org.mozilla.appservices' || group == 'org.mozilla.appservices.nightly') {
                         def name = dependency.requested.module
                         dependency.useTarget([group: group, name: name, version: version])
                     }

diff --git a/settings.gradle b/settings.gradle
@@ -80,6 +80,15 @@ def calcVersion(buildconfig) {
     }
 }
 
+def calcGroupId(buildconfig) {
+    if (gradle.rootProject.hasProperty("nightlyVersion")) {
+        return buildconfig.groupId + ".nightly"
+    } else {
+        return buildconfig.groupId
+    }
+}
+
+
 gradle.projectsLoaded { ->
     // Wait until root project is "loaded" before we set "config"
     // Note that since this is set on "rootProject.ext", it will be "in scope" during the evaluation of all projects'
@@ -90,6 +99,6 @@ gradle.projectsLoaded { ->
         // which can be depended on specifically by the ./build-scripts/substitute-local-appservices.gradle
         // but which is unlikely to be depended on by accident otherwise.
         version: calcVersion(buildconfig),
-        groupId: buildconfig.groupId,
+        groupId: calcGroupId(buildconfig),
     ]
 }
diff --git a/taskcluster/app_services_taskgraph/beetmover.py b/taskcluster/app_services_taskgraph/beetmover.py
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+def get_maven_bucket(params):
+    if params['level'] == '3':
+        return "maven-production"
+    else:
+        return "maven-staging"
diff --git a/taskcluster/app_services_taskgraph/job.py b/taskcluster/app_services_taskgraph/job.py
@@ -42,6 +42,7 @@
     Optional("use-caches"): bool,
     Optional("secrets"): [secret_schema],
     Optional("dummy-secrets"): [dummy_secret_schema],
+    Optional("run-task-command"): [str],
 })
 
 
@@ -63,6 +64,24 @@ def configure_run_commands_schema(config, job, taskdesc):
     _set_run_task_attributes(job)
     configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
 
+@run_job_using("generic-worker", "run-commands", schema=run_commands_schema)
+def configure_run_commands_schema_generic(config, job, taskdesc):
+    run = job["run"]
+    pre_commands = run.pop("pre-commands", [])
+    pre_commands += [
+        _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", [])
+    ]
+    pre_commands += [
+        _generate_secret_command(secret) for secret in run.get("secrets", [])
+    ]
+
+    all_commands = pre_commands + run.pop("commands", [])
+
+    run["command"] = _convert_commands_to_string(all_commands)
+    _inject_secrets_scopes(run, taskdesc)
+    _set_run_task_attributes(job)
+    configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
+
 
 @run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
 def configure_gradlew(config, job, taskdesc):

diff --git a/taskcluster/app_services_taskgraph/transforms/__init__.py b/taskcluster/app_services_taskgraph/transforms/__init__.py
@@ -32,7 +32,7 @@ def publications_to_artifact_map_paths(name, version, publications, preview_buil
             artifact_filename = _artifact_filename(publication_name, version, extension)
             if preview_build is not None:
                 # Both nightly and other preview builds are places in separate directory
-                destination = "maven2/org/mozilla/appservices/nightlies/{}/{}/{}".format(publication_name, version, artifact_filename)
+                destination = "maven2/org/mozilla/appservices/nightly/{}/{}/{}".format(publication_name, version, artifact_filename)
             else:
                 destination = "maven2/org/mozilla/appservices/{}/{}/{}".format(publication_name, version, artifact_filename)
             build_map_paths[f"public/build/{artifact_filename}"] = {

diff --git a/taskcluster/app_services_taskgraph/transforms/appservices.py b/taskcluster/app_services_taskgraph/transforms/appservices.py
@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from taskgraph.transforms.base import TransformSequence
+
+from ..build_config import get_version
+
+transforms = TransformSequence()
+
+@transforms.add
+def transform_routes(config, tasks):
+    version = get_version(config.params)
+    for task in tasks:
+        task["routes"] = [
+            route.replace("{appservices_version}", version)
+            for route in task.get("routes", [])
+        ]
+        yield task
+
diff --git a/taskcluster/app_services_taskgraph/transforms/beetmover.py b/taskcluster/app_services_taskgraph/transforms/beetmover.py
@@ -10,6 +10,7 @@
     publications_to_artifact_paths, publications_to_artifact_map_paths
 )
 from ..build_config import get_version
+from ..beetmover import get_maven_bucket
 
 transforms = TransformSequence()
 
@@ -71,20 +72,7 @@ def beetmover_task(config, tasks):
         task["worker"]["max-run-time"] = 600
         task["worker"]["version"] = get_version(config.params)
         task["description"] = task["description"].format(task["attributes"]["buildconfig"]["name"])
-        if config.params['level'] == '3':
-            if config.params.get('preview-build') is None:
-                task["worker"]["bucket"] = "maven-production"
-            else:
-                # Once we setup access, these should be published to the nightly maven repos
-                # task["worker"]["bucket"] = "maven-nightly-production"
-                task["worker"]["bucket"] = "maven-production"
-        else:
-            if config.params.get('preview-build') is None:
-                task["worker"]["bucket"] = "maven-staging"
-            else:
-                # Once we setup access, these should be published to the nightly maven repos
-                # task["worker"]["bucket"] = "maven-nightly-staging"
-                task["worker"]["bucket"] = "maven-staging"
+        task["worker"]["bucket"] = get_maven_bucket(config.params)
         yield task
 
 

diff --git a/taskcluster/app_services_taskgraph/transforms/module_build.py b/taskcluster/app_services_taskgraph/transforms/module_build.py
@@ -43,6 +43,11 @@ def release_upload_symbols(config, tasks):
 
 @transforms.add
 def build_task(config, tasks):
+    if config.params.get("preview-build") is None:
+        path_prefix = "/builds/worker/checkouts/vcs/build/maven/org/mozilla/appservices/"
+    else:
+        path_prefix = "/builds/worker/checkouts/vcs/build/maven/org/mozilla/appservices/nightly"
+
     for task in tasks:
         module_info = task["attributes"]["buildconfig"]
         name = module_info["name"]
@@ -61,7 +66,7 @@ def build_task(config, tasks):
                 artifact_filename = f"{publication_name}-{version}{extension}"
                 artifacts.append({
                     "name": f"public/build/{artifact_filename}",
-                    "path": f"/builds/worker/checkouts/vcs/build/maven/org/mozilla/appservices/{publication_name}/{version}/{artifact_filename}",
+                    "path": f"{path_prefix}/{publication_name}/{version}/{artifact_filename}",
                     "type": "file",
                 })
 

diff --git a/taskcluster/app_services_taskgraph/transforms/nightly_publish.py b/taskcluster/app_services_taskgraph/transforms/nightly_publish.py
@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from taskgraph.transforms.base import TransformSequence
+
+from ..build_config import get_version
+from ..beetmover import get_maven_bucket
+
+transforms = TransformSequence()
+
+@transforms.add
+def setup_command(config, tasks):
+    version = get_version(config.params)
+    if config.params['level'] == '3':
+        if config.params.get('preview-build') is None:
+            maven_channel = "maven-production"
+        else:
+            maven_channel = "maven-nightly-production"
+    else:
+        if config.params.get('preview-build') is None:
+            maven_channel = "maven-staging"
+        else:
+            maven_channel = "maven-nightly-staging"
+
+    for task in tasks:
+        task["run"]["commands"] = [
+           [
+               "/builds/worker/checkouts/vcs/taskcluster/scripts/generate-nightly-json.py",
+               "/builds/worker/checkouts/vcs/build/nightly.json",
+               "--version", version,
+               "--maven-channel", maven_channel,
+           ]
+       ]
+        task["routes"] = [
+            "index.project.application-services.v2.nightly.latest",
+            f"index.project.application-services.v2.nightly.{version}",
+        ]
+        yield task
diff --git a/taskcluster/ci/android-build/kind.yml b/taskcluster/ci/android-build/kind.yml
@@ -33,7 +33,7 @@ tasks:
         - [git, submodule, update, --init]
         - [source, taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh]
         - [source, taskcluster/scripts/toolchain/cross-compile-setup.sh]
-        - [rsync, '-a', /builds/worker/fetches/libs/, /builds/worker/checkouts/vcs/libs/]
+        - [source, taskcluster/scripts/toolchain/copy-libs-dir.sh, libs]
         - [bash, '-c', 'echo "rust.targets=linux-x86-64\n" > local.properties']
       gradlew:
         - 'clean'

diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml
@@ -20,6 +20,11 @@ workers:
       implementation: docker-worker
       os: linux
       worker-type: 'b-linux-gcp'
+    b-osx:
+      provisioner: releng-hardware
+      implementation: generic-worker
+      os: macosx
+      worker-type: applicationservices-b-1-osx1015
     images:
       provisioner: 'app-services-{level}'
       implementation: docker-worker

diff --git a/taskcluster/ci/module-build/kind.yml b/taskcluster/ci/module-build/kind.yml
@@ -33,7 +33,7 @@ task-defaults:
     pre-gradlew:
       - [source, taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh]
       - [source, taskcluster/scripts/toolchain/cross-compile-setup.sh]
-      - [rsync, '-a', /builds/worker/fetches/libs/, /builds/worker/checkouts/vcs/libs/]
+      - [source, taskcluster/scripts/toolchain/copy-libs-dir.sh, libs]
       - [bash, '-c', 'echo "rust.targets=arm,arm64,x86_64,x86,darwin,linux-x86-64\n" > local.properties']
     gradlew:
       - ':{module_name}:assembleRelease'

diff --git a/taskcluster/ci/nightly-publish/kind.yml b/taskcluster/ci/nightly-publish/kind.yml
@@ -0,0 +1,41 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+# nightly-publish: publish artifacts for successful nightly builds
+#
+# This task publishes the nightly.json file for the nightly builds.
+# nightly.json is used by consumer apps for their nightly app-services version
+# bump.
+#
+# This task only runs if nightly-summary succeeds, which implies that all other
+# tasks suceeded.  This task should not fail, since that could leave the
+# nightly artifacts in an inconsistent state.
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+  - nightly-summary
+
+transforms:
+  - app_services_taskgraph.transforms.nightly_publish:transforms
+  - taskgraph.transforms.job:transforms
+  - taskgraph.transforms.task:transforms
+
+tasks:
+  nightly_publish:
+    attributes:
+      nightly: nightly-only
+    label: "Nightly build publish task"
+    description: "Publish nightly artifacts"
+    worker-type: b-linux
+    worker:
+      chain-of-trust: true
+      docker-image: { in-tree: linux }
+      max-run-time: 1800
+      env: {}
+      artifacts:
+        - name: "public/build/nightly.json"
+          path: "/builds/worker/checkouts/vcs/build/nightly.json"
+          type: "file"
+    run:
+        using: run-commands
diff --git a/taskcluster/ci/nightly-complete/kind.yml → taskcluster/ci/nightly-summary/kind.yml b/taskcluster/ci/nightly-complete/kind.yml → taskcluster/ci/nightly-summary/kind.yml
@@ -5,9 +5,11 @@
 loader: taskgraph.loader.transform:loader
 
 kind-dependencies:
-  - branch-build-as
-  - branch-build-firefox-android
-  - branch-build-fenix
+  - module-build
+  - signing
+  - beetmover
+  - swift
+  - nimbus-fml
 
 transforms:
   - app_services_taskgraph.transforms.deps_complete:transforms

diff --git a/taskcluster/ci/nimbus-fml/kind.yml b/taskcluster/ci/nimbus-fml/kind.yml
@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# ⚠️ If you add, rename or delete a task here, please also update .mergify.yml! ⚠️
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+  - app_services_taskgraph.transforms.appservices:transforms
+  - taskgraph.transforms.job:transforms
+  - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - toolchain
+
+tasks:
+  build:
+    attributes:
+      run-on-pr-type: all
+      resource-monitor: true
+    needs-sccache: false # TODO: Bug 1623426 deal with this once we're in prod
+    run-on-tasks-for: [github-pull-request, github-release, cron]
+    description: Build and test (Swift)
+    scopes:
+      - project:releng:services/tooltool/api/download/internal
+    worker-type: b-linux
+    worker:
+      docker-image: { in-tree: linux }
+      max-run-time: 1800
+      artifacts:
+        - name: "public/build/nimbus-fml.zip"
+          path: "/builds/worker/checkouts/vcs/build/nimbus-fml.zip"
+          type: "file"
+        - name: "public/build/nimbus-fml.sha256"
+          path: "/builds/worker/checkouts/vcs/build/nimbus-fml.sha256"
+          type: "file"
+    run:
+      pre-commands:
+        - [git, submodule, update, --init]
+        - [source, taskcluster/scripts/toolchain/setup-fetched-rust-toolchain.sh]
+        - [rsync, '-a', /builds/worker/fetches/libs/, /builds/worker/checkouts/vcs/libs/]
+      commands:
+        - [
+            "/builds/worker/checkouts/vcs/taskcluster/scripts/build-nimbus-fml.py",
+            "/builds/worker/checkouts/vcs/build/",
+          ]
+      using: run-commands
+      use-caches: true
+    fetches:
+      toolchain:
+        - desktop-linux-libs
+        - rust
+    routes:
+      - index.project.application-services.v2.nimbus-fml.{appservices_version}
diff --git a/taskcluster/ci/swift/kind.yml b/taskcluster/ci/swift/kind.yml
@@ -0,0 +1,56 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# ⚠️ If you add, rename or delete a task here, please also update .mergify.yml! ⚠️
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+  - app_services_taskgraph.transforms.appservices:transforms
+  - taskgraph.transforms.job:transforms
+  - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - toolchain
+
+tasks:
+  build:
+    attributes:
+      run-on-pr-type: all
+      resource-monitor: true
+    needs-sccache: false # TODO: Bug 1623426 deal with this once we're in prod
+    run-on-tasks-for: [github-pull-request, github-release, cron]
+    description: Build and test (Swift)
+    scopes:
+      - project:releng:services/tooltool/api/download/internal
+    worker-type: b-osx
+    worker:
+      max-run-time: 1800
+      artifacts:
+        - name: "public/build/swift-components.tar.xz"
+          path: "checkouts/vcs/build/swift-components.tar.xz"
+          type: "file"
+        - name: "public/build/MozillaRustComponents.xcframework.zip"
+          path: "checkouts/vcs/build/MozillaRustComponents.xcframework.zip"
+          type: "file"
+        - name: "public/build/FocusRustComponents.xcframework.zip"
+          path: "checkouts/vcs/build/FocusRustComponents.xcframework.zip"
+          type: "file"
+        # TODO: re-enable once we get tests working
+        # - name: "public/build/raw_xcodetest.log"
+        #   path: "checkouts/vcs/logs/raw_xcodetest.log"
+        #   type: "file"
+    run:
+      pre-commands:
+        - ["taskcluster/scripts/toolchain/build-rust-toolchain-macosx.sh"]
+        - ["taskcluster/scripts/toolchain/libs-ios.sh"]
+      commands:
+        - ["taskcluster/scripts/build-and-test-swift.sh"]
+        - [ "cd", "build/" ]
+        - [ "tar", "acf", "swift-components.tar.xz", "swift-components" ]
+      using: run-commands
+      run-task-command: ["/usr/local/bin/python3", "run-task"]
+      use-caches: true
+    routes:
+      - index.project.application-services.v2.swift.{appservices_version}