From 02355a9260b8a58286cb82914e24697f37d83b0c Mon Sep 17 00:00:00 2001 From: Pieter-Jan Briers Date: Thu, 4 Mar 2021 02:17:46 +0100 Subject: [PATCH] Move content builds to github actions and centcomm --- .github/workflows/fire-jenkins.yml | 25 ---- .github/workflows/publish.yml | 49 ++++++++ Jenkinsfile | 32 ----- Resources/Changelog/Parts/builds.yml | 4 + SpaceStation14.sln | 1 - Tools/gen_build_info.py | 17 ++- Tools/package_server_build.py | 168 +++++---------------------- 7 files changed, 87 insertions(+), 209 deletions(-) delete mode 100644 .github/workflows/fire-jenkins.yml create mode 100644 .github/workflows/publish.yml delete mode 100644 Jenkinsfile create mode 100644 Resources/Changelog/Parts/builds.yml diff --git a/.github/workflows/fire-jenkins.yml b/.github/workflows/fire-jenkins.yml deleted file mode 100644 index f219489bee..0000000000 --- a/.github/workflows/fire-jenkins.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Fire Jenkins - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -on: - push: - branches: [ master ] - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Runs a single command using the runners shell - - name: Fire Jenkins - run: curl -X POST -d "" -u "$JENKINS_AUTH" "https://builds.spacestation14.io/jenkins/job/SS14%20Content/build?token=$JENKINS_SS14_CONTENT_BUILD_TOKEN&cause=GitHub+actions+push" - env: - JENKINS_AUTH: ${{ secrets.JENKINS_AUTH }} - JENKINS_SS14_CONTENT_BUILD_TOKEN: ${{ secrets.JENKINS_SS14_CONTENT_BUILD_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..e1a28858ae --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,49 @@ +name: Publish + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Setup .NET Core + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 5.0.200 + + - name: Package client + run: | + Tools/package_server_build.py -p win-x64 -p linux-x64 -p osx-x64 -p linux-arm64 + Tools/package_client_build.py + + - name: Shuffle files around + run: | + mkdir "release/${{ github.sha }}" + mv release/*.zip "release/${{ github.sha }}" + + - name: Upload files to centcomm + uses: appleboy/scp-action@master + with: + host: centcomm.spacestation14.io + username: wizards-build-push + key: ${{ secrets.CENTCOMM_WIZARDS_BUILDS_PUSH_KEY }} + source: "release/${{ github.sha }}" + target: "/mnt/ext/wizards-builds/builds/" + strip_components: 1 + + - name: Update manifest JSON + uses: appleboy/ssh-action@master + with: + host: centcomm.spacestation14.io + username: wizards-build-push + key: ${{ secrets.CENTCOMM_WIZARDS_BUILDS_PUSH_KEY }} + script: /home/wizards-build-push/push.ps1 ${{ github.sha }} + diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index fd90f90fa7..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,32 +0,0 @@ -pipeline { - agent any - - stages { - stage('Setup') { - steps { - sh 'git submodule update --init --recursive' - // Do a git fetch to make sure tags in the engine get pulled in if they've been added later. - // Can happen if somebody forgot to tag the engine then tried to fix it by tagging later. - sh 'cd RobustToolbox && git fetch && cd ..' - } - } - stage('Build') { - steps { - sh 'Tools/package_server_build.py -p windows mac linux linux-arm64' - sh 'Tools/package_client_build.py' - } - } - stage('Update build info') { - steps { - sh 'Tools/gen_build_info.py' - archiveArtifacts artifacts: 'release/*.zip' - } - } - stage('Generate checksums') { - steps { - sh 'Tools/generate_hashes.ps1' - archiveArtifacts artifacts: 'release/*.zip.sha256' - } - } - } -} diff --git a/Resources/Changelog/Parts/builds.yml b/Resources/Changelog/Parts/builds.yml new file mode 100644 index 0000000000..10bf3e8ef4 --- /dev/null +++ b/Resources/Changelog/Parts/builds.yml @@ -0,0 +1,4 @@ +author: PJB +changes: +- message: Moved content builds over to a better server and threw it behind Cloudflare caching. Should download MUCH faster now! + type: Tweak diff --git a/SpaceStation14.sln b/SpaceStation14.sln index 96eafa743e..00ce147df8 100644 --- a/SpaceStation14.sln +++ b/SpaceStation14.sln @@ -58,7 +58,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Build", "Build", "{806ED41A .appveyor.yml = .appveyor.yml Tools\gen_build_info.py = Tools\gen_build_info.py Tools\generate_hashes.ps1 = Tools\generate_hashes.ps1 - Jenkinsfile = Jenkinsfile Tools\package_client_build.py = Tools\package_client_build.py Tools\package_server_build.py = Tools\package_server_build.py EndProjectSection diff --git a/Tools/gen_build_info.py b/Tools/gen_build_info.py index e5368d4716..6c3878ba55 100755 --- a/Tools/gen_build_info.py +++ b/Tools/gen_build_info.py @@ -11,14 +11,15 @@ FILE = "SS14.Client.zip" SERVER_FILES = [ - "SS14.Server_Linux_x64.zip", - "SS14.Server_Linux_ARM64.zip", - "SS14.Server_Windows_x64.zip", - "SS14.Server_macOS_x64.zip" + "SS14.Server_linux-x64.zip", + "SS14.Server_linux-arm64.zip", + "SS14.Server_win-x64.zip", + "SS14.Server_osx-x64.zip" ] +VERSION = os.environ['GITHUB_SHA'] FORK_ID = "wizards" - +BUILD_URL = f"https://central.spacestation14.io/builds/wizards/builds/{VERSION}/{FILE}" def main() -> None: manifest = generate_manifest("release") @@ -35,15 +36,13 @@ def inject_manifest(zip_path: str, manifest: str) -> None: def generate_manifest(dir: str) -> str: # Env variables set by Jenkins. - version = os.environ["BUILD_NUMBER"] - download = f"{os.environ['BUILD_URL']}artifact/release/{FILE}" hash = sha256_file(os.path.join(dir, FILE)) engine_version = get_engine_version() return json.dumps({ - "download": download, + "download": BUILD_URL, "hash": hash, - "version": version, + "version": VERSION, "fork_id": FORK_ID, "engine_version": engine_version }) diff --git a/Tools/package_server_build.py b/Tools/package_server_build.py index 751ab79603..0821b27128 100755 --- a/Tools/package_server_build.py +++ b/Tools/package_server_build.py @@ -23,19 +23,23 @@ def __getattr__(self, name): Fore = ColorDummy() Style = ColorDummy() +class PlatformReg: + def __init__(self, rid: str, target_os: str): + self.rid = rid + self.target_os = target_os p = os.path.join -PLATFORM_WINDOWS = "windows" -PLATFORM_LINUX = "linux" -PLATFORM_LINUX_ARM64 = "linux-arm64" -PLATFORM_MACOS = "mac" +PLATFORMS = [ + PlatformReg("win-x64", "Windows"), + PlatformReg("linux-x64", "Linux"), + PlatformReg("linux-arm64", "Linux"), + PlatformReg("osx-x64", "MacOS"), +] + +PLATFORM_RIDS = {x.rid for x in PLATFORMS} SHARED_IGNORED_RESOURCES = { - "ss13model.7z", - "ResourcePack.zip", - "buildResourcePack.py", - "CONTENT_GOES_HERE", ".gitignore", ".directory", ".DS_Store" @@ -45,8 +49,6 @@ def __getattr__(self, name): "Textures", "Fonts", "Audio", - "Scenes", - "Nano", "Shaders", } @@ -70,7 +72,7 @@ def main() -> None: parser.add_argument("--platform", "-p", action="store", - choices=[PLATFORM_WINDOWS, PLATFORM_MACOS, PLATFORM_LINUX, PLATFORM_LINUX_ARM64], + choices=PLATFORM_RIDS, nargs="*", help="Which platform to build for. If not provided, all platforms will be built") @@ -83,7 +85,7 @@ def main() -> None: skip_build = args.skip_build if not platforms: - platforms = [PLATFORM_WINDOWS, PLATFORM_MACOS, PLATFORM_LINUX, PLATFORM_LINUX_ARM64] + platforms = PLATFORM_RIDS if os.path.exists("release"): print(Fore.BLUE + Style.DIM + @@ -92,25 +94,12 @@ def main() -> None: os.mkdir("release") - if PLATFORM_WINDOWS in platforms: - if not skip_build: - wipe_bin() - build_windows(skip_build) - - if PLATFORM_LINUX in platforms: - if not skip_build: - wipe_bin() - build_linux(skip_build) - - if PLATFORM_LINUX_ARM64 in platforms: - if not skip_build: - wipe_bin() - build_linux_arm64(skip_build) + # Good variable naming right here. + for platform in PLATFORMS: + if platform.rid not in platforms: + continue - if PLATFORM_MACOS in platforms: - if not skip_build: - wipe_bin() - build_macos(skip_build) + build_platform(platform, skip_build) def wipe_bin(): @@ -123,35 +112,8 @@ def wipe_bin(): shutil.rmtree("bin") -def build_windows(skip_build: bool) -> None: - # Run a full build. - print(Fore.GREEN + "Building project for Windows x64..." + Style.RESET_ALL) - - if not skip_build: - subprocess.run([ - "dotnet", - "build", - p("Content.Server", "Content.Server.csproj"), - "-c", "Release", - "--nologo", - "/v:m", - "/p:TargetOS=Windows", - "/t:Rebuild", - "/p:FullRelease=True" - ], check=True) - - publish_client_server("win-x64", "Windows") - - print(Fore.GREEN + "Packaging Windows x64 server..." + Style.RESET_ALL) - server_zip = zipfile.ZipFile(p("release", "SS14.Server_Windows_x64.zip"), "w", - compression=zipfile.ZIP_DEFLATED) - copy_dir_into_zip(p("RobustToolbox", "bin", "Server", "win-x64", "publish"), "", server_zip) - copy_resources(p("Resources"), server_zip) - copy_content_assemblies(p("Resources", "Assemblies"), server_zip) - server_zip.close() - -def build_macos(skip_build: bool) -> None: - print(Fore.GREEN + "Building project for macOS x64..." + Style.RESET_ALL) +def build_platform(platform: PlatformReg, skip_build: bool) -> None: + print(Fore.GREEN + f"Building project for {platform.rid}..." + Style.RESET_ALL) if not skip_build: subprocess.run([ @@ -161,73 +123,17 @@ def build_macos(skip_build: bool) -> None: "-c", "Release", "--nologo", "/v:m", - "/p:TargetOS=MacOS", + f"/p:TargetOS={platform.target_os}", "/t:Rebuild", "/p:FullRelease=True" ], check=True) - publish_client_server("osx-x64", "MacOS") + publish_client_server(platform.rid, platform.target_os) - print(Fore.GREEN + "Packaging macOS x64 server..." + Style.RESET_ALL) - server_zip = zipfile.ZipFile(p("release", "SS14.Server_macOS_x64.zip"), "w", + print(Fore.GREEN + "Packaging {platform.rid} server..." + Style.RESET_ALL) + server_zip = zipfile.ZipFile(p("release", f"SS14.Server_{platform.rid}.zip"), "w", compression=zipfile.ZIP_DEFLATED) - copy_dir_into_zip(p("RobustToolbox", "bin", "Server", "osx-x64", "publish"), "", server_zip) - copy_resources(p("Resources"), server_zip) - copy_content_assemblies(p("Resources", "Assemblies"), server_zip) - server_zip.close() - - -def build_linux(skip_build: bool) -> None: - # Run a full build. - print(Fore.GREEN + "Building project for Linux x64..." + Style.RESET_ALL) - - if not skip_build: - subprocess.run([ - "dotnet", - "build", - p("Content.Server", "Content.Server.csproj"), - "-c", "Release", - "--nologo", - "/v:m", - "/p:TargetOS=Linux", - "/t:Rebuild", - "/p:FullRelease=True" - ], check=True) - - publish_client_server("linux-x64", "Linux") - - print(Fore.GREEN + "Packaging Linux x64 server..." + Style.RESET_ALL) - server_zip = zipfile.ZipFile(p("release", "SS14.Server_Linux_x64.zip"), "w", - compression=zipfile.ZIP_DEFLATED) - copy_dir_into_zip(p("RobustToolbox", "bin", "Server", "linux-x64", "publish"), "", server_zip) - copy_resources(p("Resources"), server_zip) - copy_content_assemblies(p("Resources", "Assemblies"), server_zip) - server_zip.close() - - -def build_linux_arm64(skip_build: bool) -> None: - # Run a full build. - print(Fore.GREEN + "Building project for Linux ARM64 (SERVER ONLY)..." + Style.RESET_ALL) - - if not skip_build: - subprocess.run([ - "dotnet", - "build", - p("Content.Server", "Content.Server.csproj"), - "-c", "Release", - "--nologo", - "/v:m", - "/p:TargetOS=Linux", - "/t:Rebuild", - "/p:FullRelease=True" - ], check=True) - - publish_client_server("linux-arm64", "Linux") - - print(Fore.GREEN + "Packaging Linux ARM64 server..." + Style.RESET_ALL) - server_zip = zipfile.ZipFile(p("release", "SS14.Server_Linux_ARM64.zip"), "w", - compression=zipfile.ZIP_DEFLATED) - copy_dir_into_zip(p("RobustToolbox", "bin", "Server", "linux-arm64", "publish"), "", server_zip) + copy_dir_into_zip(p("RobustToolbox", "bin", "Server", platform.rid, "publish"), "", server_zip) copy_resources(p("Resources"), server_zip) copy_content_assemblies(p("Resources", "Assemblies"), server_zip) server_zip.close() @@ -255,13 +161,6 @@ def copy_resources(target, zipf): do_resource_copy(target, p("RobustToolbox", "Resources"), zipf, ignore_set) -def copy_launcher_resources(target, zipf): - # Copy all engine resources, since those are stripped down enough now. - do_resource_copy(target, p("RobustToolbox", "Resources"), zipf, SHARED_IGNORED_RESOURCES) - for folder in LAUNCHER_RESOURCES: - copy_dir_into_zip(p("Resources", folder), p(target, folder), zipf) - - def do_resource_copy(target, source, zipf, ignore_set): for filename in os.listdir(source): if filename in ignore_set: @@ -338,20 +237,5 @@ def copy_content_assemblies(target, zipf): zipf.write(p(source_dir, x), p(target, x)) -def copy_dir_or_file(src: str, dst: str): - """ - Just something from src to dst. If src is a dir it gets copied recursively. - """ - - if os.path.isfile(src): - shutil.copy2(src, dst) - - elif os.path.isdir(src): - shutil.copytree(src, dst) - - else: - raise IOError("{} is neither file nor directory. Can't copy.".format(src)) - - if __name__ == '__main__': main()