Merge pull request supercollider#6003 from dyfer/topic/sclang-test-ru…

…nner-clean-pt1 Run testsuite directly
local-becoming · Apr 30, 2023 · fbb18fe · fbb18fe
2 parents 1e888a3 + 5a94636
commit fbb18fe
Show file tree

Hide file tree

Showing 11 changed files with 557 additions and 191 deletions.
diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml
@@ -65,13 +65,12 @@ jobs:
             use-syslibs: false
             shared-libscsynth: true
 
-          - job-name: 'bionic gcc10'
-            os-version: '18.04'
+          - job-name: 'jammy gcc10 use system libraries'
+            os-version: '22.04'
             c-compiler: 'gcc-10'
             cxx-compiler: 'g++-10'
-            use-syslibs: false
+            use-syslibs: true
             shared-libscsynth: false
-            artifact-suffix: 'linux-bionic-gcc10' # set if needed - will trigger artifact upload
 
           - job-name: 'jammy gcc11'
             os-version: '22.04'
@@ -80,12 +79,13 @@ jobs:
             use-syslibs: false
             shared-libscsynth: false
 
-          - job-name: 'jammy gcc12 use system libraries'
+          - job-name: 'jammy gcc12'
             os-version: '22.04'
             c-compiler: 'gcc-12'
             cxx-compiler: 'g++-12'
-            use-syslibs: true
+            use-syslibs: false
             shared-libscsynth: false
+            artifact-suffix: 'linux-jammy-gcc12' # set if needed - will trigger artifact upload
 
           - job-name: 'focal clang7'
             os-version: '20.04'
@@ -576,22 +576,28 @@ jobs:
             artifact-extension: '.dmg'
 
           - name: Linux
-            runs-on: ubuntu-18.04
+            runs-on: ubuntu-22.04
             sclang: 'build/Install/bin/sclang'
-            artifact-suffix: linux-bionic-gcc10
+            artifact-suffix: linux-jammy-gcc12
+
+          # - name: Windows
+          #   runs-on: windows-2022
+          #   sclang: 'build/Install/SuperCollider/sclang'
+          #   artifact-suffix: win64
 
-    needs: [lint, Linux, macOS] # unfortunately we can't use matrix expression here to make Linux test depend only on Linux build
+    needs: [lint, Linux, macOS, Windows] # unfortunately we can't use matrix expression here to make Linux test depend only on Linux build
     runs-on: '${{ matrix.runs-on }}'
     name: 'test on ${{ matrix.name }}'
     env:
       INSTALL_PATH: ${{ github.workspace }}/build/Install
       ARTIFACT_FILE: 'SuperCollider-${{ needs.lint.outputs.sc-version }}-${{ matrix.artifact-suffix }}${{ matrix.artifact-extension }}'
-      QUARKS_PATH: ${{ github.workspace }}/build/Quarks
       TESTS_PATH: ${{ github.workspace }}/testsuite/classlibrary
       SCLANG: ${{ github.workspace }}/${{ matrix.sclang }}
-      SCRIPT_PROTO: ${{ github.workspace }}/testsuite/scripts/gha_test_run_proto.json
-      SCRIPT_RUN: ${{ github.workspace }}/testsuite/scripts/run/gha_test_run.json
-      QPM_URL: git+https://github.com/supercollider/qpm.git@topic/try-catch-workaround
+      TEST_LIST_PROTO: ${{ github.workspace }}/testsuite/scripts/test_run_proto_gha.scd
+      TEST_LIST_RESULT: ${{ github.workspace }}/testsuite/scripts/run/gha_result.scxtar
+      SCRIPT_RUN_TESTS: ${{ github.workspace }}/testsuite/scripts/sclang_test_runner.scd
+      SCRIPT_ADD_PATH: ${{ github.workspace }}/testsuite/scripts/add_include_path.scd
+      SCRIPT_PRINT_RESULTS: ${{ github.workspace }}/testsuite/scripts/print_results.scd
     steps:
       - uses: actions/checkout@v3
         with:
@@ -602,6 +608,7 @@ jobs:
           name: ${{ env.ARTIFACT_FILE }}
           path: ${{ env.INSTALL_PATH }}
       - name: extract artifact
+        shell: bash
         run: |
           cd $INSTALL_PATH
           echo Contents of `pwd`:
@@ -619,41 +626,54 @@ jobs:
         run: |
           # install dependencies
           sudo apt-get update
-          sudo apt-get install --yes libsndfile1 libavahi-client-dev libfftw3-dev libicu-dev libudev-dev qt5-default qtwebengine5-dev jackd1
+          sudo apt-get install --yes libsndfile1 libavahi-client-dev libfftw3-dev libicu-dev libudev-dev qtbase5-dev qtwebengine5-dev jackd1 socat
 
-          # add bin to PATH so qpm can find scsynth
+          # add bin to PATH so that scsynth can be found
           echo "$INSTALL_PATH/bin" >> $GITHUB_PATH
 
           # start jack
-          jackd --no-realtime -d dummy &
-      - name: setup python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '2.x'
-      - name: prepare tests
-        run: |
-          echo `python --version` # check version
-          sudo pip install -U pip setuptools
-          sudo pip install $QPM_URL
+          jackd --no-realtime --silent -d dummy &
 
-          # install API quark
-          mkdir $QUARKS_PATH && cd $QUARKS_PATH
-          git clone --depth=1 https://github.com/crucialfelix/API
-
-          # make working copy of the script
-          cp $SCRIPT_PROTO $SCRIPT_RUN
-      - name: run tests on Linux
-        if: runner.os == 'Linux'
-        env:
-          PYCHARM_HOSTED: 1 # enable color output
-          QPM_DEBUG: 1
-        run: xvfb-run --server-args="-screen 0, 1280x720x24" -a qpm test.run -l $SCRIPT_RUN --path $SCLANG --include $QUARKS_PATH $TESTS_PATH
-      - name: run tests on macOS
+          # start virtual display server
+          export DISPLAY=:99
+          echo "DISPLAY=$DISPLAY" >> $GITHUB_ENV
+          Xvfb $DISPLAY -screen 0 1024x768x16 &
+      - name: setup macOS environment
         if: runner.os == 'macOS'
         env:
-          PYCHARM_HOSTED: 1 # enable color output
-          QPM_DEBUG: 1
-        run: qpm test.run -l $SCRIPT_RUN --path $SCLANG --include $QUARKS_PATH $TESTS_PATH
+          HOMEBREW_NO_ANALYTICS: 1
+          HOMEBREW_NO_AUTO_UPDATE: 1
+          HOMEBREW_NO_INSTALL_CLEANUP: 1
+        run: brew install coreutils # to get 'timeout' utility
+      - name: setup Windows environment
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          # install jack
+          # even though SC on Windows doesn't use JACK directly
+          # it will use JackRouter ASIO device
+          curl -O -J -L https://github.com/jackaudio/jack2-releases/releases/download/v1.9.22/jack2-win64-v1.9.22.exe
+
+          echo "installing JACK2"
+          ./jack2-win64-v1.9.22.exe //TYPE="router" //SILENT //SUPPRESSMSGBOXES
+
+          # add bin to PATH so that scsynth can be found
+          echo "$INSTALL_PATH/SuperCollider" >> $GITHUB_PATH
+
+          # start jack
+          /c/Program\ Files/JACK2/jackd.exe --no-realtime -d dummy &
+      - name: run tests
+        shell: bash
+        continue-on-error: true
+        run: |
+          # add path
+          $SCLANG $SCRIPT_ADD_PATH $TESTS_PATH
+          # run tests
+          timeout 1200 $SCLANG $SCRIPT_RUN_TESTS $TEST_LIST_PROTO $TEST_LIST_RESULT
+      - name: post test results
+        shell: bash
+        run: |
+          $SCLANG $SCRIPT_PRINT_RESULTS $TEST_LIST_RESULT
 
   deploy_s3:
     strategy:

diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ See the [Raspberry Pi](README_RASPBERRY_PI.md) and [BeagleBone Black](README_BEA
 SuperCollider is tested with:
 - Windows 10 (32- and 64-bit) and MSVC 2019
 - macOS 11 and Xcode 12.4
-- Ubuntu 18.04 and gcc 10
+- Ubuntu 22.04 and gcc 12
 
 SuperCollider is known to support these platforms:
 - Windows Vista, 7, 8, 10, 11

diff --git a/testsuite/classlibrary/CommonTests.quark b/testsuite/classlibrary/CommonTests.quark
diff --git a/testsuite/classlibrary/TestSerialPort.sc b/testsuite/classlibrary/TestSerialPort.sc
@@ -152,7 +152,7 @@ TestSerialPort : UnitTest {
 		this.destroyPorts();
 
 		fork { 3.wait; cond.test_(true) };
-		while { in.isOpen and: cond.test.not } { 0.001.wait };
+		while { in.isOpen or: cond.test.not } { 0.001.wait };
 
 		this.assert(in.isOpen.not);
 	}

diff --git a/testsuite/classlibrary/TestThreadReschedule.sc b/testsuite/classlibrary/TestThreadReschedule.sc
@@ -40,7 +40,8 @@ TestThreadReschedule : UnitTest {
 			task.reschedule(clock2);
 			// The test requires 4 data points (to measure tempo after rescheduling).
 			// So, all 3 beats are necessary. 3 beats < 3 ms.
-			3.wait;
+			// this was increased to accommodate timing variations for systems unders stress
+			6.wait;
 			cond.unhang;
 		}.play(clock1);
 		cond.hang;

diff --git a/testsuite/scripts/add_include_path.scd b/testsuite/scripts/add_include_path.scd
@@ -0,0 +1,16 @@
+//add unit tests to the include paths
+(
+var path = thisProcess.argv[0];
+
+path ?? {path = "".resolveRelative.dirname +/+ "classlibrary"};
+
+if(File.exists(path)) {
+	postf("\nAdding include path %\n\n", path);
+	LanguageConfig.addIncludePath(path);
+	LanguageConfig.store;
+} {
+	format("Specified path doesn't exist, not adding: %", path).warn;
+};
+
+0.exit;
+)
diff --git a/testsuite/scripts/gha_test_run_proto.json b/testsuite/scripts/gha_test_run_proto.json