[Github Action] cli-test: improve naming summary (jetify-com#1448)

## Summary **Motivation** The name summary for `cli-tests` in the Github PR page is: ``` test (false, ubuntu-latest, true, 2.12.0) ``` This isn't very helpful. This PR: - removes the `booleans` and changes their true/false states to explicit names - renames `devbox-json-tests` to `project-tests` for brevity. ## How was it tested? will observe the testscripts pass... Now, it looks like: ``` cli-tests / test (not-main, ubuntu-latest, project-tests, 2.12.0) ``` which is much more understandable.
zack-sys · Sep 5, 2023 · 27be737 · 27be737
1 parent d219b01
commit 27be737
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 18 deletions.
diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml
@@ -27,6 +27,8 @@ jobs:
   tests:
     uses: ./.github/workflows/cli-tests.yaml
     with:
+      # This will run the basic testscript unit tests on MacOS.
+      # NOTE: cli-tests will NEVER run project-tests on MacOS.
       run-mac-tests: true
 
   edge:

diff --git a/.github/workflows/cli-tests.yaml b/.github/workflows/cli-tests.yaml
@@ -19,8 +19,6 @@ on:
         type: boolean
   workflow_dispatch:
     inputs:
-      run-devbox-json-tests:
-        type: boolean
       run-mac-tests:
         type: boolean
       # run the example tests with DEVBOX_DEBUG=1
@@ -76,24 +74,25 @@ jobs:
     strategy:
       matrix:
         is-main:
-          - ${{ github.ref == 'refs/heads/main' }}
+          - ${{ github.ref == 'refs/heads/main' && 'is-main' || 'not-main' }}
         os: [ubuntu-latest, macos-latest]
         # This is an optimization that runs tests twice, with and without
         # the devbox.json tests. We can require the other tests to complete before
         # merging, while keeping the others as an additional non-required signal
-        run-devbox-json-tests: [true, false]
+        run-project-tests: ["project-tests", "project-tests-off"]
         # Run tests on:
         # 1. the oldest supported nix version (which is 2.9.0? But determinate-systems installer has 2.12.0)
         # 2. nix version 2.17.0 which introduces a new code path that minimizes nixpkgs downloads.
         # 3. latest nix version (currently, that is 2.17.0, so omitted)
         nix-version: ["2.12.0", "2.17.0"]
         exclude:
-          - is-main: false
+          - is-main: "not-main"
             os: "${{ inputs.run-mac-tests && 'dummy' || 'macos-latest' }}"
-          - is-main: true
-            run-devbox-json-tests: false
-          - run-devbox-json-tests: true
+          - is-main: "is-main"
+            run-project-tests: "project-tests-off"
+          - run-project-tests: "project-tests"
             os: macos-latest
+
     runs-on: ${{ matrix.os }}
     timeout-minutes: ${{ (github.ref == 'refs/heads/main' || inputs.run-mac-tests) && 37 || 25 }}
     steps:
@@ -132,8 +131,8 @@ jobs:
         env:
           # For devbox.json tests, we default to non-debug mode since the debug output is less useful than for unit testscripts.
           # But we allow overriding via inputs.example-debug
-          DEVBOX_DEBUG: ${{ (!matrix.run-devbox-json-tests || inputs.example-debug) && '1' || '0' }}
-          DEVBOX_RUN_DEVBOX_JSON_TESTS: ${{ matrix.run-devbox-json-tests }}
+          DEVBOX_DEBUG: ${{ (matrix.run-project-tests == 'project-tests-off' || inputs.example-debug) && '1' || '0' }}
+          DEVBOX_RUN_PROJECT_TESTS: ${{ matrix.run-project-tests == 'project-tests' && '1' || '0' }}
           # Used in `go test -timeout` flag. Needs a value that time.ParseDuration can parse.
           DEVBOX_GOLANG_TEST_TIMEOUT: "${{ (github.ref == 'refs/heads/main' || inputs.run-mac-tests) && '35m' || '30m' }}"
         run: |

diff --git a/testscripts/testscripts_test.go b/testscripts/testscripts_test.go
@@ -8,9 +8,9 @@ import (
 	"go.jetpack.io/devbox/testscripts/testrunner"
 )
 
-// When true, tests that `devbox run run_test` succeeds on every devbox.json
+// When true, tests that `devbox run run_test` succeeds on every project (i.e. having devbox.json)
 // found in examples/.. and testscripts/..
-const runDevboxJSONTests = "DEVBOX_RUN_DEVBOX_JSON_TESTS"
+const runProjectTests = "DEVBOX_RUN_PROJECT_TESTS"
 
 func TestScripts(t *testing.T) {
 	// To run a specific test, say, testscripts/foo/bar.test.text, then run
@@ -24,21 +24,21 @@ func TestMain(m *testing.M) {
 
 // TestExamples runs testscripts on the devbox-projects in the examples folder.
 func TestExamples(t *testing.T) {
-	isOn, err := strconv.ParseBool(os.Getenv(runDevboxJSONTests))
+	isOn, err := strconv.ParseBool(os.Getenv(runProjectTests))
 	if err != nil || !isOn {
-		t.Skipf("Skipping TestExamples. To enable, set %s=1.", runDevboxJSONTests)
+		t.Skipf("Skipping TestExamples. To enable, set %s=1.", runProjectTests)
 	}
 
 	// To run a specific test, say, examples/foo/bar, then run
 	// go test ./testscripts -run TestExamples/foo_bar_run_test
 	testrunner.RunDevboxTestscripts(t, "../examples")
 }
 
-// TestScriptsWithDevboxJSON runs testscripts on the devbox-projects in the testscripts folder.
-func TestScriptsWithDevboxJSON(t *testing.T) {
-	isOn, err := strconv.ParseBool(os.Getenv(runDevboxJSONTests))
+// TestScriptsWithProjects runs testscripts on the devbox-projects in the testscripts folder.
+func TestScriptsWithProjects(t *testing.T) {
+	isOn, err := strconv.ParseBool(os.Getenv(runProjectTests))
 	if err != nil || !isOn {
-		t.Skipf("Skipping TestExamples. To enable, set %s=1.", runDevboxJSONTests)
+		t.Skipf("Skipping TestScriptsWithProjects. To enable, set %s=1.", runProjectTests)
 	}
 
 	testrunner.RunDevboxTestscripts(t, ".")