eng: onboard to the extension test runner (microsoft#195570)

* eng: onboard to the extension test runner Adds a `.vscode-test.js` file that uses the new extension test CLI to run tests. Also, onboards the markdown-language-features as the first built-in extension to use it. With the `ms-vscode.extension-test-runner` extension installed, the markdown-language-features' tests can be run and debugged easily in the UI :) * fixup
NikGovorov · Oct 13, 2023 · c2a6932 · c2a6932
1 parent 590c72b
commit c2a6932
Show file tree

Hide file tree

Showing 10 changed files with 324 additions and 178 deletions.
diff --git a/.gitignore b/.gitignore
@@ -19,3 +19,4 @@ vscode.db
 /cli/openssl
 product.overrides.json
 *.snap.actual
+.vscode-test
diff --git a/.vscode-test.js b/.vscode-test.js
@@ -0,0 +1,71 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+//@ts-check
+
+const path = require('path');
+const { defineConfig } = require('@vscode/test-cli');
+
+/**
+ * A list of extension folders who have opted into tests, or configuration objects.
+ * Edit me to add more!
+ *
+ * @type {Array<string | (Partial<import("@vscode/test-cli").TestConfiguration> & { label: string })>}
+ */
+const extensions = [
+	{
+		label: 'markdown-language-features',
+		workspaceFolder: `extensions/markdown-language-features/test-workspace`,
+		mocha: { timeout: 60_000 }
+	},
+];
+
+
+const defaultLaunchArgs = process.env.API_TESTS_EXTRA_ARGS?.split(' ') || [
+	'--disable-telemetry', '--skip-welcome', '--skip-release-notes', `--crash-reporter-directory=${__dirname}/.build/crashes`, `--logsPath=${__dirname}/.build/logs/integration-tests`, '--no-cached-data', '--disable-updates', '--use-inmemory-secretstorage', '--disable-extensions', '--disable-workspace-trust'
+];
+
+module.exports = defineConfig(extensions.map(extension => {
+	/** @type {import('@vscode/test-cli').TestConfiguration} */
+	const config = typeof extension === 'object'
+		? { files: `extensions/${extension.label}/out/**/*.test.js`, ...extension }
+		: { files: `extensions/${extension}/out/**/*.test.js`, label: extension };
+
+	config.mocha ??= {};
+	if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
+		let suite = '';
+		if (process.env.VSCODE_BROWSER) {
+			suite = `${process.env.VSCODE_BROWSER} Browser Integration ${config.label} tests`;
+		} else if (process.env.REMOTE_VSCODE) {
+			suite = `Remote Integration ${config.label} tests`;
+		} else {
+			suite = `Integration ${config.label} tests`;
+		}
+
+		config.mocha.reporter = 'mocha-multi-reporters';
+		config.mocha.reporterOptions = {
+			reporterEnabled: 'spec, mocha-junit-reporter',
+			mochaJunitReporterReporterOptions: {
+				testsuitesTitle: `${suite} ${process.platform}`,
+				mochaFile: path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${suite.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`)
+			}
+		};
+	}
+
+	if (!config.platform || config.platform === 'desktop') {
+		config.launchArgs = defaultLaunchArgs;
+		config.useInstallation = {
+			fromPath: process.env.INTEGRATION_TEST_ELECTRON_PATH || `${__dirname}/scripts/code.${process.platform === 'win32' ? 'cmd' : 'sh'}`,
+		};
+		config.env = {
+			...config.env,
+			VSCODE_SKIP_PRELAUNCH: '1',
+		};
+	} else {
+		// web configs not supported, yet
+	}
+
+	return config;
+}));
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -3,9 +3,10 @@
 	// for the documentation about the extensions.json format
 	"recommendations": [
 		"dbaeumer.vscode-eslint",
-		"EditorConfig.EditorConfig",
-		"GitHub.vscode-pull-request-github",
+		"editorconfig.editorconfig",
+		"github.vscode-pull-request-github",
 		"ms-vscode.vscode-github-issue-notebooks",
-		"ms-vscode.vscode-selfhost-test-provider"
+		"ms-vscode.vscode-selfhost-test-provider",
+		"ms-vscode.extension-test-runner"
 	]
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -6,6 +6,7 @@
 		".build": true,
 		".profile-oss": true,
 		"**/.DS_Store": true,
+		".vscode-test": true,
 		"cli/target": true,
 		"build/**/*.js": {
 			"when": "$(basename).ts"
@@ -143,6 +144,11 @@
 			"${workspaceFolder}/build/**/*.js"
 		]
 	},
+	"extension-test-runner.debugOptions": {
+		"outFiles": [
+			"${workspaceFolder}/extensions/*/out/**/*.js",
+		]
+	},
 	"githubPullRequests.assignCreated": "${user}",
 	"githubPullRequests.defaultMergeMethod": "squash",
 	"githubPullRequests.ignoredPullRequestBranches": [

diff --git a/package.json b/package.json
@@ -13,6 +13,7 @@
     "test-browser": "npx playwright install && node test/unit/browser/index.js",
     "test-browser-no-install": "node test/unit/browser/index.js",
     "test-node": "mocha test/unit/node/index.js --delay --ui=tdd --timeout=5000 --exit",
+    "test-extension": "vscode-test",
     "preinstall": "node build/npm/preinstall.js",
     "postinstall": "node build/npm/postinstall.js",
     "compile": "node --max_old_space_size=4095 ./node_modules/gulp/bin/gulp.js compile",
@@ -136,6 +137,8 @@
     "@vscode/gulp-electron": "^1.36.0",
     "@vscode/l10n-dev": "0.0.21",
     "@vscode/telemetry-extractor": "^1.9.10",
+    "@vscode/test-cli": "^0.0.3",
+    "@vscode/test-electron": "^2.3.5",
     "@vscode/test-web": "^0.0.42",
     "@vscode/vscode-perf": "^0.0.14",
     "ansi-colors": "^3.2.3",

diff --git a/scripts/test-integration.bat b/scripts/test-integration.bat
@@ -59,7 +59,7 @@ if %errorlevel% neq 0 exit /b %errorlevel%
 
 echo.
 echo ### Markdown tests
-call "%INTEGRATION_TEST_ELECTRON_PATH%" %~dp0\..\extensions\markdown-language-features\test-workspace --extensionDevelopmentPath=%~dp0\..\extensions\markdown-language-features --extensionTestsPath=%~dp0\..\extensions\markdown-language-features\out\test %API_TESTS_EXTRA_ARGS%
+call yarn test-extension -l markdown-language-features
 if %errorlevel% neq 0 exit /b %errorlevel%
 
 echo.

diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh
@@ -79,7 +79,7 @@ kill_app
 echo
 echo "### Markdown tests"
 echo
-"$INTEGRATION_TEST_ELECTRON_PATH" $LINUX_EXTRA_ARGS $ROOT/extensions/markdown-language-features/test-workspace --extensionDevelopmentPath=$ROOT/extensions/markdown-language-features --extensionTestsPath=$ROOT/extensions/markdown-language-features/out/test $API_TESTS_EXTRA_ARGS
+yarn test-extension -l markdown-language-features
 kill_app
 
 echo

diff --git a/test/smoke/package.json b/test/smoke/package.json
@@ -11,7 +11,6 @@
     "mocha": "node ../node_modules/mocha/bin/mocha"
   },
   "dependencies": {
-    "@vscode/test-electron": "^2.3.2",
     "mkdirp": "^1.0.4",
     "ncp": "^2.0.0",
     "node-fetch": "^2.6.7",