Add unstable test params for core and extension tests (microsoft#7513)

* Add unstable test params for core and extension tests * unset invert options for unstable script runs * Fix copypasta * Add nogpu to dacpac tests
winzig · Oct 7, 2019 · effa50a · effa50a
1 parent ac87346
commit effa50a
Show file tree

Hide file tree

Showing 24 changed files with 355 additions and 43 deletions.
diff --git a/extensions/admin-tool-ext-win/src/test/index.ts b/extensions/admin-tool-ext-win/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Database Admin Tool Extensions for Windows';
+const suite = 'admin-tool-ext-win Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {
@@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 
 testRunner.configure(options);
 
-export = testRunner;
+export = testRunner;
diff --git a/extensions/agent/src/test/index.ts b/extensions/agent/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Agent Tests';
+const suite = 'agent Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {
@@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 
 testRunner.configure(options);
 
-export = testRunner;
+export = testRunner;
diff --git a/extensions/azurecore/src/test/index.ts b/extensions/azurecore/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Integration Azure Tests';
+const suite = 'azurecore Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

diff --git a/extensions/cms/src/test/index.ts b/extensions/cms/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'CMS Unit Tests';
+const suite = 'cms Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

diff --git a/extensions/dacpac/src/test/index.ts b/extensions/dacpac/src/test/index.ts
@@ -6,17 +6,31 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'DacFx Tests';
+const suite = 'dacpac Extension Tests';
 
 const testOptions: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 60000
 };
 
-// const coverageConfig: any = {
-// 	coverConfig: '../../coverageConfig.json'
-// };
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	testOptions.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${testOptions.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	testOptions.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${testOptions.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	testOptions.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${testOptions.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	testOptions.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${testOptions.retries}`);
+}
 
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	testOptions.reporter = 'mocha-multi-reporters';

diff --git a/extensions/notebook/src/integrationTest/index.ts b/extensions/notebook/src/integrationTest/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Notebook Extension Integration Tests';
+const suite = 'notebook Extension Integration Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {
@@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 
 testRunner.configure(options);
 
-export = testRunner;
+export = testRunner;
diff --git a/extensions/notebook/src/test/book/book.test.ts b/extensions/notebook/src/test/book/book.test.ts
@@ -34,7 +34,7 @@ export function equalBookItems(book: BookTreeItem, expectedBook: ExpectedBookIte
 	}
 }
 
-describe.skip('BookTreeViewProviderTests', function() {
+describe('BookTreeViewProviderTests @UNSTABLE@', function() {
 
 	describe('BookTreeViewProvider.getChildren', function (): void {
 		let rootFolderPath: string;

diff --git a/extensions/notebook/src/test/index.ts b/extensions/notebook/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Notebook Tests';
+const suite = 'notebook Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 600000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {
@@ -27,4 +45,4 @@ if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 
 testRunner.configure(options);
 
-export = testRunner;
+export = testRunner;
diff --git a/extensions/resource-deployment/src/test/index.ts b/extensions/resource-deployment/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Resource Deployment Unit Tests';
+const suite = 'resource-deployment Extension Tests';
 
 const testOptions: any = {
 	ui: 'tdd',
 	useColors: true,
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	testOptions.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${testOptions.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	testOptions.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${testOptions.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	testOptions.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${testOptions.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	testOptions.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${testOptions.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	testOptions.reporter = 'mocha-multi-reporters';
 	testOptions.reporterOptions = {

diff --git a/extensions/schema-compare/src/test/index.ts b/extensions/schema-compare/src/test/index.ts
@@ -6,14 +6,32 @@
 const path = require('path');
 const testRunner = require('vscode/lib/testrunner');
 
-const suite = 'Schema Compare Tests';
+const suite = 'schema-compare Extension Tests';
 
 const options: any = {
 	ui: 'bdd',
 	useColors: true,
 	timeout: 60000
 };
 
+// set relevant mocha options from the environment
+if (process.env.ADS_TEST_GREP) {
+	options.grep = process.env.ADS_TEST_GREP;
+	console.log(`setting options.grep to: ${options.grep}`);
+}
+if (process.env.ADS_TEST_INVERT_GREP) {
+	options.invert = parseInt(process.env.ADS_TEST_INVERT_GREP);
+	console.log(`setting options.invert to: ${options.invert}`);
+}
+if (process.env.ADS_TEST_TIMEOUT) {
+	options.timeout = parseInt(process.env.ADS_TEST_TIMEOUT);
+	console.log(`setting options.timeout to: ${options.timeout}`);
+}
+if (process.env.ADS_TEST_RETRIES) {
+	options.retries = parseInt(process.env.ADS_TEST_RETRIES);
+	console.log(`setting options.retries to: ${options.retries}`);
+}
+
 if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
 	options.reporter = 'mocha-multi-reporters';
 	options.reporterOptions = {

diff --git a/scripts/sql-test-integration-unstable.bat b/scripts/sql-test-integration-unstable.bat
@@ -1,8 +1,9 @@
 setlocal
 
 set ADS_TEST_GREP=@UNSTABLE@
+set ADS_TEST_INVERT_GREP=
 
-echo Running unstable tests
+echo Running UNSTABLE ADS Extension Integration tests
 
 call %~dp0\sql-test-integration.bat
 

diff --git a/scripts/sql-test-integration-unstable.sh b/scripts/sql-test-integration-unstable.sh
@@ -11,7 +11,8 @@ fi
 cd $ROOT
 
 export ADS_TEST_GREP=@UNSTABLE@
+export ADS_TEST_INVERT_GREP=
 
-echo Running unstable tests
+echo Running UNSTABLE ADS Extension Integration tests
 
 ./scripts/sql-test-integration.sh
diff --git a/scripts/sql-test-integration.bat b/scripts/sql-test-integration.bat
@@ -7,6 +7,7 @@ set VSCODEEXTENSIONSDIR=%TMP%\adsext-%RANDOM%-%TIME:~6,5%
 echo VSCODEUSERDATADIR=%VSCODEUSERDATADIR%
 echo VSCODEEXTENSIONSDIR=%VSCODEEXTENSIONSDIR%
 
+:: Default to only running stable tests if test grep isn't set
 if "%ADS_TEST_GREP%" == "" (
 	echo Running stable tests only
 	set ADS_TEST_GREP=@UNSTABLE@

diff --git a/scripts/sql-test-integration.sh b/scripts/sql-test-integration.sh
@@ -12,6 +12,7 @@ else
 	VSCODEEXTDIR=`mktemp -d 2>/dev/null`
 fi
 
+# Default to only running stable tests if test grep isn't set
 if [[ "$ADS_TEST_GREP" == "" ]]; then
 	echo Running stable tests only
 	export ADS_TEST_GREP=@UNSTABLE@

diff --git a/scripts/test-extensions-unstable.bat b/scripts/test-extensions-unstable.bat
@@ -0,0 +1,12 @@
+:: Runs UNSTABLE Extension Tests
+
+setlocal
+
+set ADS_TEST_GREP=@UNSTABLE@
+set ADS_TEST_INVERT_GREP=
+
+echo Running UNSTABLE Extension Tests
+
+call %~dp0\test-extensions.bat
+
+endlocal