NLS can get mixed up when stable overwrites insider metadata (fix m…

…icrosoft#224385) (microsoft#224405)
aihazm · Jul 31, 2024 · 284a605 · 284a605
1 parent 6f2764a
commit 284a605
Show file tree

Hide file tree

Showing 4 changed files with 117 additions and 51 deletions.
diff --git a/build/lib/nls.js b/build/lib/nls.js
diff --git a/build/lib/nls.ts b/build/lib/nls.ts
@@ -8,7 +8,8 @@ import * as lazy from 'lazy.js';
 import { duplex, through } from 'event-stream';
 import * as File from 'vinyl';
 import * as sm from 'source-map';
-import * as  path from 'path';
+import * as path from 'path';
+import * as sort from 'gulp-sort';
 
 declare class FileSourceMap extends File {
 	public sourceMap: sm.RawSourceMap;
@@ -54,62 +55,64 @@ function clone<T extends object>(object: T): T {
 export function nls(options: { preserveEnglish: boolean }): NodeJS.ReadWriteStream {
 	let base: string;
 	const input = through();
-	const output = input.pipe(through(function (f: FileSourceMap) {
-		if (!f.sourceMap) {
-			return this.emit('error', new Error(`File ${f.relative} does not have sourcemaps.`));
-		}
+	const output = input
+		.pipe(sort()) // IMPORTANT: to ensure stable NLS metadata generation, we must sort the files because NLS messages are globally extracted and indexed across all files
+		.pipe(through(function (f: FileSourceMap) {
+			if (!f.sourceMap) {
+				return this.emit('error', new Error(`File ${f.relative} does not have sourcemaps.`));
+			}
 
-		let source = f.sourceMap.sources[0];
-		if (!source) {
-			return this.emit('error', new Error(`File ${f.relative} does not have a source in the source map.`));
-		}
+			let source = f.sourceMap.sources[0];
+			if (!source) {
+				return this.emit('error', new Error(`File ${f.relative} does not have a source in the source map.`));
+			}
 
-		const root = f.sourceMap.sourceRoot;
-		if (root) {
-			source = path.join(root, source);
-		}
+			const root = f.sourceMap.sourceRoot;
+			if (root) {
+				source = path.join(root, source);
+			}
 
-		const typescript = f.sourceMap.sourcesContent![0];
-		if (!typescript) {
-			return this.emit('error', new Error(`File ${f.relative} does not have the original content in the source map.`));
-		}
+			const typescript = f.sourceMap.sourcesContent![0];
+			if (!typescript) {
+				return this.emit('error', new Error(`File ${f.relative} does not have the original content in the source map.`));
+			}
 
-		base = f.base;
-		this.emit('data', _nls.patchFile(f, typescript, options));
-	}, function () {
-		for (const file of [
-			new File({
-				contents: Buffer.from(JSON.stringify({
-					keys: _nls.moduleToNLSKeys,
-					messages: _nls.moduleToNLSMessages,
-				}, null, '\t')),
-				base,
-				path: `${base}/nls.metadata.json`
-			}),
-			new File({
-				contents: Buffer.from(JSON.stringify(_nls.allNLSMessages)),
-				base,
-				path: `${base}/nls.messages.json`
-			}),
-			new File({
-				contents: Buffer.from(JSON.stringify(_nls.allNLSModulesAndKeys)),
-				base,
-				path: `${base}/nls.keys.json`
-			}),
-			new File({
-				contents: Buffer.from(`/*---------------------------------------------------------
+			base = f.base;
+			this.emit('data', _nls.patchFile(f, typescript, options));
+		}, function () {
+			for (const file of [
+				new File({
+					contents: Buffer.from(JSON.stringify({
+						keys: _nls.moduleToNLSKeys,
+						messages: _nls.moduleToNLSMessages,
+					}, null, '\t')),
+					base,
+					path: `${base}/nls.metadata.json`
+				}),
+				new File({
+					contents: Buffer.from(JSON.stringify(_nls.allNLSMessages)),
+					base,
+					path: `${base}/nls.messages.json`
+				}),
+				new File({
+					contents: Buffer.from(JSON.stringify(_nls.allNLSModulesAndKeys)),
+					base,
+					path: `${base}/nls.keys.json`
+				}),
+				new File({
+					contents: Buffer.from(`/*---------------------------------------------------------
  * Copyright (C) Microsoft Corporation. All rights reserved.
  *--------------------------------------------------------*/
 globalThis._VSCODE_NLS_MESSAGES=${JSON.stringify(_nls.allNLSMessages)};`),
-				base,
-				path: `${base}/nls.messages.js`
-			})
-		]) {
-			this.emit('data', file);
-		}
+					base,
+					path: `${base}/nls.messages.js`
+				})
+			]) {
+				this.emit('data', file);
+			}
 
-		this.emit('end');
-	}));
+			this.emit('end');
+		}));
 
 	return duplex(input, output);
 }

diff --git a/build/package.json b/build/package.json
@@ -20,6 +20,7 @@
     "@types/gulp-gzip": "^0.0.31",
     "@types/gulp-json-editor": "^2.2.31",
     "@types/gulp-rename": "^0.0.33",
+    "@types/gulp-sort": "^2.0.4",
     "@types/gulp-sourcemaps": "^0.0.32",
     "@types/mime": "0.0.29",
     "@types/minimatch": "^3.0.3",
@@ -44,6 +45,7 @@
     "esbuild": "0.23.0",
     "extract-zip": "^2.0.1",
     "gulp-merge-json": "^2.1.1",
+    "gulp-sort": "^2.0.0",
     "jsonc-parser": "^2.3.0",
     "mime": "^1.4.1",
     "mkdirp": "^1.0.4",

diff --git a/build/yarn.lock b/build/yarn.lock
@@ -483,13 +483,31 @@
   dependencies:
     "@types/node" "*"
 
+"@types/gulp-sort@^2.0.4":
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/@types/gulp-sort/-/gulp-sort-2.0.4.tgz#60625bf608dbac8f03644c6785d25c616f1b7d8c"
+  integrity sha512-HUHxH+oMox1ct0SnxPqCXBni0MSws1ygcSAoLO4ISRmR/UuvNIz40rgNveZxwxQk+p78kw09z/qKQkgKJmNUOQ==
+  dependencies:
+    "@types/gulp-util" "*"
+    "@types/node" "*"
+
 "@types/gulp-sourcemaps@^0.0.32":
   version "0.0.32"
   resolved "https://registry.yarnpkg.com/@types/gulp-sourcemaps/-/gulp-sourcemaps-0.0.32.tgz#e79ee617e0cb15729874be4533fe59c07793a175"
   integrity sha512-+7BAmptW2bxyJnJcCEuie7vLoop3FwWgCdBMzyv7MYXED/HeNMeQuX7uPCkp4vfU1TTu4CYFH0IckNPvo0VePA==
   dependencies:
     "@types/node" "*"
 
+"@types/gulp-util@*":
+  version "3.0.41"
+  resolved "https://registry.yarnpkg.com/@types/gulp-util/-/gulp-util-3.0.41.tgz#52868a6f8b6af55a099fe48ea20736833c02bed4"
+  integrity sha512-BK0kJZ8euQNlISsmD6mBr/1RZkB0mljdtBsz2usv+QHPV10alH2AJw5p05S9LU6S+VdTjbFmGU0OxpH++2W9/Q==
+  dependencies:
+    "@types/node" "*"
+    "@types/through2" "*"
+    "@types/vinyl" "*"
+    chalk "^2.2.0"
+
 "@types/gulp@^4.0.17":
   version "4.0.17"
   resolved "https://registry.yarnpkg.com/@types/gulp/-/gulp-4.0.17.tgz#b314c3762d08d8d69b7c0b86f78d069bafd65009"
@@ -591,6 +609,13 @@
     "@types/glob" "*"
     "@types/node" "*"
 
+"@types/through2@*":
+  version "2.0.41"
+  resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.41.tgz#3e5e1720d71ffdfa03c22f2aad6593d12a47034f"
+  integrity sha512-ryQ0tidWkb1O1JuYvWKyMLYEtOWDqF5mHerJzKz/gQpoAaJq2l/dsMPBF0B5BNVT34rbARYJ5/tsZwLfUi2kwQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/through2@^2.0.36":
   version "2.0.36"
   resolved "https://registry.yarnpkg.com/@types/through2/-/through2-2.0.36.tgz#35fda0db635827d44c0e08e2c94653e647574a00"
@@ -930,7 +955,7 @@ call-bind@^1.0.0:
     function-bind "^1.1.1"
     get-intrinsic "^1.0.2"
 
-chalk@^2.4.2:
+chalk@^2.2.0, chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
   integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
@@ -1546,6 +1571,13 @@ gulp-merge-json@^2.1.1:
     through "^2.3.8"
     vinyl "^2.1.0"
 
+gulp-sort@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/gulp-sort/-/gulp-sort-2.0.0.tgz#c6762a2f1f0de0a3fc595a21599d3fac8dba1aca"
+  integrity sha512-MyTel3FXOdh1qhw1yKhpimQrAmur9q1X0ZigLmCOxouQD+BD3za9/89O+HfbgBQvvh4igEbp0/PUWO+VqGYG1g==
+  dependencies:
+    through2 "^2.0.1"
+
 has-flag@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd"
@@ -2255,6 +2287,19 @@ readable-stream@^2.0.2, readable-stream@^2.3.5:
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
+readable-stream@~2.3.6:
+  version "2.3.8"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b"
+  integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==
+  dependencies:
+    core-util-is "~1.0.0"
+    inherits "~2.0.3"
+    isarray "~1.0.0"
+    process-nextick-args "~2.0.0"
+    safe-buffer "~5.1.1"
+    string_decoder "~1.1.1"
+    util-deprecate "~1.0.1"
+
 readdirp@~3.5.0:
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.5.0.tgz#9ba74c019b15d365278d2e91bb8c48d7b4d42c9e"
@@ -2505,6 +2550,14 @@ ternary-stream@^3.0.0:
     merge-stream "^2.0.0"
     through2 "^3.0.1"
 
+through2@^2.0.1:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
+  integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
+  dependencies:
+    readable-stream "~2.3.6"
+    xtend "~4.0.1"
+
 through2@^3.0.1:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/through2/-/through2-3.0.2.tgz#99f88931cfc761ec7678b41d5d7336b5b6a07bf4"
@@ -2747,6 +2800,11 @@ xmlbuilder@~11.0.0:
   resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3"
   integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==
 
+xtend@~4.0.1:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
+  integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
+
 yallist@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"