EH search - quickopen basically working (without filtering, batched r…

…esults, etc)
nexes · May 10, 2018 · a84db4a · a84db4a
1 parent 57e9559
commit a84db4a
Show file tree

Hide file tree

Showing 9 changed files with 295 additions and 31 deletions.
diff --git a/extensions/search-rg/src/extension.ts b/extensions/search-rg/src/extension.ts
@@ -5,6 +5,7 @@
 
 import * as vscode from 'vscode';
 import { RipgrepTextSearchEngine } from './ripgrepTextSearch';
+import { RipgrepFileSearchEngine } from './ripgrepFileSearch';
 
 export function activate(): void {
 	const provider = new RipgrepSearchProvider();
@@ -16,4 +17,9 @@ class RipgrepSearchProvider implements vscode.SearchProvider {
 		const engine = new RipgrepTextSearchEngine();
 		return engine.provideTextSearchResults(query, options, progress, token);
 	}
+
+	provideFileSearchResults(options: vscode.SearchOptions, progress: vscode.Progress<vscode.Uri>, token: vscode.CancellationToken): Thenable<void> {
+		const engine = new RipgrepFileSearchEngine();
+		return engine.provideFileSearchResults(options, progress, token);
+	}
 }
diff --git a/extensions/search-rg/src/normalization.ts b/extensions/search-rg/src/normalization.ts
@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * The normalize() method returns the Unicode Normalization Form of a given string. The form will be
+ * the Normalization Form Canonical Composition.
+ *
+ * @see {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/normalize}
+ */
+export const canNormalize = typeof ((<any>'').normalize) === 'function';
+
+export function normalizeNFC(str: string): string {
+	return normalize(str, 'NFC');
+}
+
+export function normalizeNFD(str: string): string {
+	return normalize(str, 'NFD');
+}
+
+const nonAsciiCharactersPattern = /[^\u0000-\u0080]/;
+function normalize(str: string, form: string): string {
+	if (!canNormalize || !str) {
+		return str;
+	}
+
+	let res: string;
+	if (nonAsciiCharactersPattern.test(str)) {
+		res = (<any>str).normalize(form);
+	} else {
+		res = str;
+	}
+
+	return res;
+}
diff --git a/extensions/search-rg/src/ripgrepFileSearch.ts b/extensions/search-rg/src/ripgrepFileSearch.ts
@@ -0,0 +1,202 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as cp from 'child_process';
+import * as path from 'path';
+import { Readable } from 'stream';
+import { NodeStringDecoder, StringDecoder } from 'string_decoder';
+import * as vscode from 'vscode';
+import { rgPath } from 'vscode-ripgrep';
+import { normalizeNFC, normalizeNFD } from './normalization';
+import { rgErrorMsgForDisplay } from './ripgrepTextSearch';
+
+const isMac = process.platform === 'darwin';
+
+// If vscode-ripgrep is in an .asar file, then the binary is unpacked.
+const rgDiskPath = rgPath.replace(/\bnode_modules\.asar\b/, 'node_modules.asar.unpacked');
+
+export class RipgrepFileSearchEngine {
+	private isDone = false;
+	private rgProc: cp.ChildProcess;
+	private killRgProcFn: (code?: number) => void;
+
+	constructor() {
+		this.killRgProcFn = () => this.rgProc && this.rgProc.kill();
+	}
+
+	provideFileSearchResults(options: vscode.SearchOptions, progress: vscode.Progress<vscode.Uri>, token: vscode.CancellationToken): Thenable<void> {
+		return new Promise((resolve, reject) => {
+			let isDone = false;
+			const cancel = () => {
+				this.isDone = true;
+				this.rgProc.kill();
+			};
+			token.onCancellationRequested(cancel);
+
+			const rgArgs = getRgArgs(options);
+
+			const cwd = options.folder.fsPath;
+
+			// TODO logging
+			// const escapedArgs = rgArgs
+			// 	.map(arg => arg.match(/^-/) ? arg : `'${arg}'`)
+			// 	.join(' ');
+			// let rgCmd = `rg ${escapedArgs}\n - cwd: ${cwd}`;
+
+			this.rgProc = cp.spawn(rgDiskPath, rgArgs, { cwd });
+			process.once('exit', this.killRgProcFn);
+			this.rgProc.on('error', e => {
+				console.log(e);
+				reject(e);
+			});
+
+			let leftover = '';
+			this.collectStdout(this.rgProc, (err, stdout, last) => {
+				if (err) {
+					reject(err);
+					return;
+				}
+
+				// Mac: uses NFD unicode form on disk, but we want NFC
+				const normalized = leftover + (isMac ? normalizeNFC(stdout) : stdout);
+				const relativeFiles = normalized.split('\n');
+
+				if (last) {
+					const n = relativeFiles.length;
+					relativeFiles[n - 1] = relativeFiles[n - 1].trim();
+					if (!relativeFiles[n - 1]) {
+						relativeFiles.pop();
+					}
+				} else {
+					leftover = relativeFiles.pop();
+				}
+
+				if (relativeFiles.length && relativeFiles[0].indexOf('\n') !== -1) {
+					reject(new Error('Splitting up files failed'));
+					return;
+				}
+
+				relativeFiles.forEach(relativeFile => {
+					progress.report(vscode.Uri.file(path.join(cwd, relativeFile)));
+				});
+
+				if (last) {
+					process.removeListener('exit', this.killRgProcFn);
+					if (isDone) {
+						resolve();
+					} else {
+						// Trigger last result
+						this.rgProc = null;
+						if (err) {
+							reject(err);
+						} else {
+							resolve();
+						}
+					}
+				}
+			});
+		});
+	}
+
+	private collectStdout(cmd: cp.ChildProcess, cb: (err: Error, stdout?: string, last?: boolean) => void): void {
+		let done = (err: Error, stdout?: string, last?: boolean) => {
+			if (err || last) {
+				done = () => { };
+			}
+
+			cb(err, stdout, last);
+		};
+
+		this.forwardData(cmd.stdout, done);
+		const stderr = this.collectData(cmd.stderr);
+
+		let gotData = false;
+		cmd.stdout.once('data', () => gotData = true);
+
+		cmd.on('error', (err: Error) => {
+			done(err);
+		});
+
+		cmd.on('close', (code: number) => {
+			// ripgrep returns code=1 when no results are found
+			let stderrText, displayMsg: string;
+			if (!gotData && (stderrText = this.decodeData(stderr)) && (displayMsg = rgErrorMsgForDisplay(stderrText))) {
+				done(new Error(`command failed with error code ${code}: ${displayMsg}`));
+			} else {
+				done(null, '', true);
+			}
+		});
+	}
+
+	private forwardData(stream: Readable, cb: (err: Error, stdout?: string) => void): NodeStringDecoder {
+		const decoder = new StringDecoder();
+		stream.on('data', (data: Buffer) => {
+			cb(null, decoder.write(data));
+		});
+		return decoder;
+	}
+
+	private collectData(stream: Readable): Buffer[] {
+		const buffers: Buffer[] = [];
+		stream.on('data', (data: Buffer) => {
+			buffers.push(data);
+		});
+		return buffers;
+	}
+
+	private decodeData(buffers: Buffer[]): string {
+		const decoder = new StringDecoder();
+		return buffers.map(buffer => decoder.write(buffer)).join('');
+	}
+}
+
+function getRgArgs(options: vscode.FileSearchOptions): string[] {
+	const args = ['--files', '--hidden', '--case-sensitive'];
+
+	options.includes.forEach(globArg => {
+		const inclusion = anchor(globArg);
+		args.push('-g', inclusion);
+		if (isMac) {
+			const normalized = normalizeNFD(inclusion);
+			if (normalized !== inclusion) {
+				args.push('-g', normalized);
+			}
+		}
+	});
+
+	options.excludes.forEach(globArg => {
+		const exclusion = `!${anchor(globArg)}`;
+		args.push('-g', exclusion);
+		if (isMac) {
+			const normalized = normalizeNFD(exclusion);
+			if (normalized !== exclusion) {
+				args.push('-g', normalized);
+			}
+		}
+	});
+
+	if (options.useIgnoreFiles) {
+		args.push('--no-ignore-parent');
+	} else {
+		// Don't use .gitignore or .ignore
+		args.push('--no-ignore');
+	}
+
+	// Follow symlinks
+	if (options.followSymlinks) {
+		args.push('--follow');
+	}
+
+	// Folder to search
+	args.push('--');
+
+	args.push('.');
+
+	return args;
+}
+
+function anchor(glob: string) {
+	return glob.startsWith('**') || glob.startsWith('/') ? glob : `/${glob}`;
+}
diff --git a/extensions/search-rg/src/ripgrepHelpers.ts b/extensions/search-rg/src/ripgrepHelpers.ts
@@ -9,17 +9,6 @@ import * as vscode from 'vscode';
 
 import * as path from 'path';
 
-export function patternsToRgGlobs(patterns: vscode.GlobPattern[]): string[] {
-	return patterns.map(p => {
-		if (typeof p === 'string') {
-			return p;
-		} else {
-			// TODO
-			return p.pattern;
-		}
-	});
-}
-
 export function fixDriveC(_path: string): string {
 	const root = path.parse(_path).root;
 	return root.toLowerCase() === 'c:/' ?

diff --git a/extensions/search-rg/src/ripgrepTextSearch.ts b/extensions/search-rg/src/ripgrepTextSearch.ts
@@ -13,7 +13,6 @@ import { StringDecoder, NodeStringDecoder } from 'string_decoder';
 
 import * as cp from 'child_process';
 import { rgPath } from 'vscode-ripgrep';
-import { patternsToRgGlobs } from './ripgrepHelpers';
 import { start } from 'repl';
 
 // If vscode-ripgrep is in an .asar file, then the binary is unpacked.
@@ -35,7 +34,6 @@ export class RipgrepTextSearchEngine {
 
 	provideTextSearchResults(query: vscode.TextSearchQuery, options: vscode.TextSearchOptions, progress: vscode.Progress<vscode.TextSearchResult>, token: vscode.CancellationToken): Thenable<void> {
 		return new Promise((resolve, reject) => {
-			let isDone = false;
 			const cancel = () => {
 				this.isDone = true;
 				this.ripgrepParser.cancel();
@@ -85,7 +83,7 @@ export class RipgrepTextSearchEngine {
 
 			this.rgProc.on('close', code => {
 				process.removeListener('exit', this.killRgProcFn);
-				if (isDone) {
+				if (this.isDone) {
 					resolve();
 				} else {
 					// Trigger last result
@@ -304,10 +302,11 @@ function getRgArgs(query: vscode.TextSearchQuery, options: vscode.TextSearchOpti
 	const args = ['--hidden', '--heading', '--line-number', '--color', 'ansi', '--colors', 'path:none', '--colors', 'line:none', '--colors', 'match:fg:red', '--colors', 'match:style:nobold'];
 	args.push(query.isCaseSensitive ? '--case-sensitive' : '--ignore-case');
 
-	patternsToRgGlobs(options.includes)
+	// TODO@roblou
+	options.includes
 		.forEach(globArg => args.push('-g', globArg));
 
-	patternsToRgGlobs(options.excludes)
+	options.excludes
 		.forEach(rgGlob => args.push('-g', `!${rgGlob}`));
 
 	if (options.maxFileSize) {

diff --git a/src/vs/vscode.proposed.d.ts b/src/vs/vscode.proposed.d.ts
@@ -74,6 +74,8 @@ declare module 'vscode' {
 		encoding?: string;
 	}
 
+	export interface FileSearchOptions extends SearchOptions { }
+
 	export interface TextSearchResult {
 		uri: Uri;
 		range: Range;
@@ -83,7 +85,7 @@ declare module 'vscode' {
 	}
 
 	export interface SearchProvider {
-		provideFileSearchResults?(query: string, options: SearchOptions, progress: Progress<Uri>, token: CancellationToken): Thenable<void>;
+		provideFileSearchResults?(options: FileSearchOptions, progress: Progress<Uri>, token: CancellationToken): Thenable<void>;
 		provideTextSearchResults?(query: TextSearchQuery, options: TextSearchOptions, progress: Progress<TextSearchResult>, token: CancellationToken): Thenable<void>;
 	}
 

diff --git a/src/vs/workbench/api/electron-browser/mainThreadSearch.ts b/src/vs/workbench/api/electron-browser/mainThreadSearch.ts
@@ -108,7 +108,7 @@ class RemoteSearchProvider implements ISearchResultProvider {
 			this._searches.set(search.id, search);
 
 			outer = query.type === QueryType.File
-				? this._proxy.$provideFileSearchResults(this._handle, search.id, query.filePattern)
+				? this._proxy.$provideFileSearchResults(this._handle, search.id, query)
 				: this._proxy.$provideTextSearchResults(this._handle, search.id, query.contentPattern, query);
 
 			outer.then(() => {

diff --git a/src/vs/workbench/api/node/extHost.protocol.ts b/src/vs/workbench/api/node/extHost.protocol.ts
@@ -595,7 +595,7 @@ export interface ExtHostFileSystemShape {
 }
 
 export interface ExtHostSearchShape {
-	$provideFileSearchResults(handle: number, session: number, query: string): TPromise<void>;
+	$provideFileSearchResults(handle: number, session: number, query: IRawSearchQuery): TPromise<void>;
 	$provideTextSearchResults(handle: number, session: number, pattern: IPatternInfo, query: IRawSearchQuery): TPromise<void>;
 }