feat: Support --file-list cli option (#2130)

* feat: Support `--file-list` cli option Related to #1850 * dev: read fileLists when checking files. * dev: Improve error handling * Update GitPod config to reduce number of pre-builds
streetsidesoftware · Dec 29, 2021 · eef7b92 · eef7b92
1 parent bccdbf2
commit eef7b92
Show file tree

Hide file tree

Showing 21 changed files with 274 additions and 74 deletions.
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,2 +1,15 @@
 tasks:
-  - init: npm install && npm run build
+  - init: npm install
+    command: npm run build
+vscode:
+  extensions:
+    - streetsidesoftware.code-spell-checker
+github:
+  prebuilds:
+    master: true
+    branches: false
+    pullRequests: false
+    pullRequestsFromForks: false
+    addCheck: false
+    addComment: false
+    addBadge: false
diff --git a/cspell-dict.txt b/cspell-dict.txt
@@ -42,6 +42,7 @@ packlist
 pagekit
 patreon
 popd
+prebuilds
 pushd
 pycontribs
 rebased

diff --git a/cspell.code-workspace b/cspell.code-workspace
@@ -93,5 +93,12 @@
         "cSpell.customDictionaries": {
             "workspace": true
         }
+    },
+    "extensions": {
+        "recommendations": [
+            "streetsidesoftware.code-spell-checker",
+            "dbaeumer.vscode-eslint",
+            "esbenp.prettier-vscode"
+        ]
     }
 }
diff --git a/packages/cspell/fixtures/features/file-list/files-to-check-missing.txt b/packages/cspell/fixtures/features/file-list/files-to-check-missing.txt
@@ -0,0 +1,3 @@
+missing-file.txt
+../../../src/app.ts
+../../../README.md
diff --git a/packages/cspell/fixtures/features/file-list/files-to-check.txt b/packages/cspell/fixtures/features/file-list/files-to-check.txt
@@ -0,0 +1,2 @@
+../../../src/app.ts
+../../../README.md
diff --git a/packages/cspell/fixtures/fileHelper/file-list.txt b/packages/cspell/fixtures/fileHelper/file-list.txt
@@ -0,0 +1,3 @@
+file1
+../file2
+dir/file3
diff --git a/packages/cspell/fixtures/fileHelper/nested/file-list-2.txt b/packages/cspell/fixtures/fileHelper/nested/file-list-2.txt
@@ -0,0 +1 @@
+file2.txt
diff --git a/packages/cspell/src/__snapshots__/app.test.ts.snap b/packages/cspell/src/__snapshots__/app.test.ts.snap
@@ -11,7 +11,7 @@ Array [
   "  -h, --help                  display help for command",
   "",
   "Commands:",
-  "  lint [options] [files...]   Check spelling",
+  "  lint [options] [globs...]   Check spelling",
   "  trace [options] <words...>  Trace words",
   "    Search for words in the configuration and dictionaries.",
   "  check [options] <files...>  Spell check file(s) and display the result. The",
@@ -289,7 +289,7 @@ exports[`Validate cli app must find with error Expect Error: [Function CheckFail
 
 exports[`Validate cli app no-args Expect Error: outputHelp 1`] = `
 Array [
-  "Usage: cspell lint [options] [files...]",
+  "Usage: cspell lint [options] [globs...]",
   "",
   "Check spelling",
   "",
@@ -306,11 +306,12 @@ Array [
   "                               dictionaries.",
   "  -u, --unique                 Only output the first instance of a word not",
   "                               found in the dictionaries.",
-  "  --debug                      Output information useful for debugging",
-  "                               cspell.json files.",
   "  -e, --exclude <glob>         Exclude files matching the glob pattern. This",
   "                               option can be used multiple times to add",
   "                               multiple globs.",
+  "  --file-list <path or stdin>  Specify a list of files to be spell checked. The",
+  "                               list is filtered against the glob file patterns.",
+  "                               Note: the format is 1 file path per line.",
   "  --no-issues                  Do not show the spelling errors.",
   "  --no-progress                Turn off progress messages",
   "  --no-summary                 Turn off summary message in console",
@@ -320,7 +321,6 @@ Array [
   "  --show-context               Show the surrounding text around an issue.",
   "  --show-suggestions           Show spelling suggestions.",
   "  --no-must-find-files         Do not error if no files are found",
-  "  --legacy                     Legacy output",
   "  --cache                      Only check changed files (default: false)",
   "  --cache-strategy <strategy>  Strategy to use for detecting changed files",
   "                               (choices: \\"metadata\\", \\"content\\")",
@@ -335,6 +335,8 @@ Array [
   "                               root.",
   "  --no-color                   Turn off color.",
   "  --color                      Force color",
+  "  --debug                      Output information useful for debugging",
+  "                               cspell.json files.",
   "  -h, --help                   display help for command",
   "",
   "",

diff --git a/packages/cspell/src/app.ts b/packages/cspell/src/app.ts
@@ -9,7 +9,7 @@ import { ApplicationError } from './util/errors';
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const npmPackage = require(path.join(__dirname, '..', 'package.json'));
 
-export { LinterCliOptions as Options } from './commandLint';
+export { LinterCliOptions as Options } from './options';
 export { CheckFailed } from './util/errors';
 
 export async function run(program?: commander.Command, argv?: string[]): Promise<void> {

diff --git a/packages/cspell/src/application.ts b/packages/cspell/src/application.ts
@@ -11,8 +11,8 @@ export type { TraceResult } from 'cspell-lib';
 
 export type AppError = NodeJS.ErrnoException;
 
-export function lint(files: string[], options: LinterOptions, emitters: CSpellReporter): Promise<RunResult> {
-    const cfg = new LintRequest(files, options, emitters);
+export function lint(fileGlobs: string[], options: LinterOptions, emitters: CSpellReporter): Promise<RunResult> {
+    const cfg = new LintRequest(fileGlobs, options, emitters);
     return runLint(cfg);
 }
 

diff --git a/packages/cspell/src/cli-reporter.ts b/packages/cspell/src/cli-reporter.ts
@@ -2,8 +2,8 @@ import chalk = require('chalk');
 import type { CSpellReporter, Issue, MessageType, ProgressItem, RunResult } from '@cspell/cspell-types';
 import { ImportError, isSpellingDictionaryLoadError, SpellingDictionaryLoadError } from 'cspell-lib';
 import * as path from 'path';
-import { Options } from './app';
 import { URI } from 'vscode-uri';
+import { LinterCliOptions } from './options';
 
 const templateIssue = `{green $filename}:{yellow $row:$col} - $message ({red $text})`;
 const templateIssueWithSuggestions = `{green $filename}:{yellow $row:$col} - $message ({red $text}) Suggestions: {yellow [$suggestions]}`;
@@ -79,7 +79,26 @@ function reportTime(elapsedTimeMs: number | undefined, cached: boolean): string
     return color(elapsedTimeMs.toFixed(2) + 'ms');
 }
 
-export function getReporter(options: Options): CSpellReporter {
+export interface ReporterOptions
+    extends Pick<
+        LinterCliOptions,
+        | 'debug'
+        | 'issues'
+        | 'legacy'
+        | 'progress'
+        | 'relative'
+        | 'root'
+        | 'showContext'
+        | 'showSuggestions'
+        | 'silent'
+        | 'summary'
+        | 'verbose'
+        | 'wordsOnly'
+    > {
+    fileGlobs: string[];
+}
+
+export function getReporter(options: ReporterOptions): CSpellReporter {
     const issueTemplate = options.wordsOnly
         ? templateIssueWordsOnly
         : options.legacy
@@ -91,7 +110,7 @@ export function getReporter(options: Options): CSpellReporter {
         : options.showSuggestions
         ? templateIssueWithSuggestions
         : templateIssue;
-    const { files, silent, summary, issues, progress, verbose, debug } = options;
+    const { fileGlobs, silent, summary, issues, progress, verbose, debug } = options;
 
     const emitters: InfoEmitter = {
         Debug: !silent && debug ? (s) => console.info(chalk.cyan(s)) : nullEmitter,
@@ -117,7 +136,7 @@ export function getReporter(options: Options): CSpellReporter {
     }
 
     const resultEmitter = (result: RunResult) => {
-        if (!files.length && !result.files) {
+        if (!fileGlobs.length && !result.files) {
             return;
         }
         console.error(

diff --git a/packages/cspell/src/commandLint.ts b/packages/cspell/src/commandLint.ts
@@ -1,23 +1,10 @@
 import { Command, Option as CommanderOption } from 'commander';
 import * as App from './application';
 import { getReporter } from './cli-reporter';
-import { LinterOptions } from './options';
+import { LinterCliOptions, LinterOptions } from './options';
 import { DEFAULT_CACHE_LOCATION } from './util/cache';
 import { CheckFailed } from './util/errors';
 
-export interface LinterCliOptions extends LinterOptions {
-    files: string[];
-    legacy?: boolean;
-    summary: boolean;
-    issues: boolean;
-    silent: boolean;
-    mustFindFiles: boolean;
-    progress?: boolean;
-    /**
-     * issues are shown with a relative path to the root or `cwd`
-     */
-    relative?: boolean;
-}
 // interface InitOptions extends Options {}
 
 const usage = `
@@ -63,12 +50,18 @@ export function commandLint(prog: Command): Command {
             new CommanderOption('--wordsOnly', 'Only output the words not found in the dictionaries.').hideHelp()
         )
         .option('-u, --unique', 'Only output the first instance of a word not found in the dictionaries.')
-        .option('--debug', 'Output information useful for debugging cspell.json files.')
         .option(
             '-e, --exclude <glob>',
             'Exclude files matching the glob pattern. This option can be used multiple times to add multiple globs. ',
             collect
         )
+        .option(
+            '--file-list <path or stdin>',
+            'Specify a list of files to be spell checked.' +
+                ' The list is filtered against the glob file patterns.' +
+                ' Note: the format is 1 file path per line.',
+            collect
+        )
         .option('--no-issues', 'Do not show the spelling errors.')
         .option('--no-progress', 'Turn off progress messages')
         .option('--no-summary', 'Turn off summary message in console')
@@ -82,7 +75,7 @@ export function commandLint(prog: Command): Command {
         // The following options are planned features
         // .option('-w, --watch', 'Watch for any changes to the matching files and report any errors')
         // .option('--force', 'Force the exit value to always be 0')
-        .option('--legacy', 'Legacy output')
+        .addOption(new CommanderOption('--legacy', 'Legacy output').hideHelp())
         .addOption(new CommanderOption('--local <local>', 'Deprecated -- Use: --locale').hideHelp())
         .option('--cache', 'Only check changed files', false)
         .addOption(
@@ -98,14 +91,15 @@ export function commandLint(prog: Command): Command {
         .option('--gitignore-root <path>', 'Prevent searching for .gitignore files past root.', collect)
         .option('--no-color', 'Turn off color.')
         .option('--color', 'Force color')
+        .option('--debug', 'Output information useful for debugging cspell.json files.')
         .addHelpText('after', usage)
-        .arguments('[files...]')
-        .action((files: string[], options: LinterCliOptions) => {
-            options.files = files;
-            const { mustFindFiles } = options;
-            const cliReporter = getReporter(options);
-            return App.lint(files, options, cliReporter).then((result) => {
-                if (!files.length && !result.files) {
+        .arguments('[globs...]')
+        .action((fileGlobs: string[], options: LinterCliOptions) => {
+            const { mustFindFiles, fileList } = options;
+            const cliReporter = getReporter({ ...options, fileGlobs });
+            const lintOptions: LinterOptions = { ...options, fileLists: fileList };
+            return App.lint(fileGlobs, lintOptions, cliReporter).then((result) => {
+                if (!fileGlobs.length && !result.files && !result.errors) {
                     spellCheckCommand.outputHelp();
                     throw new CheckFailed('outputHelp', 1);
                 }

diff --git a/packages/cspell/src/fileHelper.test.ts b/packages/cspell/src/fileHelper.test.ts
@@ -0,0 +1,36 @@
+import { readFileListFile, readFileListFiles } from './fileHelper';
+import * as path from 'path';
+
+const fixtures = path.join(__dirname, '../fixtures/fileHelper');
+const fileListFile = path.join(fixtures, 'file-list.txt');
+const fileListFile2 = path.join(fixtures, 'nested/file-list-2.txt');
+
+const oc = expect.objectContaining;
+
+describe('fileHelper', () => {
+    test('readFileListFile', async () => {
+        try {
+            const files = ['a', 'b', 'c'];
+            process.stdin.isTTY = false;
+            const pResult = readFileListFile('stdin');
+            process.stdin.push(files.join('\n'));
+            // need to delay the `end` event or it might become a race condition.
+            setTimeout(() => process.stdin.emit('end'), 1);
+            const r = await pResult;
+            expect(r).toEqual(files.map((f) => path.resolve(f)));
+        } finally {
+            process.stdin.isTTY = true;
+        }
+    });
+
+    test('readFileListFiles', async () => {
+        const files = ['file1', '../file2', 'dir/file3', 'nested/file2.txt'];
+        const r = await readFileListFiles([fileListFile, fileListFile2]);
+        expect(r).toEqual(files.map((f) => path.resolve(fixtures, f)));
+    });
+
+    test('readFileListFiles Error', () => {
+        const r = readFileListFiles(['not-found.txt']);
+        return expect(r).rejects.toEqual(oc({ message: 'Error reading file: "not-found.txt"' }));
+    });
+});
diff --git a/packages/cspell/src/fileHelper.ts b/packages/cspell/src/fileHelper.ts
@@ -4,6 +4,7 @@ import getStdin from 'get-stdin';
 import { GlobOptions, globP } from './util/glob';
 import * as path from 'path';
 import { CSpellUserSettings, Document, fileToDocument, Issue } from 'cspell-lib';
+import { toApplicationError } from './util/errors';
 
 const UTF8: BufferEncoding = 'utf8';
 const STDIN = 'stdin';
@@ -70,10 +71,7 @@ export function readFileInfo(filename: string, encoding: string = UTF8): Promise
         (error) => {
             return error.code === 'EISDIR'
                 ? Promise.resolve({ text: '', filename })
-                : Promise.reject({
-                      ...error,
-                      message: `Error reading file: "${filename}"`,
-                  });
+                : Promise.reject(toApplicationError(error, `Error reading file: "${filename}"`));
         }
     );
 }
@@ -117,3 +115,40 @@ export function calcFinalConfigInfo(
         languageIds,
     };
 }
+
+/**
+ * Read
+ * @param listFiles - array of file paths to read that will contain a list of files. Paths contained in each
+ *   file will be resolved relative to the containing file.
+ * @returns - a list of files to be processed.
+ */
+export async function readFileListFiles(listFiles: string[]): Promise<string[]> {
+    return flatten(await Promise.all(listFiles.map(readFileListFile)));
+}
+
+/**
+ * Read a `listFile` and return the containing file paths resolved relative to the `listFile`.
+ * @param listFiles - array of file paths to read that will contain a list of files. Paths contained in each
+ *   file will be resolved relative to the containing file.
+ * @returns - a list of files to be processed.
+ */
+export async function readFileListFile(listFile: string): Promise<string[]> {
+    const relTo = path.resolve(path.dirname(listFile));
+    const content = await readFile(listFile);
+    const lines = content
+        .split('\n')
+        .map((a) => a.trim())
+        .filter((a) => !!a)
+        .map((file) => path.resolve(relTo, file));
+    return lines;
+}
+
+function flatten(fileLists: string[][]): string[] {
+    function* f() {
+        for (const list of fileLists) {
+            yield* list;
+        }
+    }
+
+    return [...f()];
+}
diff --git a/packages/cspell/src/lint/LintRequest.ts b/packages/cspell/src/lint/LintRequest.ts
@@ -15,8 +15,9 @@ export class LintRequest {
     readonly root: string;
     readonly showContext: number;
     readonly enableGlobDot: boolean | undefined;
+    readonly fileLists: string[];
 
-    constructor(readonly files: string[], readonly options: LinterOptions, readonly reporter: CSpellReporter) {
+    constructor(readonly fileGlobs: string[], readonly options: LinterOptions, readonly reporter: CSpellReporter) {
         this.root = path.resolve(options.root || process.cwd());
         this.configFile = options.config;
         this.excludes = calcExcludeGlobInfo(this.root, options.exclude);
@@ -25,5 +26,6 @@ export class LintRequest {
         this.uniqueFilter = options.unique ? util.uniqueFilterFnGenerator((issue: Issue) => issue.text) : () => true;
         this.showContext =
             options.showContext === true ? defaultContextRange : options.showContext ? options.showContext : 0;
+        this.fileLists = options.fileLists || [];
     }
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -42,6 +42,7 @@ packlist @@
     pagekit
     patreon
     popd
+    prebuilds
     pushd
     pycontribs
     rebased
@@ Expand Down @@