fix(v2): truncate docuhash return value in order to avoid ERRNAMETOOL…

…ONG error (facebook#4899) * fix: truncate docuhash return value in order to avoid ERRNAMETOOLONG error * chore: add deep file path test page to website * refactor: reorganize docuHash/pathUtils code and tests * chore: git support longpaths on v2 windows tests workflow Co-authored-by: slorber <lorber.sebastien@gmail.com>
JoelMarcey · Jun 15, 2021 · 34411e1 · 34411e1
1 parent 3d95a3e
commit 34411e1
Show file tree

Hide file tree

Showing 8 changed files with 202 additions and 50 deletions.
diff --git a/.github/workflows/v2-tests-windows.yml b/.github/workflows/v2-tests-windows.yml
@@ -17,6 +17,8 @@ jobs:
       matrix:
         node: ['12', '14']
     steps:
+      - name: Support longpaths
+        run: git config --system core.longpaths true
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ matrix.node }}
         uses: actions/setup-node@v2

diff --git a/packages/docusaurus-utils/src/__tests__/docuHash.test.ts b/packages/docusaurus-utils/src/__tests__/docuHash.test.ts
@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {docuHash} from '../docuHash';
+
+describe('docuHash', () => {
+  test('docuHash works', () => {
+    const asserts: Record<string, string> = {
+      '': '-d41',
+      '/': 'index',
+      '/foo-bar': 'foo-bar-096',
+      '/foo/bar': 'foo-bar-1df',
+      '/endi/lie': 'endi-lie-9fa',
+      '/endi-lie': 'endi-lie-fd3',
+      '/yangshun/tay': 'yangshun-tay-48d',
+      '/yangshun-tay': 'yangshun-tay-f3b',
+      '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar':
+        'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo--d46',
+      '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2':
+        'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test--787',
+    };
+    Object.keys(asserts).forEach((file) => {
+      expect(docuHash(file)).toBe(asserts[file]);
+    });
+  });
+});
diff --git a/packages/docusaurus-utils/src/__tests__/index.test.ts b/packages/docusaurus-utils/src/__tests__/index.test.ts
@@ -8,8 +8,6 @@
 import path from 'path';
 import {
   fileToPath,
-  simpleHash,
-  docuHash,
   genComponentName,
   genChunkName,
   idx,
@@ -68,37 +66,6 @@ describe('load utils', () => {
     });
   });
 
-  test('simpleHash', () => {
-    const asserts: Record<string, string> = {
-      '': 'd41',
-      '/foo-bar': '096',
-      '/foo/bar': '1df',
-      '/endi/lie': '9fa',
-      '/endi-lie': 'fd3',
-      '/yangshun/tay': '48d',
-      '/yangshun-tay': 'f3b',
-    };
-    Object.keys(asserts).forEach((file) => {
-      expect(simpleHash(file, 3)).toBe(asserts[file]);
-    });
-  });
-
-  test('docuHash', () => {
-    const asserts: Record<string, string> = {
-      '': '-d41',
-      '/': 'index',
-      '/foo-bar': 'foo-bar-096',
-      '/foo/bar': 'foo-bar-1df',
-      '/endi/lie': 'endi-lie-9fa',
-      '/endi-lie': 'endi-lie-fd3',
-      '/yangshun/tay': 'yangshun-tay-48d',
-      '/yangshun-tay': 'yangshun-tay-f3b',
-    };
-    Object.keys(asserts).forEach((file) => {
-      expect(docuHash(file)).toBe(asserts[file]);
-    });
-  });
-
   test('fileToPath', () => {
     const asserts: Record<string, string> = {
       'index.md': '/',

diff --git a/packages/docusaurus-utils/src/__tests__/pathUtils.test.ts b/packages/docusaurus-utils/src/__tests__/pathUtils.test.ts
@@ -0,0 +1,82 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {simpleHash, isNameTooLong, shortName} from '../pathUtils';
+
+describe('pathUtils', () => {
+  test('simpleHash', () => {
+    const asserts: Record<string, string> = {
+      '': 'd41',
+      '/foo-bar': '096',
+      '/foo/bar': '1df',
+      '/endi/lie': '9fa',
+      '/endi-lie': 'fd3',
+      '/yangshun/tay': '48d',
+      '/yangshun-tay': 'f3b',
+      '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar':
+        'd46',
+      '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2':
+        '787',
+    };
+    Object.keys(asserts).forEach((file) => {
+      expect(simpleHash(file, 3)).toBe(asserts[file]);
+    });
+  });
+
+  test('isNameTooLong', () => {
+    const asserts: Record<string, boolean> = {
+      '': false,
+      'foo-bar-096': false,
+      'foo-bar-1df': false,
+      'endi-lie-9fa': false,
+      'endi-lie-fd3': false,
+      'yangshun-tay-48d': false,
+      'yangshun-tay-f3b': false,
+      'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-d46': true,
+      'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-2-787': true,
+    };
+    Object.keys(asserts).forEach((path) => {
+      expect(isNameTooLong(path)).toBe(asserts[path]);
+    });
+  });
+
+  describe('shortName', () => {
+    test('works', () => {
+      const asserts: Record<string, string> = {
+        '': '',
+        'foo-bar': 'foo-bar',
+        'endi-lie': 'endi-lie',
+        'yangshun-tay': 'yangshun-tay',
+        'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar':
+          'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-',
+        'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-2':
+          'foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-foo-bar-test-1-test-',
+      };
+      Object.keys(asserts).forEach((file) => {
+        expect(shortName(file)).toBe(asserts[file]);
+      });
+    });
+
+    // Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
+
+    const SHORT_PATH = `/short/path/without/trailing/slash`;
+    const VERY_LONG_PATH = `/${`x`.repeat(256)}/`;
+    const VERY_LONG_PATH_NON_LATIN = `/${`あ`.repeat(255)}/`;
+
+    it(`Truncates long paths correctly`, () => {
+      const truncatedPathLatin = shortName(VERY_LONG_PATH);
+      const truncatedPathNonLatin = shortName(VERY_LONG_PATH_NON_LATIN);
+      expect(truncatedPathLatin.length).toBeLessThanOrEqual(255);
+      expect(truncatedPathNonLatin.length).toBeLessThanOrEqual(255);
+    });
+
+    it(`Does not truncate short paths`, () => {
+      const truncatedPath = shortName(SHORT_PATH);
+      expect(truncatedPath).toEqual(SHORT_PATH);
+    });
+  });
+});
diff --git a/packages/docusaurus-utils/src/docuHash.ts b/packages/docusaurus-utils/src/docuHash.ts
@@ -0,0 +1,28 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {kebabCase} from 'lodash';
+
+import {shortName, isNameTooLong, simpleHash} from './pathUtils';
+
+/**
+ * Given an input string, convert to kebab-case and append a hash.
+ * Avoid str collision.
+ * Also removes part of the string if its larger than the allowed
+ * filename per OS. Avoids ERRNAMETOOLONG error.
+ */
+export function docuHash(str: string): string {
+  if (str === '/') {
+    return 'index';
+  }
+  const shortHash = simpleHash(str, 3);
+  const parsedPath = `${kebabCase(str)}-${shortHash}`;
+  if (isNameTooLong(parsedPath)) {
+    return `${shortName(kebabCase(str))}-${shortHash}`;
+  }
+  return parsedPath;
+}
diff --git a/packages/docusaurus-utils/src/index.ts b/packages/docusaurus-utils/src/index.ts
@@ -8,7 +8,7 @@
 import chalk from 'chalk';
 import path from 'path';
 import {createHash} from 'crypto';
-import {camelCase, kebabCase, mapValues} from 'lodash';
+import {camelCase, mapValues} from 'lodash';
 import escapeStringRegexp from 'escape-string-regexp';
 import fs from 'fs-extra';
 import {URL} from 'url';
@@ -22,13 +22,17 @@ import {
 import resolvePathnameUnsafe from 'resolve-pathname';
 
 import {posixPath as posixPathImport} from './posixPath';
+import {simpleHash} from './pathUtils';
+import {docuHash} from './docuHash';
 
 export const posixPath = posixPathImport;
 
 export * from './codeTranslationsUtils';
 export * from './markdownParser';
 export * from './markdownLinks';
 export * from './escapePath';
+export * from './docuHash';
+export {simpleHash} from './pathUtils';
 
 const fileHash = new Map();
 export async function generate(
@@ -98,22 +102,6 @@ export function encodePath(userpath: string): string {
     .join('/');
 }
 
-export function simpleHash(str: string, length: number): string {
-  return createHash('md5').update(str).digest('hex').substr(0, length);
-}
-
-/**
- * Given an input string, convert to kebab-case and append a hash.
- * Avoid str collision.
- */
-export function docuHash(str: string): string {
-  if (str === '/') {
-    return 'index';
-  }
-  const shortHash = simpleHash(str, 3);
-  return `${kebabCase(str)}-${shortHash}`;
-}
-
 /**
  * Convert first string character to the upper case.
  * E.g: docusaurus -> Docusaurus

diff --git a/packages/docusaurus-utils/src/pathUtils.ts b/packages/docusaurus-utils/src/pathUtils.ts
@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
+
+import {createHash} from 'crypto';
+
+// MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars, Others = 255 bytes
+const MAX_PATH_SEGMENT_CHARS = 255;
+const MAX_PATH_SEGMENT_BYTES = 255;
+// Space for appending things to the string like file extensions and so on
+const SPACE_FOR_APPENDING = 10;
+
+const isMacOs = process.platform === `darwin`;
+const isWindows = process.platform === `win32`;
+
+export const isNameTooLong = (str: string): boolean => {
+  return isMacOs || isWindows
+    ? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
+    : Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes)
+};
+
+export const shortName = (str: string): string => {
+  if (isMacOs || isWindows) {
+    const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS;
+    return str.slice(
+      0,
+      str.length - overflowingChars - SPACE_FOR_APPENDING - 1,
+    );
+  }
+  const strBuffer = Buffer.from(str);
+  const overflowingBytes =
+    Buffer.byteLength(strBuffer) - MAX_PATH_SEGMENT_BYTES;
+  return strBuffer
+    .slice(
+      0,
+      Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1,
+    )
+    .toString();
+};
+
+export function simpleHash(str: string, length: number): string {
+  return createHash('md5').update(str).digest('hex').substr(0, length);
+}
diff --git a/...ar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test-file.md b/...ar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test-file.md
@@ -0,0 +1,7 @@
+---
+title: Markdown page example
+---
+
+# Markdown page example
+
+You don't need React to write simple standalone pages.