diff --git a/eleventy-fetch.js b/eleventy-fetch.js index 25fe66b..e37074a 100644 --- a/eleventy-fetch.js +++ b/eleventy-fetch.js @@ -1,8 +1,9 @@ const { default: PQueue } = require("p-queue"); const debug = require("debug")("Eleventy:Fetch"); -const RemoteAssetCache = require("./src/RemoteAssetCache"); -const AssetCache = require("./src/AssetCache"); +const Sources = require("./src/Sources.js"); +const RemoteAssetCache = require("./src/RemoteAssetCache.js"); +const AssetCache = require("./src/AssetCache.js"); const globalOptions = { type: "buffer", @@ -24,38 +25,6 @@ const globalOptions = { hashLength: 30, }; -function isFullUrl(url) { - try { - new URL(url); - return true; - } catch (e) { - // invalid url OR already a local path - return false; - } -} - -function isAwaitable(maybeAwaitable) { - return ( - (typeof maybeAwaitable === "object" && typeof maybeAwaitable.then === "function") || - maybeAwaitable.constructor.name === "AsyncFunction" - ); -} - -async function save(source, options) { - if (!(isFullUrl(source) || isAwaitable(source))) { - return Promise.reject(new Error("Caching an already local asset is not yet supported.")); - } - - if (isAwaitable(source) && !options.formatUrlForDisplay) { - return Promise.reject( - new Error("formatUrlForDisplay must be implemented, as a Promise has been provided."), - ); - } - - let asset = new RemoteAssetCache(source, options.directory, options); - return asset.fetch(options); -} - /* Queue */ let queue = new PQueue({ concurrency: globalOptions.concurrency, @@ -68,11 +37,9 @@ queue.on("active", () => { let inProgress = {}; function queueSave(source, queueCallback, options) { - let sourceKey; - if(typeof source === "string") { - sourceKey = source; - } else { - sourceKey = RemoteAssetCache.getUid(source, options); + let sourceKey = RemoteAssetCache.getRequestId(source, options); + if(!sourceKey) { + return Promise.reject(Sources.getInvalidSourceError(source)); } if (!inProgress[sourceKey]) { @@ -85,9 +52,14 @@ function queueSave(source, queueCallback, options) { } module.exports = function (source, options) { + if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) { + throw new Error("Caching an already local asset is not yet supported."); + } + let mergedOptions = Object.assign({}, globalOptions, options); return queueSave(source, () => { - return save(source, mergedOptions); + let asset = new RemoteAssetCache(source, mergedOptions.directory, mergedOptions); + return asset.fetch(mergedOptions); }, mergedOptions); }; @@ -102,7 +74,8 @@ Object.defineProperty(module.exports, "concurrency", { module.exports.queue = queueSave; module.exports.Util = { - isFullUrl, + isFullUrl: Sources.isFullUrl, }; module.exports.RemoteAssetCache = RemoteAssetCache; module.exports.AssetCache = AssetCache; +module.exports.Sources = Sources; diff --git a/package.json b/package.json index 8732f50..7c025ec 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@11ty/eleventy-fetch", - "version": "5.0.0-beta.5", + "version": "5.0.0-beta.6", "description": "Fetch and locally cache remote API calls and assets.", "publishConfig": { "access": "public" diff --git a/src/AssetCache.js b/src/AssetCache.js index 3609f77..fd38dcd 100644 --- a/src/AssetCache.js +++ b/src/AssetCache.js @@ -4,61 +4,32 @@ const path = require("path"); const { create: FlatCacheCreate } = require("flat-cache"); const { createHash } = require("crypto"); +const Sources = require("./Sources.js"); + const debug = require("debug")("Eleventy:Fetch"); class AssetCache { #customFilename; - static getCacheKey(source, options) { - // RemoteAssetCache sends this an Array, which skips this altogether - if ( - (typeof source === "object" && typeof source.then === "function") || - (typeof source === "function" && source.constructor.name === "AsyncFunction") - ) { - if(typeof options.formatUrlForDisplay !== "function") { - throw new Error("When caching an arbitrary promise source, an options.formatUrlForDisplay() callback is required."); - } - - return options.formatUrlForDisplay(); + constructor(source, cacheDirectory, options = {}) { + if(!Sources.isValidSource(source)) { + throw Sources.getInvalidSourceError(source); } - return source; - } - - constructor(url, cacheDirectory, options = {}) { - let uniqueKey; - // RemoteAssetCache passes in an array - if(Array.isArray(uniqueKey)) { - uniqueKey = uniqueKey.join(","); - } else { - uniqueKey = AssetCache.getCacheKey(url, options); - } + let uniqueKey = AssetCache.getCacheKey(source, options); this.uniqueKey = uniqueKey; - this.hash = AssetCache.getHash(uniqueKey, options.hashLength); + this.cacheDirectory = cacheDirectory || ".cache"; this.defaultDuration = "1d"; this.options = options; // Compute the filename only once if (typeof this.options.filenameFormat === "function") { - this.#customFilename = this.options.filenameFormat(uniqueKey, this.hash); - - if (typeof this.#customFilename !== "string") { - throw new Error(`The provided cacheFilename callback function did not return a string.`); - } + this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash)); - if (typeof this.#customFilename.length === 0) { - throw new Error(`The provided cacheFilename callback function returned an empty string.`); - } - - // Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk) - if (this.#customFilename.match(/([\/\\<>:"|?*]+?)/)) { - const sanitizedFilename = this.#customFilename.replace(/[\/\\<>:"|?*]+/g, ""); - console.warn( - `[AssetCache] Some illegal characters were removed from the cache filename: ${this.#customFilename} will be cached as ${sanitizedFilename}.`, - ); - this.#customFilename = sanitizedFilename; + if (typeof this.#customFilename !== "string" || this.#customFilename.length === 0) { + throw new Error(`The provided filenameFormat callback function needs to return valid filename characters.`); } } } @@ -71,6 +42,40 @@ class AssetCache { } } + static cleanFilename(filename) { + // Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk) + if (filename.match(/([\/\\<>:"|?*]+?)/)) { + let sanitizedFilename = filename.replace(/[\/\\<>:"|?*]+/g, ""); + debug( + `[@11ty/eleventy-fetch] Some illegal characters were removed from the cache filename: ${filename} will be cached as ${sanitizedFilename}.`, + ); + return sanitizedFilename; + } + + return filename; + } + + static getCacheKey(source, options) { + // RemoteAssetCache passes in a string here, which skips this check (requestId is already used upstream) + if (Sources.isValidComplexSource(source)) { + if(options.requestId) { + return options.requestId; + } + + if(typeof source.toString === "function") { + // return source.toString(); + let toStr = source.toString(); + if(toStr !== "function() {}" && toStr !== "[object Object]") { + return toStr; + } + } + + throw Sources.getInvalidSourceError(source); + } + + return source; + } + // Defult hashLength also set in global options, duplicated here for tests // v5.0+ key can be Array or literal static getHash(key, hashLength = 30) { diff --git a/src/RemoteAssetCache.js b/src/RemoteAssetCache.js index 9a436df..1731057 100644 --- a/src/RemoteAssetCache.js +++ b/src/RemoteAssetCache.js @@ -1,57 +1,63 @@ -const AssetCache = require("./AssetCache"); +const Sources = require("./Sources.js"); +const AssetCache = require("./AssetCache.js"); // const debug = require("debug")("Eleventy:Fetch"); class RemoteAssetCache extends AssetCache { - constructor(url, cacheDirectory, options = {}) { - let cleanUrl = url; - if (options.removeUrlQueryParams) { - cleanUrl = RemoteAssetCache.cleanUrl(cleanUrl); - } - - // Must run after removeUrlQueryParams - let displayUrl = RemoteAssetCache.convertUrlToString(cleanUrl, options); - let cacheKeyArray = RemoteAssetCache.getCacheKey(displayUrl, options); + constructor(source, cacheDirectory, options = {}) { + let requestId = RemoteAssetCache.getRequestId(source, options); + super(requestId, cacheDirectory, options); - super(cacheKeyArray, cacheDirectory, options); - - this.url = url; + this.source = source; this.options = options; - - this.displayUrl = displayUrl; + this.displayUrl = RemoteAssetCache.convertUrlToString(source, options); } - static getUid(source, options) { - let displayUrl = RemoteAssetCache.convertUrlToString(source, options); - let cacheKeyArray = RemoteAssetCache.getCacheKey(displayUrl, options); - return cacheKeyArray.join(","); + static getRequestId(source, options) { + if (Sources.isValidComplexSource(source)) { + return this.getCacheKey(source, options); + } + + if (options.removeUrlQueryParams) { + let cleaned = this.cleanUrl(source); + return this.getCacheKey(cleaned, options); + } + + return this.getCacheKey(source, options); } static getCacheKey(source, options) { - // Promise sources are handled upstream - let cacheKey = [source]; + let cacheKey = { + source: AssetCache.getCacheKey(source, options), + }; if (options.fetchOptions) { if (options.fetchOptions.method && options.fetchOptions.method !== "GET") { - cacheKey.push(options.fetchOptions.method); + cacheKey.method = options.fetchOptions.method; } if (options.fetchOptions.body) { - cacheKey.push(options.fetchOptions.body); + cacheKey.body = options.fetchOptions.body; } } - return cacheKey; + if(Object.keys(cacheKey).length > 1) { + return JSON.stringify(cacheKey); + } + + return cacheKey.source; } static cleanUrl(url) { - if(typeof url !== "string" && !(url instanceof URL)) { + if(!Sources.isFullUrl(url)) { return url; } let cleanUrl; - if(typeof url === "string") { + if(typeof url === "string" || typeof url.toString === "function") { cleanUrl = new URL(url); } else if(url instanceof URL) { cleanUrl = url; + } else { + throw new Error("Invalid source for cleanUrl: " + url) } cleanUrl.search = new URLSearchParams([]); @@ -60,20 +66,15 @@ class RemoteAssetCache extends AssetCache { } static convertUrlToString(source, options = {}) { + // removes query params + source = RemoteAssetCache.cleanUrl(source); + let { formatUrlForDisplay } = options; if (formatUrlForDisplay && typeof formatUrlForDisplay === "function") { - return formatUrlForDisplay(source); + return "" + formatUrlForDisplay(source); } - return source; - } - - get url() { - return this._url; - } - - set url(url) { - this._url = url; + return "" + source; } async getResponseValue(response, type) { @@ -100,15 +101,19 @@ class RemoteAssetCache extends AssetCache { let body; let type = optionsOverride.type || this.options.type; - if (typeof this.url === "object" && typeof this.url.then === "function") { - body = await this.url; - } else if (typeof this.url === "function" && this.url.constructor.name === "AsyncFunction") { - body = await this.url(); + if (typeof this.source === "object" && typeof this.source.then === "function") { + body = await this.source; + } else if (typeof this.source === "function") { + // sync or async function + body = await this.source(); } else { let fetchOptions = optionsOverride.fetchOptions || this.options.fetchOptions || {}; + if(!Sources.isFullUrl(this.source)) { + throw Sources.getInvalidSourceError(this.source); + } // v5: now using global (Node-native or otherwise) fetch instead of node-fetch - let response = await fetch(this.url, fetchOptions); + let response = await fetch(this.source, fetchOptions); if (!response.ok) { throw new Error( `Bad response for ${this.displayUrl} (${response.status}): ${response.statusText}`, diff --git a/src/Sources.js b/src/Sources.js new file mode 100644 index 0000000..686cdf3 --- /dev/null +++ b/src/Sources.js @@ -0,0 +1,50 @@ +class Sources { + static isFullUrl(url) { + try { + if(url instanceof URL) { + return true; + } + + new URL(url); + return true; + } catch (e) { + // invalid url OR already a local path + return false; + } + } + + static isValidSource(source) { + // String (url?) + if(typeof source === "string") { + return true; + } + if(this.isValidComplexSource(source)) { + return true; + } + return false; + } + + static isValidComplexSource(source) { + // Async/sync Function + if(typeof source === "function") { + return true; + } + if(typeof source === "object") { + // Raw promise + if(typeof source.then === "function") { + return true; + } + // anything string-able + if(typeof source.toString === "function") { + return true; + } + } + return false; + } + + static getInvalidSourceError(source, errorCause) { + return new Error("Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: " + source, { cause: errorCause }); + } +} + +module.exports = Sources; diff --git a/test/AssetCacheTest.js b/test/AssetCacheTest.js index 03fc1dc..076da25 100644 --- a/test/AssetCacheTest.js +++ b/test/AssetCacheTest.js @@ -62,13 +62,43 @@ test("Cache path should handle slashes without creating directories, issue #14", t.is(cachePath, "/tmp/.cache/eleventy-fetch-135797dbf5ab1187e5003c49162602"); }); -test("Uses formatUrlForDisplay when caching a promise", async (t) => { - let promise = Promise.resolve(); - let displayUrl = "mock-display-url"; - let asset = new AssetCache(promise, ".customcache", { - formatUrlForDisplay() { - return displayUrl; - }, +test("Uses `requestId` property when caching a promise", async (t) => { + let asset = new AssetCache(Promise.resolve(), ".customcache", { + requestId: "mock-display-url-2", + }); + let cachePath = normalizePath(asset.cachePath); + let jsonCachePath = normalizePath(asset.getCachedContentsPath("json")); + + await asset.save({ name: "Sophia Smith" }, "json"); + + t.truthy(fs.existsSync(jsonCachePath)); + + await asset.destroy(); + + t.falsy(fs.existsSync(cachePath)); + t.falsy(fs.existsSync(jsonCachePath)); +}); + +test("Uses `requestId` property when caching a function", async (t) => { + let asset = new AssetCache(function() {}, ".cache", { + requestId: "mock-function", + }); + let cachePath = normalizePath(asset.cachePath); + let jsonCachePath = normalizePath(asset.getCachedContentsPath("json")); + + await asset.save({ name: "Sophia Smith" }, "json"); + + t.truthy(fs.existsSync(jsonCachePath)); + + await asset.destroy(); + + t.falsy(fs.existsSync(cachePath)); + t.falsy(fs.existsSync(jsonCachePath)); +}); + +test("Uses `requestId` property when caching an async function", async (t) => { + let asset = new AssetCache(async function() {}, ".cache", { + requestId: "mock-async-function", }); let cachePath = normalizePath(asset.cachePath); let jsonCachePath = normalizePath(asset.getCachedContentsPath("json")); diff --git a/test/QueueTest.js b/test/QueueTest.js index 043be46..78466c4 100644 --- a/test/QueueTest.js +++ b/test/QueueTest.js @@ -48,15 +48,11 @@ test("Double Fetch async function (dry run)", async (t) => { let ac1 = Cache(fetch, { dryRun: true, - formatUrlForDisplay() { - return "fetch-1"; - }, + requestId: "fetch-1", }); let ac2 = Cache(fetch, { dryRun: true, - formatUrlForDisplay() { - return "fetch-2"; - }, + requestId: "fetch-2", }); // Make sure we only fetch once! diff --git a/test/RemoteAssetCacheTest.js b/test/RemoteAssetCacheTest.js index 0d591da..92c9a03 100644 --- a/test/RemoteAssetCacheTest.js +++ b/test/RemoteAssetCacheTest.js @@ -5,7 +5,7 @@ const AssetCache = require("../src/AssetCache"); const RemoteAssetCache = require("../src/RemoteAssetCache"); test("getDurationMs", (t) => { - let cache = new RemoteAssetCache("lksdjflkjsdf"); + let cache = new RemoteAssetCache("https://example.com/"); t.is(cache.getDurationMs("1s"), 1000); t.is(cache.getDurationMs("1m"), 60 * 1000); t.is(cache.getDurationMs("1h"), 60 * 60 * 1000); @@ -135,18 +135,17 @@ test("Fetching pass in URL", async (t) => { }); test("Fetching pass non-stringable", async (t) => { - class B {} + let e = await t.throwsAsync(async () => { + class B {} + let source = new B(); - let ac = new RemoteAssetCache(new B(), undefined, { - dryRun: true, + let ac = new RemoteAssetCache(source, undefined, { + dryRun: true, + }); + await ac.fetch(); }); - try { - await ac.fetch(); - } catch (e) { - t.is(e.message, "Failed to parse URL from [object Object]"); - t.truthy(e.cause); - } + t.is(e.message, "Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: [object Object]"); }); test("Fetching pass class with toString()", async (t) => { @@ -196,6 +195,21 @@ test("formatUrlForDisplay (using removeUrlQueryParams)", async (t) => { ); }); +test("formatUrlForDisplay (using removeUrlQueryParams and requestId)", async (t) => { + let longUrl = + "https://example.com/207115/photos/243-0-1.jpg?Policy=FAKE_THING~2123ksjhd&Signature=FAKE_THING~2123ksjhd&Key-Pair-Id=FAKE_THING~2123ksjhd"; + t.is( + new RemoteAssetCache(function() {}, ".cache", { + requestId: longUrl, + removeUrlQueryParams: true, + formatUrlForDisplay(url) { + return url; + }, + }).displayUrl, + "function() {}", + ); +}); + test("Issue #6, URLs with HTTP Auth", async (t) => { let url = "https://${USERNAME}:${PASSWORD}@api.pinboard.in/v1/posts/all?format=json&tag=read"; t.true(Util.isFullUrl(url)); @@ -225,9 +239,7 @@ test("supports promises that resolve", async (t) => { let promise = Promise.resolve(expected); let asset = new RemoteAssetCache(promise, undefined, { type: "json", - formatUrlForDisplay() { - return "resolve-promise"; - }, + requestId: "resolve-promise", }); let actual = await asset.fetch(); @@ -245,9 +257,7 @@ test("supports promises that reject", async (t) => { let promise = Promise.reject(new Error(expected, { cause })); let asset = new RemoteAssetCache(promise, undefined, { - formatUrlForDisplay() { - return "reject-promise"; - }, + requestId: "reject-promise", }); try { @@ -269,9 +279,7 @@ test("supports async functions that return data", async (t) => { }; let asset = new RemoteAssetCache(asyncFunction, undefined, { type: "json", - formatUrlForDisplay() { - return "async-return"; - }, + requestId: "async-return", }); let actual = await asset.fetch(); @@ -290,14 +298,11 @@ test("supports async functions that throw", async (t) => { throw new Error(expected, { cause }); }; - let asset = new RemoteAssetCache(asyncFunction, undefined, { - formatUrlForDisplay() { - return "async-throws"; - }, - }); - try { - await asset.fetch(); + let ac = new RemoteAssetCache(asyncFunction, undefined, { + requestId: "async-throws", + }); + await ac.fetch(); } catch (e) { t.is(e.message, expected); t.is(e.cause, cause);