diff --git a/eleventy-fetch.js b/eleventy-fetch.js
index 25fe66b..e37074a 100644
--- a/eleventy-fetch.js
+++ b/eleventy-fetch.js
@@ -1,8 +1,9 @@
const { default: PQueue } = require("p-queue");
const debug = require("debug")("Eleventy:Fetch");
-const RemoteAssetCache = require("./src/RemoteAssetCache");
-const AssetCache = require("./src/AssetCache");
+const Sources = require("./src/Sources.js");
+const RemoteAssetCache = require("./src/RemoteAssetCache.js");
+const AssetCache = require("./src/AssetCache.js");
const globalOptions = {
type: "buffer",
@@ -24,38 +25,6 @@ const globalOptions = {
hashLength: 30,
};
-function isFullUrl(url) {
- try {
- new URL(url);
- return true;
- } catch (e) {
- // invalid url OR already a local path
- return false;
- }
-}
-
-function isAwaitable(maybeAwaitable) {
- return (
- (typeof maybeAwaitable === "object" && typeof maybeAwaitable.then === "function") ||
- maybeAwaitable.constructor.name === "AsyncFunction"
- );
-}
-
-async function save(source, options) {
- if (!(isFullUrl(source) || isAwaitable(source))) {
- return Promise.reject(new Error("Caching an already local asset is not yet supported."));
- }
-
- if (isAwaitable(source) && !options.formatUrlForDisplay) {
- return Promise.reject(
- new Error("formatUrlForDisplay must be implemented, as a Promise has been provided."),
- );
- }
-
- let asset = new RemoteAssetCache(source, options.directory, options);
- return asset.fetch(options);
-}
-
/* Queue */
let queue = new PQueue({
concurrency: globalOptions.concurrency,
@@ -68,11 +37,9 @@ queue.on("active", () => {
let inProgress = {};
function queueSave(source, queueCallback, options) {
- let sourceKey;
- if(typeof source === "string") {
- sourceKey = source;
- } else {
- sourceKey = RemoteAssetCache.getUid(source, options);
+ let sourceKey = RemoteAssetCache.getRequestId(source, options);
+ if(!sourceKey) {
+ return Promise.reject(Sources.getInvalidSourceError(source));
}
if (!inProgress[sourceKey]) {
@@ -85,9 +52,14 @@ function queueSave(source, queueCallback, options) {
}
module.exports = function (source, options) {
+ if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) {
+ throw new Error("Caching an already local asset is not yet supported.");
+ }
+
let mergedOptions = Object.assign({}, globalOptions, options);
return queueSave(source, () => {
- return save(source, mergedOptions);
+ let asset = new RemoteAssetCache(source, mergedOptions.directory, mergedOptions);
+ return asset.fetch(mergedOptions);
}, mergedOptions);
};
@@ -102,7 +74,8 @@ Object.defineProperty(module.exports, "concurrency", {
module.exports.queue = queueSave;
module.exports.Util = {
- isFullUrl,
+ isFullUrl: Sources.isFullUrl,
};
module.exports.RemoteAssetCache = RemoteAssetCache;
module.exports.AssetCache = AssetCache;
+module.exports.Sources = Sources;
diff --git a/package.json b/package.json
index 8732f50..7c025ec 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "@11ty/eleventy-fetch",
- "version": "5.0.0-beta.5",
+ "version": "5.0.0-beta.6",
"description": "Fetch and locally cache remote API calls and assets.",
"publishConfig": {
"access": "public"
diff --git a/src/AssetCache.js b/src/AssetCache.js
index 3609f77..fd38dcd 100644
--- a/src/AssetCache.js
+++ b/src/AssetCache.js
@@ -4,61 +4,32 @@ const path = require("path");
const { create: FlatCacheCreate } = require("flat-cache");
const { createHash } = require("crypto");
+const Sources = require("./Sources.js");
+
const debug = require("debug")("Eleventy:Fetch");
class AssetCache {
#customFilename;
- static getCacheKey(source, options) {
- // RemoteAssetCache sends this an Array, which skips this altogether
- if (
- (typeof source === "object" && typeof source.then === "function") ||
- (typeof source === "function" && source.constructor.name === "AsyncFunction")
- ) {
- if(typeof options.formatUrlForDisplay !== "function") {
- throw new Error("When caching an arbitrary promise source, an options.formatUrlForDisplay() callback is required.");
- }
-
- return options.formatUrlForDisplay();
+ constructor(source, cacheDirectory, options = {}) {
+ if(!Sources.isValidSource(source)) {
+ throw Sources.getInvalidSourceError(source);
}
- return source;
- }
-
- constructor(url, cacheDirectory, options = {}) {
- let uniqueKey;
- // RemoteAssetCache passes in an array
- if(Array.isArray(uniqueKey)) {
- uniqueKey = uniqueKey.join(",");
- } else {
- uniqueKey = AssetCache.getCacheKey(url, options);
- }
+ let uniqueKey = AssetCache.getCacheKey(source, options);
this.uniqueKey = uniqueKey;
-
this.hash = AssetCache.getHash(uniqueKey, options.hashLength);
+
this.cacheDirectory = cacheDirectory || ".cache";
this.defaultDuration = "1d";
this.options = options;
// Compute the filename only once
if (typeof this.options.filenameFormat === "function") {
- this.#customFilename = this.options.filenameFormat(uniqueKey, this.hash);
-
- if (typeof this.#customFilename !== "string") {
- throw new Error(`The provided cacheFilename callback function did not return a string.`);
- }
+ this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash));
- if (typeof this.#customFilename.length === 0) {
- throw new Error(`The provided cacheFilename callback function returned an empty string.`);
- }
-
- // Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk)
- if (this.#customFilename.match(/([\/\\<>:"|?*]+?)/)) {
- const sanitizedFilename = this.#customFilename.replace(/[\/\\<>:"|?*]+/g, "");
- console.warn(
- `[AssetCache] Some illegal characters were removed from the cache filename: ${this.#customFilename} will be cached as ${sanitizedFilename}.`,
- );
- this.#customFilename = sanitizedFilename;
+ if (typeof this.#customFilename !== "string" || this.#customFilename.length === 0) {
+ throw new Error(`The provided filenameFormat callback function needs to return valid filename characters.`);
}
}
}
@@ -71,6 +42,40 @@ class AssetCache {
}
}
+ static cleanFilename(filename) {
+ // Ensure no illegal characters are present (Windows or Linux: forward/backslash, chevrons, colon, double-quote, pipe, question mark, asterisk)
+ if (filename.match(/([\/\\<>:"|?*]+?)/)) {
+ let sanitizedFilename = filename.replace(/[\/\\<>:"|?*]+/g, "");
+ debug(
+ `[@11ty/eleventy-fetch] Some illegal characters were removed from the cache filename: ${filename} will be cached as ${sanitizedFilename}.`,
+ );
+ return sanitizedFilename;
+ }
+
+ return filename;
+ }
+
+ static getCacheKey(source, options) {
+ // RemoteAssetCache passes in a string here, which skips this check (requestId is already used upstream)
+ if (Sources.isValidComplexSource(source)) {
+ if(options.requestId) {
+ return options.requestId;
+ }
+
+ if(typeof source.toString === "function") {
+ // return source.toString();
+ let toStr = source.toString();
+ if(toStr !== "function() {}" && toStr !== "[object Object]") {
+ return toStr;
+ }
+ }
+
+ throw Sources.getInvalidSourceError(source);
+ }
+
+ return source;
+ }
+
// Defult hashLength also set in global options, duplicated here for tests
// v5.0+ key can be Array or literal
static getHash(key, hashLength = 30) {
diff --git a/src/RemoteAssetCache.js b/src/RemoteAssetCache.js
index 9a436df..1731057 100644
--- a/src/RemoteAssetCache.js
+++ b/src/RemoteAssetCache.js
@@ -1,57 +1,63 @@
-const AssetCache = require("./AssetCache");
+const Sources = require("./Sources.js");
+const AssetCache = require("./AssetCache.js");
// const debug = require("debug")("Eleventy:Fetch");
class RemoteAssetCache extends AssetCache {
- constructor(url, cacheDirectory, options = {}) {
- let cleanUrl = url;
- if (options.removeUrlQueryParams) {
- cleanUrl = RemoteAssetCache.cleanUrl(cleanUrl);
- }
-
- // Must run after removeUrlQueryParams
- let displayUrl = RemoteAssetCache.convertUrlToString(cleanUrl, options);
- let cacheKeyArray = RemoteAssetCache.getCacheKey(displayUrl, options);
+ constructor(source, cacheDirectory, options = {}) {
+ let requestId = RemoteAssetCache.getRequestId(source, options);
+ super(requestId, cacheDirectory, options);
- super(cacheKeyArray, cacheDirectory, options);
-
- this.url = url;
+ this.source = source;
this.options = options;
-
- this.displayUrl = displayUrl;
+ this.displayUrl = RemoteAssetCache.convertUrlToString(source, options);
}
- static getUid(source, options) {
- let displayUrl = RemoteAssetCache.convertUrlToString(source, options);
- let cacheKeyArray = RemoteAssetCache.getCacheKey(displayUrl, options);
- return cacheKeyArray.join(",");
+ static getRequestId(source, options) {
+ if (Sources.isValidComplexSource(source)) {
+ return this.getCacheKey(source, options);
+ }
+
+ if (options.removeUrlQueryParams) {
+ let cleaned = this.cleanUrl(source);
+ return this.getCacheKey(cleaned, options);
+ }
+
+ return this.getCacheKey(source, options);
}
static getCacheKey(source, options) {
- // Promise sources are handled upstream
- let cacheKey = [source];
+ let cacheKey = {
+ source: AssetCache.getCacheKey(source, options),
+ };
if (options.fetchOptions) {
if (options.fetchOptions.method && options.fetchOptions.method !== "GET") {
- cacheKey.push(options.fetchOptions.method);
+ cacheKey.method = options.fetchOptions.method;
}
if (options.fetchOptions.body) {
- cacheKey.push(options.fetchOptions.body);
+ cacheKey.body = options.fetchOptions.body;
}
}
- return cacheKey;
+ if(Object.keys(cacheKey).length > 1) {
+ return JSON.stringify(cacheKey);
+ }
+
+ return cacheKey.source;
}
static cleanUrl(url) {
- if(typeof url !== "string" && !(url instanceof URL)) {
+ if(!Sources.isFullUrl(url)) {
return url;
}
let cleanUrl;
- if(typeof url === "string") {
+ if(typeof url === "string" || typeof url.toString === "function") {
cleanUrl = new URL(url);
} else if(url instanceof URL) {
cleanUrl = url;
+ } else {
+ throw new Error("Invalid source for cleanUrl: " + url)
}
cleanUrl.search = new URLSearchParams([]);
@@ -60,20 +66,15 @@ class RemoteAssetCache extends AssetCache {
}
static convertUrlToString(source, options = {}) {
+ // removes query params
+ source = RemoteAssetCache.cleanUrl(source);
+
let { formatUrlForDisplay } = options;
if (formatUrlForDisplay && typeof formatUrlForDisplay === "function") {
- return formatUrlForDisplay(source);
+ return "" + formatUrlForDisplay(source);
}
- return source;
- }
-
- get url() {
- return this._url;
- }
-
- set url(url) {
- this._url = url;
+ return "" + source;
}
async getResponseValue(response, type) {
@@ -100,15 +101,19 @@ class RemoteAssetCache extends AssetCache {
let body;
let type = optionsOverride.type || this.options.type;
- if (typeof this.url === "object" && typeof this.url.then === "function") {
- body = await this.url;
- } else if (typeof this.url === "function" && this.url.constructor.name === "AsyncFunction") {
- body = await this.url();
+ if (typeof this.source === "object" && typeof this.source.then === "function") {
+ body = await this.source;
+ } else if (typeof this.source === "function") {
+ // sync or async function
+ body = await this.source();
} else {
let fetchOptions = optionsOverride.fetchOptions || this.options.fetchOptions || {};
+ if(!Sources.isFullUrl(this.source)) {
+ throw Sources.getInvalidSourceError(this.source);
+ }
// v5: now using global (Node-native or otherwise) fetch instead of node-fetch
- let response = await fetch(this.url, fetchOptions);
+ let response = await fetch(this.source, fetchOptions);
if (!response.ok) {
throw new Error(
`Bad response for ${this.displayUrl} (${response.status}): ${response.statusText}`,
diff --git a/src/Sources.js b/src/Sources.js
new file mode 100644
index 0000000..686cdf3
--- /dev/null
+++ b/src/Sources.js
@@ -0,0 +1,50 @@
+class Sources {
+ static isFullUrl(url) {
+ try {
+ if(url instanceof URL) {
+ return true;
+ }
+
+ new URL(url);
+ return true;
+ } catch (e) {
+ // invalid url OR already a local path
+ return false;
+ }
+ }
+
+ static isValidSource(source) {
+ // String (url?)
+ if(typeof source === "string") {
+ return true;
+ }
+ if(this.isValidComplexSource(source)) {
+ return true;
+ }
+ return false;
+ }
+
+ static isValidComplexSource(source) {
+ // Async/sync Function
+ if(typeof source === "function") {
+ return true;
+ }
+ if(typeof source === "object") {
+ // Raw promise
+ if(typeof source.then === "function") {
+ return true;
+ }
+ // anything string-able
+ if(typeof source.toString === "function") {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static getInvalidSourceError(source, errorCause) {
+ return new Error("Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: " + source, { cause: errorCause });
+ }
+}
+
+module.exports = Sources;
diff --git a/test/AssetCacheTest.js b/test/AssetCacheTest.js
index 03fc1dc..076da25 100644
--- a/test/AssetCacheTest.js
+++ b/test/AssetCacheTest.js
@@ -62,13 +62,43 @@ test("Cache path should handle slashes without creating directories, issue #14",
t.is(cachePath, "/tmp/.cache/eleventy-fetch-135797dbf5ab1187e5003c49162602");
});
-test("Uses formatUrlForDisplay when caching a promise", async (t) => {
- let promise = Promise.resolve();
- let displayUrl = "mock-display-url";
- let asset = new AssetCache(promise, ".customcache", {
- formatUrlForDisplay() {
- return displayUrl;
- },
+test("Uses `requestId` property when caching a promise", async (t) => {
+ let asset = new AssetCache(Promise.resolve(), ".customcache", {
+ requestId: "mock-display-url-2",
+ });
+ let cachePath = normalizePath(asset.cachePath);
+ let jsonCachePath = normalizePath(asset.getCachedContentsPath("json"));
+
+ await asset.save({ name: "Sophia Smith" }, "json");
+
+ t.truthy(fs.existsSync(jsonCachePath));
+
+ await asset.destroy();
+
+ t.falsy(fs.existsSync(cachePath));
+ t.falsy(fs.existsSync(jsonCachePath));
+});
+
+test("Uses `requestId` property when caching a function", async (t) => {
+ let asset = new AssetCache(function() {}, ".cache", {
+ requestId: "mock-function",
+ });
+ let cachePath = normalizePath(asset.cachePath);
+ let jsonCachePath = normalizePath(asset.getCachedContentsPath("json"));
+
+ await asset.save({ name: "Sophia Smith" }, "json");
+
+ t.truthy(fs.existsSync(jsonCachePath));
+
+ await asset.destroy();
+
+ t.falsy(fs.existsSync(cachePath));
+ t.falsy(fs.existsSync(jsonCachePath));
+});
+
+test("Uses `requestId` property when caching an async function", async (t) => {
+ let asset = new AssetCache(async function() {}, ".cache", {
+ requestId: "mock-async-function",
});
let cachePath = normalizePath(asset.cachePath);
let jsonCachePath = normalizePath(asset.getCachedContentsPath("json"));
diff --git a/test/QueueTest.js b/test/QueueTest.js
index 043be46..78466c4 100644
--- a/test/QueueTest.js
+++ b/test/QueueTest.js
@@ -48,15 +48,11 @@ test("Double Fetch async function (dry run)", async (t) => {
let ac1 = Cache(fetch, {
dryRun: true,
- formatUrlForDisplay() {
- return "fetch-1";
- },
+ requestId: "fetch-1",
});
let ac2 = Cache(fetch, {
dryRun: true,
- formatUrlForDisplay() {
- return "fetch-2";
- },
+ requestId: "fetch-2",
});
// Make sure we only fetch once!
diff --git a/test/RemoteAssetCacheTest.js b/test/RemoteAssetCacheTest.js
index 0d591da..92c9a03 100644
--- a/test/RemoteAssetCacheTest.js
+++ b/test/RemoteAssetCacheTest.js
@@ -5,7 +5,7 @@ const AssetCache = require("../src/AssetCache");
const RemoteAssetCache = require("../src/RemoteAssetCache");
test("getDurationMs", (t) => {
- let cache = new RemoteAssetCache("lksdjflkjsdf");
+ let cache = new RemoteAssetCache("https://example.com/");
t.is(cache.getDurationMs("1s"), 1000);
t.is(cache.getDurationMs("1m"), 60 * 1000);
t.is(cache.getDurationMs("1h"), 60 * 60 * 1000);
@@ -135,18 +135,17 @@ test("Fetching pass in URL", async (t) => {
});
test("Fetching pass non-stringable", async (t) => {
- class B {}
+ let e = await t.throwsAsync(async () => {
+ class B {}
+ let source = new B();
- let ac = new RemoteAssetCache(new B(), undefined, {
- dryRun: true,
+ let ac = new RemoteAssetCache(source, undefined, {
+ dryRun: true,
+ });
+ await ac.fetch();
});
- try {
- await ac.fetch();
- } catch (e) {
- t.is(e.message, "Failed to parse URL from [object Object]");
- t.truthy(e.cause);
- }
+ t.is(e.message, "Invalid source: must be a string, function, or Promise. If a function or Promise, you must provide a `toString()` method or an `options.requestId` unique key. Received: [object Object]");
});
test("Fetching pass class with toString()", async (t) => {
@@ -196,6 +195,21 @@ test("formatUrlForDisplay (using removeUrlQueryParams)", async (t) => {
);
});
+test("formatUrlForDisplay (using removeUrlQueryParams and requestId)", async (t) => {
+ let longUrl =
+ "https://example.com/207115/photos/243-0-1.jpg?Policy=FAKE_THING~2123ksjhd&Signature=FAKE_THING~2123ksjhd&Key-Pair-Id=FAKE_THING~2123ksjhd";
+ t.is(
+ new RemoteAssetCache(function() {}, ".cache", {
+ requestId: longUrl,
+ removeUrlQueryParams: true,
+ formatUrlForDisplay(url) {
+ return url;
+ },
+ }).displayUrl,
+ "function() {}",
+ );
+});
+
test("Issue #6, URLs with HTTP Auth", async (t) => {
let url = "https://${USERNAME}:${PASSWORD}@api.pinboard.in/v1/posts/all?format=json&tag=read";
t.true(Util.isFullUrl(url));
@@ -225,9 +239,7 @@ test("supports promises that resolve", async (t) => {
let promise = Promise.resolve(expected);
let asset = new RemoteAssetCache(promise, undefined, {
type: "json",
- formatUrlForDisplay() {
- return "resolve-promise";
- },
+ requestId: "resolve-promise",
});
let actual = await asset.fetch();
@@ -245,9 +257,7 @@ test("supports promises that reject", async (t) => {
let promise = Promise.reject(new Error(expected, { cause }));
let asset = new RemoteAssetCache(promise, undefined, {
- formatUrlForDisplay() {
- return "reject-promise";
- },
+ requestId: "reject-promise",
});
try {
@@ -269,9 +279,7 @@ test("supports async functions that return data", async (t) => {
};
let asset = new RemoteAssetCache(asyncFunction, undefined, {
type: "json",
- formatUrlForDisplay() {
- return "async-return";
- },
+ requestId: "async-return",
});
let actual = await asset.fetch();
@@ -290,14 +298,11 @@ test("supports async functions that throw", async (t) => {
throw new Error(expected, { cause });
};
- let asset = new RemoteAssetCache(asyncFunction, undefined, {
- formatUrlForDisplay() {
- return "async-throws";
- },
- });
-
try {
- await asset.fetch();
+ let ac = new RemoteAssetCache(asyncFunction, undefined, {
+ requestId: "async-throws",
+ });
+ await ac.fetch();
} catch (e) {
t.is(e.message, expected);
t.is(e.cause, cause);