diff --git a/eleventy-fetch.js b/eleventy-fetch.js index e37074a..21715e9 100644 --- a/eleventy-fetch.js +++ b/eleventy-fetch.js @@ -4,6 +4,7 @@ const debug = require("debug")("Eleventy:Fetch"); const Sources = require("./src/Sources.js"); const RemoteAssetCache = require("./src/RemoteAssetCache.js"); const AssetCache = require("./src/AssetCache.js"); +const DirectoryManager = require("./src/DirectoryManager.js"); const globalOptions = { type: "buffer", @@ -34,33 +35,37 @@ queue.on("active", () => { debug(`Concurrency: ${queue.concurrency}, Size: ${queue.size}, Pending: ${queue.pending}`); }); -let inProgress = {}; +let instCache = {}; -function queueSave(source, queueCallback, options) { +let directoryManager = new DirectoryManager(); + +function createRemoteAssetCache(source, rawOptions = {}) { + if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) { + return Promise.reject(new Error("Invalid source. Received: " + source)); + } + + let options = Object.assign({}, globalOptions, rawOptions); let sourceKey = RemoteAssetCache.getRequestId(source, options); if(!sourceKey) { return Promise.reject(Sources.getInvalidSourceError(source)); } - if (!inProgress[sourceKey]) { - inProgress[sourceKey] = queue.add(queueCallback).finally(() => { - delete inProgress[sourceKey]; - }); + if(instCache[sourceKey]) { + return instCache[sourceKey]; } - return inProgress[sourceKey]; + let inst = new RemoteAssetCache(source, options.directory, options); + inst.setQueue(queue); + inst.setDirectoryManager(directoryManager); + + instCache[sourceKey] = inst; + + return inst; } module.exports = function (source, options) { - if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) { - throw new Error("Caching an already local asset is not yet supported."); - } - - let mergedOptions = Object.assign({}, globalOptions, options); - return queueSave(source, () => { - let asset = new RemoteAssetCache(source, mergedOptions.directory, mergedOptions); - return asset.fetch(mergedOptions); - }, mergedOptions); + let instance = createRemoteAssetCache(source, options); + return instance.queue(); }; Object.defineProperty(module.exports, "concurrency", { @@ -72,7 +77,15 @@ Object.defineProperty(module.exports, "concurrency", { }, }); -module.exports.queue = queueSave; +module.exports.Fetch = createRemoteAssetCache; + +// Deprecated API kept for backwards compat, instead: use default export directly. +// Intentional: queueCallback is ignored here +module.exports.queue = function(source, queueCallback, options) { + let instance = createRemoteAssetCache(source, options); + return instance.queue(); +}; + module.exports.Util = { isFullUrl: Sources.isFullUrl, }; diff --git a/package.json b/package.json index 5019ee8..948dc0b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@11ty/eleventy-fetch", - "version": "5.0.1", + "version": "5.0.2", "description": "Fetch and locally cache remote API calls and assets.", "publishConfig": { "access": "public" diff --git a/src/AssetCache.js b/src/AssetCache.js index d9f1c73..c040915 100644 --- a/src/AssetCache.js +++ b/src/AssetCache.js @@ -2,13 +2,23 @@ const fs = require("graceful-fs"); const path = require("path"); const { create: FlatCacheCreate } = require("flat-cache"); const { createHash } = require("crypto"); +const debugUtil = require("debug"); const Sources = require("./Sources.js"); +const DirectoryManager = require("./DirectoryManager.js"); -const debug = require("debug")("Eleventy:Fetch"); +const debug = debugUtil("Eleventy:Fetch"); +const debugAssets = debugUtil("Eleventy:Assets"); class AssetCache { + #source; + #hash; #customFilename; + #cache; + #cacheDirectory; + #cacheLocationDirty = false; + #directoryManager; + #rawContents = {} constructor(source, cacheDirectory, options = {}) { if(!Sources.isValidSource(source)) { @@ -20,9 +30,11 @@ class AssetCache { this.hash = AssetCache.getHash(uniqueKey, options.hashLength); this.cacheDirectory = cacheDirectory || ".cache"; - this.defaultDuration = "1d"; this.options = options; + this.defaultDuration = "1d"; + this.duration = options.duration || this.defaultDuration; + // Compute the filename only once if (typeof this.options.filenameFormat === "function") { this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash)); @@ -97,35 +109,35 @@ class AssetCache { } get source() { - return this._source; + return this.#source; } set source(source) { - this._source = source; + this.#source = source; } get hash() { - return this._hash; + return this.#hash; } set hash(value) { - if (value !== this._hash) { - this._cacheLocationDirty = true; + if (value !== this.#hash) { + this.#cacheLocationDirty = true; } - this._hash = value; + this.#hash = value; } get cacheDirectory() { - return this._cacheDirectory; + return this.#cacheDirectory; } set cacheDirectory(dir) { - if (dir !== this._cacheDirectory) { - this._cacheLocationDirty = true; + if (dir !== this.#cacheDirectory) { + this.#cacheLocationDirty = true; } - this._cacheDirectory = dir; + this.#cacheDirectory = dir; } get cacheFilename() { @@ -160,16 +172,16 @@ class AssetCache { } get cache() { - if (!this._cache || this._cacheLocationDirty) { + if (!this.#cache || this.#cacheLocationDirty) { let cache = FlatCacheCreate({ cacheId: this.cacheFilename, cacheDir: this.rootDir, }); - this._cache = cache; - this._cacheLocationDirty = false; + this.#cache = cache; + this.#cacheLocationDirty = false; } - return this._cache; + return this.#cache; } getDurationMs(duration = "0s") { @@ -203,56 +215,69 @@ class AssetCache { return `${this.cachePath}.${type}`; } - get isDirEnsured() { - return this._dirEnsured; + setDirectoryManager(manager) { + this.#directoryManager = manager; } ensureDir() { - if (this.options.dryRun || this._dirEnsured) { + if (this.options.dryRun) { return; } - this._dirEnsured = true; + if(!this.#directoryManager) { + // standalone fallback (for tests) + this.#directoryManager = new DirectoryManager(); + } - fs.mkdirSync(this.cacheDirectory, { - recursive: true, - }); + this.#directoryManager.create(this.cacheDirectory); } async save(contents, type = "buffer", metadata = {}) { - if (this.options.dryRun) { - debug("An attempt was made to save to the file system with `dryRun: true`. Skipping."); - return; + if(!contents) { + throw new Error("save(contents) expects contents (was falsy)"); } - if(!this.isDirEnsured) { - this.ensureDir(); - } + this.cache.set(this.hash, { + cachedAt: Date.now(), + type: type, + metadata, + }); + + let contentPath = this.getCachedContentsPath(type); if (type === "json" || type === "parsed-xml") { contents = JSON.stringify(contents); } - let contentPath = this.getCachedContentsPath(type); + this.#rawContents[type] = contents; + + if(this.options.dryRun) { + debug(`Dry run writing ${contentPath}`); + return; + } + + this.ensureDir(); + + debugAssets("[11ty/eleventy-fetch] Writing %o from %o", contentPath, this.source); // the contents must exist before the cache metadata are saved below fs.writeFileSync(contentPath, contents); - debug(`Writing ${contentPath}`); - this.cache.set(this.hash, { - cachedAt: Date.now(), - type: type, - metadata, - }); - this.cache.save(); } - async getCachedContents(type) { + async #getCachedContents(type) { let contentPath = this.getCachedContentsPath(type); + debug(`Fetching from cache ${contentPath}`); + if(this.source) { + debugAssets("[11ty/eleventy-fetch] Reading via %o", this.source); + } else { + debugAssets("[11ty/eleventy-fetch] Reading %o", contentPath); + } + if (type === "json" || type === "parsed-xml") { return require(contentPath); } @@ -260,6 +285,15 @@ class AssetCache { return fs.readFileSync(contentPath, type !== "buffer" ? "utf8" : null); } + async getCachedContents(type) { + if(!this.#rawContents[type]) { + this.#rawContents[type] = this.#getCachedContents(type); + } + + // already saved on this instance in-memory + return this.#rawContents[type]; + } + _backwardsCompatibilityGetCachedValue(type) { if (type === "json") { return this.cachedObject.contents; @@ -291,7 +325,11 @@ class AssetCache { return this.getCachedContents(type); } - isCacheValid(duration = this.defaultDuration) { + getCachedTimestamp() { + return this.cachedObject?.cachedAt; + } + + isCacheValid(duration = this.duration) { if (!this.cachedObject) { // not cached return false; @@ -329,16 +367,16 @@ class AssetCache { return !this.isCacheValid(duration); } - async fetch(options) { - if (this.isCacheValid(options.duration)) { + // This is only included for completenes—not on the docs. + async fetch(optionsOverride = {}) { + if (this.isCacheValid(optionsOverride.duration)) { // promise - this.log(`Using cached version of: ${this.uniqueKey}`); + debug(`Using cached version of: ${this.uniqueKey}`); return this.getCachedValue(); } - this.log(`Saving ${this.uniqueKey} to ${this.cacheFilename}`); - - await this.save(this.source, options.type); + debug(`Saving ${this.uniqueKey} to ${this.cacheFilename}`); + await this.save(this.source, optionsOverride.type); return this.source; } diff --git a/src/DirectoryManager.js b/src/DirectoryManager.js new file mode 100644 index 0000000..e6ea5a5 --- /dev/null +++ b/src/DirectoryManager.js @@ -0,0 +1,24 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const debugUtil = require("debug"); +const debugAssets = debugUtil("Eleventy:Assets"); + +class DirectoryManager { + #dirs = new Set(); + + isCreated(dir) { + return this.#dirs.has(dir); + } + + create(dir) { + if(this.isCreated(dir)) { + return; + } + + this.#dirs.add(dir); + debugAssets("[11ty/eleventy-fetch] Creating directory %o", dir); + fs.mkdirSync(dir, { recursive: true }); + } +} + +module.exports = DirectoryManager; diff --git a/src/RemoteAssetCache.js b/src/RemoteAssetCache.js index a2988e5..0eeb615 100644 --- a/src/RemoteAssetCache.js +++ b/src/RemoteAssetCache.js @@ -1,10 +1,18 @@ +const debugUtil = require("debug"); const { parseXml } = require('@rgrove/parse-xml'); const Sources = require("./Sources.js"); const AssetCache = require("./AssetCache.js"); -// const debug = require("debug")("Eleventy:Fetch"); + +const debug = debugUtil("Eleventy:Fetch"); +const debugAssets = debugUtil("Eleventy:Assets"); class RemoteAssetCache extends AssetCache { + #queue; + #queuePromise; + #fetchPromise; + #lastFetchType; + constructor(source, cacheDirectory, options = {}) { let requestId = RemoteAssetCache.getRequestId(source, options); super(requestId, cacheDirectory, options); @@ -95,18 +103,51 @@ class RemoteAssetCache extends AssetCache { return Buffer.from(await response.arrayBuffer()); } - async fetch(optionsOverride = {}) { - let duration = optionsOverride.duration || this.options.duration; + setQueue(queue) { + this.#queue = queue; + } + + // Returns raw Promise + queue() { + if(!this.#queue) { + throw new Error("Missing `#queue` instance."); + } + + if(this.#queuePromise) { + return this.#queuePromise; + } + + // optionsOverride not supported on fetch here for re-use + this.#queuePromise = this.#queue.add(() => this.fetch()); + + return this.#queuePromise; + } + + isCacheValid(duration = undefined) { + // uses this.options.duration if not explicitly defined here + return super.isCacheValid(duration); + } + + // if last fetch was a cache hit (no fetch occurred) or a cache miss (fetch did occur) + // used by Eleventy Image in disk cache checks. + wasLastFetchCacheHit() { + return this.#lastFetchType === "hit"; + } + + async #fetch(optionsOverride = {}) { // Important: no disk writes when dryRun // As of Fetch v4, reads are now allowed! - if (super.isCacheValid(duration)) { - this.log(`Cache hit for ${this.displayUrl}`); + if (this.isCacheValid(optionsOverride.duration)) { + debug(`Cache hit for ${this.displayUrl}`); + this.#lastFetchType = "hit"; return super.getCachedValue(); } + this.#lastFetchType = "miss"; + try { let isDryRun = optionsOverride.dryRun || this.options.dryRun; - this.log(`${isDryRun ? "Fetching" : "Cache miss for"} ${this.displayUrl}`); + this.log(`Fetching ${this.displayUrl}`); let body; let metadata = {}; @@ -124,6 +165,8 @@ class RemoteAssetCache extends AssetCache { this.fetchCount++; + debugAssets("[11ty/eleventy-fetch] Fetching %o", this.source); + // v5: now using global (Node-native or otherwise) fetch instead of node-fetch let response = await fetch(this.source, fetchOptions); if (!response.ok) { @@ -157,13 +200,26 @@ class RemoteAssetCache extends AssetCache { return body; } catch (e) { if (this.cachedObject) { - this.log(`Error fetching ${this.displayUrl}. Message: ${e.message}`); - this.log(`Failing gracefully with an expired cache entry.`); + debug(`Error fetching ${this.displayUrl}. Message: ${e.message}`); + debug(`Failing gracefully with an expired cache entry.`); return super.getCachedValue(); } else { return Promise.reject(e); } } } + + // async but not explicitly declared for promise equality checks + // returns a Promise + async fetch(optionsOverride = {}) { + if(!this.#fetchPromise) { + // one at a time. clear when finished + this.#fetchPromise = this.#fetch(optionsOverride).finally(() => { + this.#fetchPromise = undefined; + }); + } + + return this.#fetchPromise; + } } module.exports = RemoteAssetCache; diff --git a/test/QueueTest.js b/test/QueueTest.js index 4d1b6e9..fc501ea 100644 --- a/test/QueueTest.js +++ b/test/QueueTest.js @@ -1,7 +1,6 @@ const test = require("ava"); -const Cache = require("../"); -const queue = Cache.queue; -const RemoteAssetCache = require("../src/RemoteAssetCache"); +const Cache = require("../eleventy-fetch.js"); +const { queue, Fetch } = Cache; test("Queue without options", async (t) => { let example = "https://example.com/"; @@ -28,7 +27,7 @@ test("Double Fetch", async (t) => { await ac1; await ac2; - let forDestroyOnly = new RemoteAssetCache(pngUrl); + let forDestroyOnly = Fetch(pngUrl); // file is now accessible try { await forDestroyOnly.destroy(); @@ -46,7 +45,8 @@ test("Double Fetch (dry run)", async (t) => { await ac1; await ac2; - let forTestOnly = new RemoteAssetCache(pngUrl, ".cache", { + let forTestOnly = Fetch(pngUrl, { + cacheDirectory: ".cache", dryRun: true, }); // file is now accessible @@ -95,3 +95,124 @@ test("Double Fetch 404 errors should only fetch once", async (t) => { await t.throwsAsync(async () => await ac2); }); +test("Docs example https://www.11ty.dev/docs/plugins/fetch/#manually-store-your-own-data-in-the-cache", async (t) => { + t.plan(2); + + async function fn() { + t.true(true); + return new Promise(resolve => { + setTimeout(() => { + resolve({ followerCount: 1000 }) + }); + }); + } + + let fakeFollowers = Cache(fn, { + type: "json", + dryRun: true, + requestId: "zachleat_twitter_followers" + }); + + t.deepEqual(await fakeFollowers, { + followerCount: 1000 + }); +}); + +test("Raw Fetch using queue method", async (t) => { + let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=1"; + let ac1 = Fetch(pngUrl); + let ac2 = Fetch(pngUrl); + + // Destroy to clear any existing cache + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} + + // Make sure the instance is the same + t.is(ac1, ac2); + + let result1 = await ac1.queue(); + t.false(ac1.wasLastFetchCacheHit()) + + let result2 = await ac1.queue(); + // reuses the same fetch + t.false(ac1.wasLastFetchCacheHit()) + + t.is(result1, result2); + + // file is now accessible + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} +}); + + +test("Raw Fetch using fetch method", async (t) => { + let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=2"; + let ac1 = Fetch(pngUrl); + let ac2 = Fetch(pngUrl); + + // Destroy to clear any existing cache + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} + + // Make sure the instance is the same + t.is(ac1, ac2); + + let result1 = await ac1.fetch(); + t.false(ac1.wasLastFetchCacheHit()) + + let result2 = await ac1.fetch(); + t.true(ac1.wasLastFetchCacheHit()) + + t.is(result1, result2); + + // file is now accessible + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} +}); + +test("Raw Fetch using fetch method (check parallel fetch promise reuse)", async (t) => { + let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=3"; + let ac1 = Fetch(pngUrl); + let ac2 = Fetch(pngUrl); + + // Destroy to clear any existing cache + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} + + // Make sure the instance is the same + t.is(ac1, ac2); + + let [result1, result2] = await Promise.all([ac1.fetch(), ac1.fetch()]); + + t.is(result1, result2); + + t.false(ac1.wasLastFetchCacheHit()) + + // file is now accessible + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} +}); diff --git a/test/RemoteAssetCacheTest.js b/test/RemoteAssetCacheTest.js index 8902009..15c6b08 100644 --- a/test/RemoteAssetCacheTest.js +++ b/test/RemoteAssetCacheTest.js @@ -6,6 +6,8 @@ const RemoteAssetCache = require("../src/RemoteAssetCache"); test("getDurationMs", (t) => { let cache = new RemoteAssetCache("https://example.com/"); + // t.is(cache.getDurationMs("0"), 0); + t.is(cache.getDurationMs("0s"), 0); t.is(cache.getDurationMs("1s"), 1000); t.is(cache.getDurationMs("1m"), 60 * 1000); t.is(cache.getDurationMs("1h"), 60 * 60 * 1000); @@ -14,11 +16,16 @@ test("getDurationMs", (t) => { t.is(cache.getDurationMs("1y"), 365 * 24 * 60 * 60 * 1000); t.is(cache.getDurationMs("5s"), 5000); + t.is(cache.getDurationMs("0m"), 0); t.is(cache.getDurationMs("7m"), 60 * 7000); t.is(cache.getDurationMs("9h"), 60 * 60 * 9000); + t.is(cache.getDurationMs("0h"), 0); t.is(cache.getDurationMs("11d"), 24 * 60 * 60 * 11000); + t.is(cache.getDurationMs("0d"), 0); t.is(cache.getDurationMs("13w"), 7 * 24 * 60 * 60 * 13000); + t.is(cache.getDurationMs("0w"), 0); t.is(cache.getDurationMs("15y"), 365 * 24 * 60 * 60 * 15000); + t.is(cache.getDurationMs("0y"), 0); }); test("Local hash file names", async (t) => {