Skip to content

Commit

Permalink
Adds wasLastFetchHit method used by Eleventy Image plugin to invali…
Browse files Browse the repository at this point in the history 
…date disk cache. Removes `setInitialCacheTimestamp` which is no longer needed for this use case (and wasn’t released). Changes in memory cache to persist instances after requests complete.
  • Loading branch information
@zachleat
zachleat committed Dec 19, 2024
1 parent 51311c8 commit c608520
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 86 deletions.
43 changes: 26 additions & 17 deletions eleventy-fetch.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,33 +34,34 @@ queue.on("active", () => {
debug(`Concurrency: ${queue.concurrency}, Size: ${queue.size}, Pending: ${queue.pending}`);
});

let inProgress = {};
let instCache = {};

function queueSave(source, queueCallback, options) {
function createRemoteAssetCache(source, rawOptions = {}) {
if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) {
return Promise.reject(new Error("Invalid source. Received: " + source));
}

let options = Object.assign({}, globalOptions, rawOptions);
let sourceKey = RemoteAssetCache.getRequestId(source, options);
if(!sourceKey) {
return Promise.reject(Sources.getInvalidSourceError(source));
}

if (!inProgress[sourceKey]) {
inProgress[sourceKey] = queue.add(queueCallback).finally(() => {
delete inProgress[sourceKey];
});
if(instCache[sourceKey]) {
return instCache[sourceKey];
}

return inProgress[sourceKey];
let inst = new RemoteAssetCache(source, options.directory, options);
inst.setQueue(queue);

instCache[sourceKey] = inst;

return inst;
}

module.exports = function (source, options) {
if (!Sources.isFullUrl(source) && !Sources.isValidSource(source)) {
throw new Error("Caching an already local asset is not yet supported.");
}

let mergedOptions = Object.assign({}, globalOptions, options);
return queueSave(source, () => {
let asset = new RemoteAssetCache(source, mergedOptions.directory, mergedOptions);
return asset.fetch(mergedOptions);
}, mergedOptions);
let instance = createRemoteAssetCache(source, options);
return instance.queue();
};

Object.defineProperty(module.exports, "concurrency", {
Expand All @@ -72,7 +73,15 @@ Object.defineProperty(module.exports, "concurrency", {
},
});

module.exports.queue = queueSave;
module.exports.Fetch = createRemoteAssetCache;

// Deprecated API kept for backwards compat, instead: use default export directly.
// Intentional: queueCallback is ignored here
module.exports.queue = function(source, queueCallback, options) {
let instance = createRemoteAssetCache(source, options);
return instance.queue();
};

module.exports.Util = {
isFullUrl: Sources.isFullUrl,
};
Expand Down
34 changes: 15 additions & 19 deletions src/AssetCache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ class AssetCache {
this.hash = AssetCache.getHash(uniqueKey, options.hashLength);

this.cacheDirectory = cacheDirectory || ".cache";
this.defaultDuration = "1d";
this.options = options;

this.defaultDuration = "1d";
this.duration = options.duration || this.defaultDuration;

// Compute the filename only once
if (typeof this.options.filenameFormat === "function") {
this.#customFilename = AssetCache.cleanFilename(this.options.filenameFormat(uniqueKey, this.hash));
Expand All @@ -39,10 +41,6 @@ class AssetCache {
}
}

setInitialCacheTimestamp(timestamp) {
this.initialCacheTimestamp = timestamp;
}

log(message) {
if (this.options.verbose) {
console.log(`[11ty/eleventy-fetch] ${message}`);
Expand Down Expand Up @@ -234,31 +232,29 @@ class AssetCache {
throw new Error("save(contents) expects contents (was falsy)");
}

let contentPath = this.getCachedContentsPath(type);
if(this.options.dryRun) {
debug(`Dry run writing ${contentPath}`);
return;
}

this.ensureDir();

if (type === "json" || type === "parsed-xml") {
contents = JSON.stringify(contents);
}

let contentPath = this.getCachedContentsPath(type);

// the contents must exist before the cache metadata are saved below
if(!this.options.dryRun) {
fs.writeFileSync(contentPath, contents);
debug(`Writing ${contentPath}`);
} else {
debug(`Dry run writing ${contentPath}`);
}
fs.writeFileSync(contentPath, contents);
debug(`Writing ${contentPath}`);

this.cache.set(this.hash, {
cachedAt: this.initialCacheTimestamp || Date.now(),
cachedAt: Date.now(),
type: type,
metadata,
});

if(!this.options.dryRun) {
this.cache.save();
}
this.cache.save();
}

async getCachedContents(type) {
Expand Down Expand Up @@ -304,10 +300,10 @@ class AssetCache {
}

getCachedTimestamp() {
return this.cachedObject?.cachedAt || this.initialCacheTimestamp;
return this.cachedObject?.cachedAt;
}

isCacheValid(duration = this.defaultDuration) {
isCacheValid(duration = this.duration) {
if (!this.cachedObject) {
// not cached
return false;
Expand Down
44 changes: 41 additions & 3 deletions src/RemoteAssetCache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@ const { parseXml } = require('@rgrove/parse-xml');

const Sources = require("./Sources.js");
const AssetCache = require("./AssetCache.js");
// const debug = require("debug")("Eleventy:Fetch");
const assetDebug = require("debug")("Eleventy:Assets");

class RemoteAssetCache extends AssetCache {
#queue;
#queuePromise;
#lastFetchType;

constructor(source, cacheDirectory, options = {}) {
let requestId = RemoteAssetCache.getRequestId(source, options);
super(requestId, cacheDirectory, options);
Expand Down Expand Up @@ -95,15 +99,48 @@ class RemoteAssetCache extends AssetCache {
return Buffer.from(await response.arrayBuffer());
}

setQueue(queue) {
this.#queue = queue;
}

// Returns raw Promise
queue() {
if(!this.#queue) {
throw new Error("Missing `#queue` instance.");
}

if(this.#queuePromise) {
return this.#queuePromise;
}

// optionsOverride not supported on fetch here for re-use
this.#queuePromise = this.#queue.add(() => this.fetch());

return this.#queuePromise;
}

isCacheValid(duration = undefined) {
// uses this.options.duration if not explicitly defined here
return super.isCacheValid(duration);
}

// if last fetch was a cache hit (no fetch occurred) or a cache miss (fetch did occur)
// used by Eleventy Image in disk cache checks.
wasLastFetchHit() {
return this.#lastFetchType === "hit";
}

async fetch(optionsOverride = {}) {
let duration = optionsOverride.duration || this.options.duration;
// Important: no disk writes when dryRun
// As of Fetch v4, reads are now allowed!
if (super.isCacheValid(duration)) {
if (this.isCacheValid(optionsOverride.duration)) {
this.log(`Cache hit for ${this.displayUrl}`);
this.#lastFetchType = "hit";
return super.getCachedValue();
}

this.#lastFetchType = "miss";

try {
let isDryRun = optionsOverride.dryRun || this.options.dryRun;
this.log(`${isDryRun ? "Fetching" : "Cache miss for"} ${this.displayUrl}`);
Expand All @@ -124,6 +161,7 @@ class RemoteAssetCache extends AssetCache {

this.fetchCount++;

assetDebug("Fetching remote asset: %o", this.source);
// v5: now using global (Node-native or otherwise) fetch instead of node-fetch
let response = await fetch(this.source, fetchOptions);
if (!response.ok) {
Expand Down
42 changes: 0 additions & 42 deletions test/AssetCacheTest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,45 +137,3 @@ test("Uses filenameFormat", async (t) => {
t.falsy(fs.existsSync(cachePath));
t.falsy(fs.existsSync(jsonCachePath));
});

test("setInitialCacheTimestamp method (used by Eleventy Image to establish a consistent cached file name in synchronous contexts)", async (t) => {
let cache = new AssetCache("this_is_a_test", ".cache", {
dryRun: true
});
let timestamp = (new Date(2024,1,1)).getTime();
cache.setInitialCacheTimestamp(timestamp);

await cache.save("test");

t.is(cache.getCachedTimestamp(), timestamp);
});

test("setInitialCacheTimestamp method after save is ignored", async (t) => {
let cache = new AssetCache("this_is_a_test2", ".cache", {
dryRun: true
});

let timestamp = (new Date(2024,1,1)).getTime();

await cache.save("test");

cache.setInitialCacheTimestamp(timestamp);

t.not(cache.getCachedTimestamp(), timestamp);
});

test("setInitialCacheTimestamp method before a second save is used", async (t) => {
let cache = new AssetCache("this_is_a_test3", ".cache", {
dryRun: true
});

let timestamp = (new Date(2024,1,1)).getTime();

await cache.save("test");

cache.setInitialCacheTimestamp(timestamp);

await cache.save("test");

t.is(cache.getCachedTimestamp(), timestamp);
});
10 changes: 5 additions & 5 deletions test/QueueTest.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
const test = require("ava");
const Cache = require("../");
const queue = Cache.queue;
const RemoteAssetCache = require("../src/RemoteAssetCache");
const Cache = require("../eleventy-fetch.js");
const { queue, Fetch } = Cache;

test("Queue without options", async (t) => {
let example = "https://example.com/";
Expand All @@ -28,7 +27,7 @@ test("Double Fetch", async (t) => {
await ac1;
await ac2;

let forDestroyOnly = new RemoteAssetCache(pngUrl);
let forDestroyOnly = Fetch(pngUrl);
// file is now accessible
try {
await forDestroyOnly.destroy();
Expand All @@ -46,7 +45,8 @@ test("Double Fetch (dry run)", async (t) => {
await ac1;
await ac2;

let forTestOnly = new RemoteAssetCache(pngUrl, ".cache", {
let forTestOnly = Fetch(pngUrl, {
cacheDirectory: ".cache",
dryRun: true,
});
// file is now accessible
Expand Down

0 comments on commit c608520

Please sign in to comment.