Skip to content

Commit

Permalink
Check implementation of all thumbnail serverless functions. Attempt t…
Browse files Browse the repository at this point in the history 
…o fix firebase cloud function.
  • Loading branch information
@frozar
frozar committed Oct 7, 2021
1 parent 879f0bd commit 34c5fb5
Show file tree
Hide file tree
Showing 7 changed files with 138 additions and 154 deletions.
62 changes: 43 additions & 19 deletions functions/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -874,6 +874,7 @@ exports.checkThumbnails = functions

// ***** 1 - Scrap chapters for every manga available in DB
const toWait = [];
idsManga = idsManga.filter((x) => x === "gintama");
for (const idManga of idsManga) {
toWait.push(getMangaChapters(idManga));
}
Expand All @@ -893,8 +894,6 @@ exports.checkThumbnails = functions
}
}

functions.logger.log(`DBG applicationBaseUrl ${applicationBaseUrl}`);
console.log("DBG", process.env);
// Limit the number of thumbnail to (re)generate
const LIMIT_MAX_THUMBNAIL = 10;
// ***** 2 - Check every thumbnail
Expand All @@ -906,24 +905,12 @@ exports.checkThumbnails = functions
0,
LIMIT_MAX_THUMBNAIL
);
axios.post(
applicationBaseUrl + "/api/thumbnails/create",
{
mangaPath: idManga,
chapterIndexes,
},
{
headers: {
"Content-Type": "application/json",
},
}
functions.logger.log(
`${idManga} - computeMissingThumbnails ${chapterIndexes}`
);
} else {
let chapterIndexes = await fetchableThumbnail(chapters);
chapterIndexes = chapterIndexes.slice(0, LIMIT_MAX_THUMBNAIL);
if (chapterIndexes.length !== 0) {
axios.post(
applicationBaseUrl + "/api/thumbnails/recreate",
axios
.post(
applicationBaseUrl + "/api/thumbnails/create",
{
mangaPath: idManga,
chapterIndexes,
Expand All @@ -933,7 +920,44 @@ exports.checkThumbnails = functions
"Content-Type": "application/json",
},
}
)
.then(() => {
functions.logger.log(
"[check - oneThumbnailAtLeastIsMissing] OK"
);
})
.catch((error) => {
functions.logger.error(
"[check - oneThumbnailAtLeastIsMissing]",
error
);
});
} else {
let chapterIndexes = await fetchableThumbnail(chapters);
chapterIndexes = chapterIndexes.slice(0, LIMIT_MAX_THUMBNAIL);
if (chapterIndexes.length !== 0) {
functions.logger.log(
`${idManga} - unfetchableThumbnail ${chapterIndexes}`
);
axios
.post(
applicationBaseUrl + "/api/thumbnails/recreate",
{
mangaPath: idManga,
chapterIndexes,
},
{
headers: {
"Content-Type": "application/json",
},
}
)
.then(() => {
functions.logger.log("[check - fetchableThumbnail] OK");
})
.catch((error) => {
functions.logger.error("[check - fetchableThumbnail]", error);
});
}
}
}
Expand Down
64 changes: 25 additions & 39 deletions pages/api/thumbnail/recreate.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
const fs = require("fs");

import { db, storage, functions } from "../../../utils/serverSide/firebase";
import { db, functions } from "../../../utils/serverSide/firebase";
import {
thumbnailURLtoStoragePath,
createThumbnail,
createAndUploadThumbnail,
deleteThumbnailFromStorage,
} from "../../../utils/serverSide/thumbnail";

const LELSCANS_ROOT = "lelscans";
Expand Down Expand Up @@ -34,47 +32,35 @@ export default async (req, res) => {
const snapshot = await docRef.get();
const chapters = snapshot.data();

const chapterIndexes = [chapterIndex];

// ***** 2 - Delete thumbnail in bucket
const storageBucket = storage.bucket();
if (chapters[chapterIndex].thumbnail !== "") {
const thumbnailPath = thumbnailURLtoStoragePath(
chapters[chapterIndex].thumbnail
);
try {
await storageBucket.file(thumbnailPath).delete();
} catch (error) {
functions.logger.error("Cannot delete ", thumbnailPath);
}
const toWait0 = [];
for (const idx of chapterIndexes) {
toWait0.push(deleteThumbnailFromStorage(chapters, idx));
}

await Promise.all(toWait0);
// 2.0 - If chapter has a thumbnail in DB, delete it
if (chapters[chapterIndex].thumbnail !== "") {
chapters[chapterIndex].thumbnail = "";
}
chapterIndexes.forEach((idx) => {
if (chapters[idx].thumbnail !== "") {
chapters[idx].thumbnail = "";
}
});

// ***** 3 - Compute thumbnail
const uri = chapters[chapterIndex].content[0];
const [thumbnailFileName, thumbnailPath] = await createThumbnail(uri);

const uploadFile = async (filePath, destFileName) => {
const [resUpload] = await storageBucket.upload(filePath, {
destination: destFileName,
public: true,
});

const [metadata] = await resUpload.getMetadata();
const url = metadata.mediaLink;
functions.logger.log("[recreate] new thumbnail", url);

return url;
};

const destFileName = "thumbnails/" + thumbnailFileName;
const thumbnailURL = await uploadFile(thumbnailPath, destFileName);
fs.unlinkSync(thumbnailPath);
// 3.0 - Trigger the creation of all thumbnails
const indexNThumbnail = [];
const toWait1 = [];
chapterIndexes.forEach((idx) => {
toWait1.push(createAndUploadThumbnail(chapters, idx, indexNThumbnail));
});
await Promise.all(toWait1);

// ***** 4 - Update the chapter field in DB to write
chapters[chapterIndex].thumbnail = thumbnailURL;
for (const [idx, url] of indexNThumbnail) {
functions.logger.log(`[thumbnail - recreate] write ${mangaPath} ${idx}`);
chapters[idx].thumbnail = url;
}

// ***** 5 - Write updated chapter field in DB
docRef.set(chapters, { merge: true });
Expand Down
7 changes: 5 additions & 2 deletions pages/api/thumbnails/check.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ export default async (req, res) => {
}
}

functions.logger.log(`DBG applicationBaseUrl ${applicationBaseUrl}`);
console.log("DBG", process.env);
// functions.logger.log(`DBG applicationBaseUrl ${applicationBaseUrl}`);
// console.log("DBG", process.env);
// Limit the number of thumbnail to (re)generate
const LIMIT_MAX_THUMBNAIL = 10;
// ***** 2 - Check every thumbnail
Expand Down Expand Up @@ -110,6 +110,9 @@ export default async (req, res) => {
let chapterIndexes = await fetchableThumbnail(chapters);
chapterIndexes = chapterIndexes.slice(0, LIMIT_MAX_THUMBNAIL);
if (chapterIndexes.length !== 0) {
functions.logger.log(
`${idManga} - fetchableThumbnail ${fetchableThumbnail}`
);
axios.post(
applicationBaseUrl + "/api/thumbnails/recreate",
{
Expand Down
44 changes: 7 additions & 37 deletions pages/api/thumbnails/create.js
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
const fs = require("fs");

import { db, storage, functions } from "../../../utils/serverSide/firebase";
import { db, functions } from "../../../utils/serverSide/firebase";
import { setCORSHeader } from "../../../utils/serverSide/request";
import { createThumbnail } from "../../../utils/serverSide/thumbnail";
import { createAndUploadThumbnail } from "../../../utils/serverSide/thumbnail";

const LELSCANS_ROOT = "lelscans";

Expand Down Expand Up @@ -44,45 +42,17 @@ export default async (req, res) => {
}

// ***** 2 - Compute thumbnail
const indexNThumbnail = [];
const process = async (idx) => {
const uri = chapters[idx].content[0];
const [thumbnailFileName, thumbnailPath] = await createThumbnail(uri);

const uploadFile = async (filePath, destFileName) => {
const storageBucket = storage.bucket();
const [resUpload] = await storageBucket.upload(filePath, {
destination: destFileName,
public: true,
});

const [metadata] = await resUpload.getMetadata();
const url = metadata.mediaLink;
functions.logger.log("[create] new thumbnail", url);
indexNThumbnail.push([idx, url]);
};

if (thumbnailFileName !== null && thumbnailPath !== null) {
const destFileName = "thumbnails/" + thumbnailFileName;
await uploadFile(thumbnailPath, destFileName);
fs.unlinkSync(thumbnailPath);
}
if (thumbnailFileName === null && thumbnailPath !== null) {
functions.logger.log(`[create] ${idx} : thumbnail placeholder`);
indexNThumbnail.push([idx, thumbnailPath]);
}
};

// 2.0 - Trigger the creation of all thumbnails
const toWait = [];
const indexNThumbnail = [];
const toWait1 = [];
chapterIndexes.forEach((idx) => {
toWait.push(process(idx));
toWait1.push(createAndUploadThumbnail(chapters, idx, indexNThumbnail));
});
await Promise.all(toWait);
await Promise.all(toWait1);

// ***** 3 - Update the chapter field in DB to write
for (const [idx, url] of indexNThumbnail) {
console.log(`[create] ${idx}`);
functions.logger.log(`[thumbnails - create] write ${mangaPath} ${idx}`);
chapters[idx].thumbnail = url;
}

Expand Down
13 changes: 3 additions & 10 deletions pages/api/thumbnails/delete.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { db, storage, functions } from "../../../utils/serverSide/firebase";
import { db, functions } from "../../../utils/serverSide/firebase";
import { setCORSHeader } from "../../../utils/serverSide/request";
import { thumbnailURLtoStoragePath } from "../../../utils/serverSide/thumbnail";
import { deleteThumbnailFromStorage } from "../../../utils/serverSide/thumbnail";

const LELSCANS_ROOT = "lelscans";

Expand Down Expand Up @@ -46,17 +46,10 @@ export default async (req, res) => {
}

// ***** 2 - Delete thumbnail in bucket
const storageBucket = storage.bucket();
try {
const toWait = [];
chapterIndexes.forEach((idx) => {
const process = async (idx) => {
const thumbnailPath = thumbnailURLtoStoragePath(
chapters[idx].thumbnail
);
await storageBucket.file(thumbnailPath).delete();
};
toWait.push(process(idx));
toWait.push(deleteThumbnailFromStorage(chapters, idx));
});
await Promise.all(toWait);
} catch (error) {
Expand Down
50 changes: 7 additions & 43 deletions pages/api/thumbnails/recreate.js
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
const fs = require("fs");

import { db, storage, functions } from "../../../utils/serverSide/firebase";
import { db, functions } from "../../../utils/serverSide/firebase";
import { setCORSHeader } from "../../../utils/serverSide/request";
import {
thumbnailURLtoStoragePath,
createThumbnail,
deleteThumbnailFromStorage,
createAndUploadThumbnail,
} from "../../../utils/serverSide/thumbnail";

const LELSCANS_ROOT = "lelscans";
Expand Down Expand Up @@ -39,22 +37,9 @@ export default async (req, res) => {
const chapters = snapshot.data();

// ***** 2 - Delete thumbnail in bucket
const storageBucket = storage.bucket();
const toWait0 = [];
for (const idx of chapterIndexes) {
const process = async (idx) => {
if (chapters[idx].thumbnail !== "") {
const thumbnailPath = thumbnailURLtoStoragePath(
chapters[idx].thumbnail
);
try {
await storageBucket.file(thumbnailPath).delete();
} catch (error) {
functions.logger.error("Cannot delete ", thumbnailPath);
}
}
};
toWait0.push(process(idx));
toWait0.push(deleteThumbnailFromStorage(chapters, idx));
}
await Promise.all(toWait0);
// 2.0 - If chapter has a thumbnail in DB, delete it
Expand All @@ -65,38 +50,17 @@ export default async (req, res) => {
});

// ***** 3 - Compute thumbnail
const indexNThumbnail = [];
const process = async (idx) => {
const uri = chapters[idx].content[0];
const [thumbnailFileName, thumbnailPath] = await createThumbnail(uri);

const uploadFile = async (filePath, destFileName) => {
const storageBucket = storage.bucket();
const [resUpload] = await storageBucket.upload(filePath, {
destination: destFileName,
public: true,
});

const [metadata] = await resUpload.getMetadata();
const url = metadata.mediaLink;
functions.logger.log("[create] new thumbnail", url);
indexNThumbnail.push([idx, url]);
};

const destFileName = "thumbnails/" + thumbnailFileName;
await uploadFile(thumbnailPath, destFileName);
fs.unlinkSync(thumbnailPath);
};

// 3.0 - Trigger the creation of all thumbnails
const indexNThumbnail = [];
const toWait1 = [];
chapterIndexes.forEach((idx) => {
toWait1.push(process(idx));
toWait1.push(createAndUploadThumbnail(chapters, idx, indexNThumbnail));
});
await Promise.all(toWait1);

// ***** 4 - Update the chapter field in DB to write
for (const [idx, url] of indexNThumbnail) {
functions.logger.log(`[thumbnails - recreate] write ${mangaPath} ${idx}`);
chapters[idx].thumbnail = url;
}

Expand Down
Loading

1 comment on commit 34c5fb5

@vercel
Copy link

@vercel vercel bot commented on 34c5fb5 Oct 7, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Successfully deployed to the following URLs:

Please sign in to comment.