fix(api-page-builder-import-export): bg tasks for pages export and im…

…port (#3851)
webiny · Mar 11, 2024 · 627d334 · 627d334
1 parent c0c5b23
commit 627d334
Show file tree

Hide file tree

Showing 141 changed files with 4,118 additions and 2,085 deletions.
diff --git a/packages/api-aco/src/createAcoContext.ts b/packages/api-aco/src/createAcoContext.ts
@@ -89,18 +89,31 @@ const setupAcoContext = async (
             });
 
             return withModel(async model => {
-                const results = await context.cms.storageOperations.entries.list(model, {
-                    limit: 100_000,
-                    where: {
-                        type,
-
-                        // Folders always work with latest entries. We never publish them.
-                        latest: true
-                    },
-                    sort: ["title_ASC"]
-                });
-
-                return results.items.map(pickEntryFieldValues<Folder>);
+                try {
+                    const results = await context.cms.storageOperations.entries.list(model, {
+                        limit: 100_000,
+                        where: {
+                            type,
+
+                            // Folders always work with latest entries. We never publish them.
+                            latest: true
+                        },
+                        sort: ["title_ASC"]
+                    });
+
+                    return results.items.map(pickEntryFieldValues<Folder>);
+                } catch (ex) {
+                    /**
+                     * Skip throwing an error if the error is related to the search phase execution.
+                     * This is a temporary solution to avoid breaking the entire system when no entries were ever inserted in the index.
+                     *
+                     * TODO: figure out better way to handle this.
+                     */
+                    if (ex.message === "search_phase_execution_exception") {
+                        return [];
+                    }
+                    throw ex;
+                }
             });
         },
         canUseTeams: () => context.wcp.canUseTeams(),

diff --git a/packages/api-elasticsearch-tasks/__tests__/mocks/log.ts b/packages/api-elasticsearch-tasks/__tests__/mocks/log.ts
@@ -0,0 +1,14 @@
+import { ITask, ITaskLog } from "@webiny/tasks";
+import { createMockIdentity } from "~tests/mocks/identity";
+
+export const createTaskLogMock = (task: Pick<ITask, "id">): ITaskLog => {
+    return {
+        id: "acustomtasklogid",
+        createdOn: new Date().toISOString(),
+        createdBy: createMockIdentity(),
+        executionName: "acustomexecutionname",
+        task: task.id,
+        iteration: 1,
+        items: []
+    };
+};
diff --git a/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts b/packages/api-elasticsearch-tasks/__tests__/mocks/store.ts
@@ -1,14 +1,17 @@
 import { TaskManagerStore } from "@webiny/tasks/runner/TaskManagerStore";
-import { Context, ITaskData } from "@webiny/tasks/types";
+import { Context, ITask, ITaskLog } from "@webiny/tasks/types";
 import { createTaskMock } from "~tests/mocks/task";
 import { createContextMock } from "~tests/mocks/context";
+import { createTaskLogMock } from "~tests/mocks/log";
 
 interface Params {
     context?: Context;
-    task?: ITaskData;
+    task?: ITask;
+    log?: ITaskLog;
 }
 export const createTaskManagerStoreMock = (params?: Params) => {
     const context = params?.context || createContextMock();
     const task = params?.task || createTaskMock();
-    return new TaskManagerStore(context, task);
+    const log = params?.log || createTaskLogMock(task);
+    return new TaskManagerStore(context, task, log);
 };
diff --git a/packages/api-elasticsearch-tasks/__tests__/mocks/task.ts b/packages/api-elasticsearch-tasks/__tests__/mocks/task.ts
@@ -1,18 +1,19 @@
 import { createMockIdentity } from "./identity";
-import { ITaskData, TaskDataStatus } from "@webiny/tasks/types";
+import { ITask, TaskDataStatus } from "@webiny/tasks/types";
 
-export const createTaskMock = (task?: Partial<ITaskData>): ITaskData => {
+export const createTaskMock = (task?: Partial<ITask>): ITask => {
     return {
         id: "myCustomTaskDataId",
         definitionId: "myCustomTaskDefinition",
         input: {},
         name: "A custom task defined via method",
-        log: [],
         createdOn: new Date().toISOString(),
         savedOn: new Date().toISOString(),
         taskStatus: TaskDataStatus.PENDING,
         createdBy: createMockIdentity(),
         eventResponse: undefined,
+        executionName: "mycustomexecutionname",
+        iterations: 0,
         ...task
     };
 };
diff --git a/packages/api-elasticsearch-tasks/__tests__/tasks/reindexing/reindexing.test.ts b/packages/api-elasticsearch-tasks/__tests__/tasks/reindexing/reindexing.test.ts
@@ -69,16 +69,6 @@ describe("reindexing", () => {
             finishedOn: expect.toBeDateString(),
             taskStatus: TaskDataStatus.SUCCESS,
             iterations: 1
-            // log: [
-            //     {
-            //         message: "Task started.",
-            //         createdOn: expect.toBeDateString()
-            //     },
-            //     {
-            //         createdOn: expect.toBeDateString(),
-            //         message: "No more items to process - no last evaluated keys."
-            //     }
-            // ]
         });
     });
 
@@ -111,17 +101,6 @@ describe("reindexing", () => {
             finishedOn: undefined,
             taskStatus: TaskDataStatus.RUNNING,
             iterations: 1
-            // log: [
-            //     {
-            //         message: "Task started.",
-            //         createdOn: expect.toBeDateString()
-            //     },
-            //     {
-            //         createdOn: expect.toBeDateString(),
-            //         message: "Task continuing.",
-            //         input: {}
-            //     }
-            // ]
         });
         /**
          * Should end the task when there are no more items to process.
@@ -152,21 +131,6 @@ describe("reindexing", () => {
             finishedOn: expect.toBeDateString(),
             taskStatus: TaskDataStatus.SUCCESS,
             iterations: 2
-            // log: [
-            //     {
-            //         message: "Task started.",
-            //         createdOn: expect.toBeDateString()
-            //     },
-            //     {
-            //         createdOn: expect.toBeDateString(),
-            //         message: "Task continuing.",
-            //         input: {}
-            //     },
-            //     {
-            //         createdOn: expect.toBeDateString(),
-            //         message: "No more items to process - no last evaluated keys."
-            //     }
-            // ]
         });
     });
 });
diff --git a/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts b/packages/api-elasticsearch-tasks/src/tasks/reindexing/reindexingTaskDefinition.ts
@@ -7,16 +7,14 @@ export const createElasticsearchReindexingTask = (params?: IElasticsearchTaskCon
         title: "Elasticsearch reindexing",
         run: async ({ context, isCloseToTimeout, response, input, isAborted, store }) => {
             const { Manager } = await import(
-                /* webpackChunkName: "ApiElasticsearchTasksReindexingManager" */
+                /* webpackChunkName: "ElasticsearchReindexingManager" */
                 "../Manager"
             );
             const { IndexManager } = await import(
-                /* webpackChunkName: "ApiElasticsearchTasksReindexingSettings" */
-                "~/settings"
+                /* webpackChunkName: "ElasticsearchReindexingSettings" */ "~/settings"
             );
             const { ReindexingTaskRunner } = await import(
-                /* webpackChunkName: "ApiElasticsearchTasksReindexingTaskRunner" */
-                "./ReindexingTaskRunner"
+                /* webpackChunkName: "ElasticsearchReindexingTaskRunner" */ "./ReindexingTaskRunner"
             );
 
             const manager = new Manager({

diff --git a/packages/api-headless-cms-ddb-es/__tests__/tasks/createIndexTask.test.ts b/packages/api-headless-cms-ddb-es/__tests__/tasks/createIndexTask.test.ts
@@ -212,6 +212,38 @@ describe("create index task", () => {
             webinyTaskId: task.id
         });
 
+        expect(result).toEqual({
+            status: "done",
+            webinyTaskId: task.id,
+            webinyTaskDefinitionId: createIndexesTask.id,
+            tenant: "root",
+            locale: "en-US",
+            message: "Indexes created.",
+            output: {
+                done: [
+                    createIndexName({
+                        tenant: "root",
+                        locale: "en-US",
+                        modelId: "car"
+                    }),
+                    createIndexName({
+                        tenant: "webiny",
+                        locale: "en-US",
+                        modelId: "car"
+                    }),
+                    createIndexName({
+                        tenant: "dev",
+                        locale: "en-US",
+                        modelId: "car"
+                    }),
+                    createIndexName({
+                        tenant: "sales",
+                        locale: "en-US",
+                        modelId: "car"
+                    })
+                ]
+            }
+        });
         expect(result).toBeInstanceOf(ResponseDoneResult);
 
         const doneTask = await context.tasks.getTask(task.id);

diff --git a/packages/api-headless-cms/src/utils/caching/CacheKey.ts b/packages/api-headless-cms/src/utils/caching/CacheKey.ts
@@ -1,29 +1,16 @@
-import crypto from "crypto";
-import { ICacheKey, ICacheKeyKeys } from "./types";
-
-const getCacheKey = (input: ICacheKeyKeys): string => {
-    if (typeof input === "string") {
-        return input;
-    } else if (typeof input === "number") {
-        return `${input}`;
-    }
-    return JSON.stringify(input);
-};
-
-const createHash = (input: ICacheKeyKeys): string => {
-    const key = getCacheKey(input);
-    const hash = crypto.createHash("sha1");
-    hash.update(key);
-    return hash.digest("hex");
-};
+import { ICacheKey } from "./types";
+import { createCacheKey as createCacheKeyValue, ICacheKeyKeys } from "@webiny/utils";
 
 class CacheKey implements ICacheKey {
     private readonly key: string;
     public readonly keys: ICacheKeyKeys;
 
     private constructor(keys: ICacheKeyKeys) {
         this.keys = keys;
-        this.key = createHash(keys);
+        this.key = createCacheKeyValue(keys, {
+            algorithm: "sha1",
+            encoding: "hex"
+        });
     }
 
     public static create(key: ICacheKeyKeys): ICacheKey {

diff --git a/packages/api-headless-cms/src/utils/caching/types.ts b/packages/api-headless-cms/src/utils/caching/types.ts
@@ -1,4 +1,6 @@
-export type ICacheKeyKeys = Record<string, string | undefined> | string | number;
+import { ICacheKeyKeys } from "@webiny/utils";
+
+export { ICacheKeyKeys };
 
 export interface ICacheKey {
     get(): string;

diff --git a/packages/api-page-builder-import-export/package.json b/packages/api-page-builder-import-export/package.json
@@ -16,6 +16,7 @@
   "dependencies": {
     "@babel/runtime": "^7.22.6",
     "@commodo/fields": "beta",
+    "@smithy/node-http-handler": "^2.1.6",
     "@webiny/api": "0.0.0",
     "@webiny/api-file-manager": "0.0.0",
     "@webiny/api-form-builder": "0.0.0",
@@ -27,9 +28,10 @@
     "@webiny/handler-aws": "0.0.0",
     "@webiny/handler-graphql": "0.0.0",
     "@webiny/pubsub": "0.0.0",
+    "@webiny/tasks": "0.0.0",
     "@webiny/utils": "0.0.0",
     "@webiny/validation": "0.0.0",
-    "archiver": "^5.3.0",
+    "archiver": "^7.0.0",
     "commodo-fields-object": "^1.0.6",
     "dot-prop-immutable": "^2.1.0",
     "fs-extra": "^9.1.0",

diff --git a/packages/api-page-builder-import-export/src/export/combine/index.ts b/packages/api-page-builder-import-export/src/export/combine/index.ts
@@ -3,7 +3,6 @@ import { SecurityIdentity } from "@webiny/api-security/types";
 import { createRawEventHandler } from "@webiny/handler-aws";
 import { blocksHandler } from "~/export/combine/blocksHandler";
 import { formsHandler } from "~/export/combine/formsHandler";
-import { pagesHandler } from "~/export/combine/pagesHandler";
 import { templatesHandler } from "~/export/combine/templatesHandler";
 
 export interface Payload {
@@ -35,7 +34,8 @@ export default () => {
                         return templatesHandler(payload, context);
                     }
                     default: {
-                        return pagesHandler(payload, context);
+                        console.log("Export PB combine", JSON.stringify(payload));
+                        throw new Error("Invalid type provided: pb combine.");
                     }
                 }
             });