feat(api-elasticsearch): add locale to index names (#2323)

webiny · Apr 13, 2022 · 1b480ea · 1b480ea
1 parent e33bc4a
commit 1b480ea
Show file tree

Hide file tree

Showing 28 changed files with 591 additions and 126 deletions.
diff --git a/package.json b/package.json
@@ -137,6 +137,7 @@
     "prettier": "prettier \"**/**/*.{js,jsx,ts,tsx,json}\" --config .prettierrc.js",
     "prettier:check": "yarn prettier --check",
     "prettier:fix": "yarn prettier --write",
+    "lint:fix": "yarn eslint:fix && yarn prettier:fix",
     "setup-project": "node scripts/setupProject",
     "setup-env-files": "node scripts/setupEnvFiles",
     "setup-ci-cypress": "node scripts/setupCiCypress",

diff --git a/packages/api-file-manager-ddb-es/__tests__/elasticsearchIndex.test.ts b/packages/api-file-manager-ddb-es/__tests__/elasticsearchIndex.test.ts
@@ -0,0 +1,93 @@
+import { configurations } from "~/configurations";
+
+describe("Elasticsearch index", () => {
+    const withLocaleItems = [
+        ["root", "en-US"],
+        ["admin", "en-EN"],
+        ["root", "de-DE"],
+        ["admin", "en-GB"],
+        ["root,", "de"]
+    ];
+
+    beforeEach(() => {
+        process.env.ELASTICSEARCH_SHARED_INDEXES = undefined;
+        process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = undefined;
+    });
+
+    it.each(withLocaleItems)(
+        "should create index with locale code as part of the name",
+        async (tenant, locale) => {
+            process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
+
+            const { index } = configurations.es({
+                tenant,
+                locale
+            });
+
+            expect(index).toEqual(`${tenant}-${locale}-file-manager`.toLowerCase());
+        }
+    );
+
+    it.each(withLocaleItems)(
+        "should create index without locale code as part of the name",
+        async (tenant, locale) => {
+            const { index } = configurations.es({
+                tenant,
+                locale
+            });
+
+            expect(index).toEqual(`${tenant}-file-manager`.toLowerCase());
+        }
+    );
+
+    it("should throw error when missing tenant but it is required", async () => {
+        expect(() => {
+            configurations.es({
+                tenant: null as any,
+                locale: "en-US"
+            });
+        }).toThrowError(
+            `Missing "tenant" parameter when trying to create Elasticsearch index name.`
+        );
+    });
+
+    it("should throw error when missing locale but it is required", async () => {
+        process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
+
+        expect(() => {
+            configurations.es({
+                tenant: "root",
+                locale: null as any
+            });
+        }).toThrowError(
+            `Missing "locale" parameter when trying to create Elasticsearch index name.`
+        );
+    });
+
+    it.each(withLocaleItems)(
+        "should be root tenant in the index, no matter which one is sent",
+        async (tenant, locale) => {
+            process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
+
+            const { index: noLocaleIndex } = configurations.es({
+                tenant,
+                locale
+            });
+            expect(noLocaleIndex).toEqual("root-file-manager");
+        }
+    );
+
+    it.each(withLocaleItems)(
+        "should be root tenant in the index, no matter which one is sent",
+        async (tenant, locale) => {
+            process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
+            process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
+
+            const { index: noLocaleIndex } = configurations.es({
+                tenant,
+                locale
+            });
+            expect(noLocaleIndex).toEqual(`root-${locale}-file-manager`.toLowerCase());
+        }
+    );
+});
diff --git a/packages/api-file-manager-ddb-es/src/configurations.ts b/packages/api-file-manager-ddb-es/src/configurations.ts
@@ -0,0 +1,47 @@
+import WebinyError from "@webiny/error";
+
+export interface ElasticsearchConfigurationParams {
+    tenant: string;
+    locale: string;
+}
+export const configurations = {
+    es(params: ElasticsearchConfigurationParams) {
+        const { tenant, locale } = params;
+
+        if (!tenant) {
+            throw new WebinyError(
+                `Missing "tenant" parameter when trying to create Elasticsearch index name.`,
+                "TENANT_ERROR"
+            );
+        }
+
+        const sharedIndex = process.env.ELASTICSEARCH_SHARED_INDEXES === "true";
+
+        const tenantId = sharedIndex ? "root" : tenant;
+        let localeCode: string | null = null;
+        if (process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE === "true") {
+            if (!locale) {
+                throw new WebinyError(
+                    `Missing "locale" parameter when trying to create Elasticsearch index name.`,
+                    "LOCALE_ERROR"
+                );
+            }
+            localeCode = locale;
+        }
+
+        const index = [tenantId, localeCode, "file-manager"]
+            .filter(Boolean)
+            .join("-")
+            .toLowerCase();
+
+        const prefix = process.env.ELASTIC_SEARCH_INDEX_PREFIX;
+        if (!prefix) {
+            return {
+                index
+            };
+        }
+        return {
+            index: prefix + index
+        };
+    }
+};
diff --git a/packages/api-file-manager-ddb-es/src/operations/configurations.ts b/packages/api-file-manager-ddb-es/src/operations/configurations.ts
diff --git a/packages/api-file-manager-ddb-es/src/operations/files/FilesStorageOperations.ts b/packages/api-file-manager-ddb-es/src/operations/files/FilesStorageOperations.ts
@@ -20,7 +20,7 @@ import defineTable from "~/definitions/table";
 import defineEsTable from "~/definitions/tableElasticsearch";
 import defineFilesEntity from "~/definitions/filesEntity";
 import defineFilesEsEntity from "~/definitions/filesElasticsearchEntity";
-import { configurations } from "~/operations/configurations";
+import { configurations } from "~/configurations";
 import { decodeCursor, encodeCursor } from "@webiny/api-elasticsearch/cursors";
 import { createElasticsearchBody } from "~/operations/files/body";
 import { transformFromIndex, transformToIndex } from "~/operations/files/transformers";
@@ -61,17 +61,6 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
     private readonly esTable: Table;
     private readonly entity: Entity<any>;
     private readonly esEntity: Entity<any>;
-    private _esIndex?: string;
-
-    private get esIndex(): string {
-        if (!this._esIndex) {
-            const { index: esIndex } = configurations.es({
-                tenant: this.context.tenancy.getCurrentTenant().id
-            });
-            this._esIndex = esIndex;
-        }
-        return this._esIndex;
-    }
 
     private get esClient() {
         const ctx = this.context;
@@ -148,7 +137,7 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
         const esCompressedData = await compress(this.context.plugins, esData);
         const esItem: EsFileItem = {
             ...keys,
-            index: this.esIndex,
+            index: this.getElasticsearchIndex(),
             data: esCompressedData
         };
         try {
@@ -188,7 +177,7 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
         const esCompressedData = await compress(this.context.plugins, esData);
         const esItem: EsFileItem = {
             ...keys,
-            index: this.esIndex,
+            index: this.getElasticsearchIndex(),
             data: esCompressedData
         };
         try {
@@ -256,7 +245,7 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
             esItems.push(
                 this.esEntity.putBatch({
                     ...keys,
-                    index: this.esIndex,
+                    index: this.getElasticsearchIndex(),
                     data: esCompressedData
                 })
             );
@@ -313,9 +302,7 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
         let response: ElasticsearchSearchResponse<File>;
         try {
             response = await this.esClient.search({
-                ...configurations.es({
-                    tenant: this.context.tenancy.getCurrentTenant().id
-                }),
+                index: this.getElasticsearchIndex(),
                 body
             });
         } catch (ex) {
@@ -359,10 +346,6 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
     ): Promise<FileManagerFilesStorageOperationsTagsResponse> {
         const { where, limit: initialLimit } = params;
 
-        const esDefaults = configurations.es({
-            tenant: this.context.tenancy.getCurrentTenant().id
-        });
-
         const must: any[] = [];
         if (where.locale) {
             must.push({ term: { "locale.keyword": where.locale } });
@@ -396,7 +379,7 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
 
         try {
             response = await this.esClient.search({
-                ...esDefaults,
+                index: this.getElasticsearchIndex(),
                 body
             });
         } catch (ex) {
@@ -441,4 +424,16 @@ export class FilesStorageOperations implements FileManagerFilesStorageOperations
     private getFileIndexTransformPlugins(): FileIndexTransformPlugin[] {
         return this.context.plugins.byType<FileIndexTransformPlugin>(FileIndexTransformPlugin.type);
     }
+
+    private getElasticsearchIndex(): string {
+        const locale = this.context.i18nContent.getCurrentLocale();
+        if (!locale) {
+            throw new WebinyError("Missing locale in FilesStorageOperations.", "LOCALE_ERROR");
+        }
+        const { index } = configurations.es({
+            tenant: this.context.tenancy.getCurrentTenant().id,
+            locale: locale.code
+        });
+        return index;
+    }
 }
diff --git a/packages/api-file-manager-ddb-es/src/upgrades/v5.0.0/index.ts b/packages/api-file-manager-ddb-es/src/upgrades/v5.0.0/index.ts
@@ -4,7 +4,7 @@
 // @ts-nocheck
 import { UpgradePlugin } from "@webiny/api-upgrade/types";
 import { paginateBatch } from "../utils";
-import { configurations } from "~/operations/configurations";
+import { configurations } from "~/configurations";
 import { FileManagerContext } from "@webiny/api-file-manager/types";
 import { ElasticsearchContext } from "@webiny/api-elasticsearch/types";
 import { DbContext } from "@webiny/handler-db/types";

diff --git a/packages/api-file-manager-ddb/src/definitions/table.ts b/packages/api-file-manager-ddb/src/definitions/table.ts
@@ -1,5 +1,4 @@
 import { Table } from "dynamodb-toolbox";
-import configurations from "~/operations/configurations";
 import { getDocumentClient, getTable } from "~/operations/utils";
 import { FileManagerContext } from "@webiny/api-file-manager/types";
 
@@ -9,7 +8,7 @@ export interface TableParams {
 export default (params: TableParams): Table => {
     const { context } = params;
     return new Table({
-        name: configurations.db().table || getTable(context),
+        name: process.env.DB_TABLE_FILE_MANGER || process.env.DB_TABLE || getTable(context),
         partitionKey: "PK",
         sortKey: "SK",
         DocumentClient: getDocumentClient(context)

diff --git a/packages/api-file-manager-ddb/src/operations/configurations.ts b/packages/api-file-manager-ddb/src/operations/configurations.ts
diff --git a/packages/api-form-builder-so-ddb-es/__tests__/elasticsearchIndex.test.ts b/packages/api-form-builder-so-ddb-es/__tests__/elasticsearchIndex.test.ts
@@ -0,0 +1,92 @@
+import { configurations } from "~/configurations";
+
+describe("Elasticsearch index", () => {
+    const withLocaleItems = [
+        ["root", "en-US"],
+        ["admin", "en-EN"],
+        ["root", "de-DE"],
+        ["admin", "en-GB"],
+        ["root,", "de"]
+    ];
+
+    beforeEach(() => {
+        process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = undefined;
+    });
+
+    it.each(withLocaleItems)(
+        "should create index with locale code as part of the name",
+        async (tenant, locale) => {
+            process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
+
+            const { index } = configurations.es({
+                tenant,
+                locale
+            });
+
+            expect(index).toEqual(`${tenant}-${locale}-form-builder`.toLowerCase());
+        }
+    );
+
+    it.each(withLocaleItems)(
+        "should create index without locale code as part of the name",
+        async (tenant, locale) => {
+            const { index } = configurations.es({
+                tenant,
+                locale
+            });
+
+            expect(index).toEqual(`${tenant}-form-builder`.toLowerCase());
+        }
+    );
+
+    it("should throw error when missing tenant but it is required", async () => {
+        expect(() => {
+            configurations.es({
+                tenant: null as any,
+                locale: "en-US"
+            });
+        }).toThrowError(
+            `Missing "tenant" parameter when trying to create Elasticsearch index name.`
+        );
+    });
+
+    it("should throw error when missing locale but it is required", async () => {
+        process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
+
+        expect(() => {
+            configurations.es({
+                tenant: "root",
+                locale: null as any
+            });
+        }).toThrowError(
+            `Missing "locale" parameter when trying to create Elasticsearch index name.`
+        );
+    });
+
+    it.each(withLocaleItems)(
+        "should be root tenant in the index, no matter which one is sent",
+        async (tenant, locale) => {
+            process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
+
+            const { index: noLocaleIndex } = configurations.es({
+                tenant,
+                locale
+            });
+            expect(noLocaleIndex).toEqual("root-form-builder");
+        }
+    );
+
+    it.each(withLocaleItems)(
+        "should be root tenant in the index, no matter which one is sent",
+        async (tenant, locale) => {
+            process.env.ELASTICSEARCH_SHARED_INDEXES = "true";
+            process.env.WEBINY_ELASTICSEARCH_INDEX_LOCALE = "true";
+
+            const { index: noLocaleIndex } = configurations.es({
+                tenant,
+                locale
+            });
+            expect(noLocaleIndex).toEqual(`root-${locale}-form-builder`.toLowerCase());
+        }
+    );
+});