feat: private files and asset delivery (#3799)

webiny · Jan 9, 2024 · 73eebb5 · 73eebb5
1 parent 34aab96
commit 73eebb5
Show file tree

Hide file tree

Showing 194 changed files with 3,879 additions and 1,644 deletions.
diff --git a/apps/api/fileManager/download/package.json b/apps/api/fileManager/download/package.json
diff --git a/apps/api/fileManager/download/src/index.ts b/apps/api/fileManager/download/src/index.ts
diff --git a/apps/api/fileManager/download/tsconfig.json b/apps/api/fileManager/download/tsconfig.json
diff --git a/apps/api/fileManager/download/webiny.config.ts b/apps/api/fileManager/download/webiny.config.ts
diff --git a/apps/api/fileManager/transform/package.json b/apps/api/fileManager/transform/package.json
diff --git a/apps/api/fileManager/transform/src/index.ts b/apps/api/fileManager/transform/src/index.ts
diff --git a/apps/api/fileManager/transform/tsconfig.json b/apps/api/fileManager/transform/tsconfig.json
diff --git a/apps/api/fileManager/transform/webiny.config.ts b/apps/api/fileManager/transform/webiny.config.ts
diff --git a/apps/api/graphql/src/index.ts b/apps/api/graphql/src/index.ts
@@ -23,7 +23,7 @@ import {
 } from "@webiny/api-file-manager";
 import { createFileManagerStorageOperations } from "@webiny/api-file-manager-ddb";
 import logsPlugins from "@webiny/handler-logs";
-import fileManagerS3 from "@webiny/api-file-manager-s3";
+import fileManagerS3, { createAssetDelivery } from "@webiny/api-file-manager-s3";
 import { createFormBuilder } from "@webiny/api-form-builder";
 import { createFormBuilderStorageOperations } from "@webiny/api-form-builder-so-ddb";
 import { createHeadlessCmsContext, createHeadlessCmsGraphQL } from "@webiny/api-headless-cms";
@@ -36,8 +36,6 @@ import { createAco } from "@webiny/api-aco";
 import { createAcoPageBuilderContext } from "@webiny/api-page-builder-aco";
 import { createAuditLogs } from "@webiny/api-audit-logs";
 import { createBackgroundTasks } from "@webiny/api-background-tasks-ddb";
-
-// Imports plugins created via scaffolding utilities.
 import scaffoldsPlugins from "./plugins/scaffolds";
 import { createBenchmarkEnablePlugin } from "~/plugins/benchmarkEnable";
 import { createCountDynamoDbTask } from "~/plugins/countDynamoDbTask";
@@ -75,6 +73,7 @@ export const handler = createHandler({
             })
         }),
         createFileManagerGraphQL(),
+        createAssetDelivery({ documentClient }),
         fileManagerS3(),
         prerenderingServicePlugins({
             eventBus: String(process.env.EVENT_BUS)

diff --git a/package.json b/package.json
@@ -98,6 +98,7 @@
     "rimraf": "^3.0.2",
     "rxjs": "^6.5.5",
     "semver": "^7.5.4",
+    "ts-expect": "^1.3.0",
     "ts-jest": "^29.1.0",
     "typescript": "4.7.4",
     "typescript-transform-paths": "^2.2.3",

diff --git a/packages/api-file-manager-s3/package.json b/packages/api-file-manager-s3/package.json
@@ -15,16 +15,20 @@
     "@webiny/api-security": "0.0.0",
     "@webiny/aws-sdk": "0.0.0",
     "@webiny/error": "0.0.0",
+    "@webiny/handler": "0.0.0",
     "@webiny/handler-graphql": "0.0.0",
     "@webiny/plugins": "0.0.0",
+    "@webiny/tasks": "0.0.0",
     "@webiny/utils": "0.0.0",
     "@webiny/validation": "0.0.0",
     "form-data": "^4.0.0",
     "mime": "^3.0.0",
     "node-fetch": "^2.6.1",
+    "object-hash": "^3.0.0",
     "p-map": "4.0.0",
     "p-reduce": "2.1.0",
-    "sanitize-filename": "^1.6.3"
+    "sanitize-filename": "^1.6.3",
+    "sharp": "0.32.6"
   },
   "devDependencies": {
     "@babel/cli": "^7.22.6",

diff --git a/packages/api-file-manager-s3/src/assetDelivery/assetDeliveryConfig.ts b/packages/api-file-manager-s3/src/assetDelivery/assetDeliveryConfig.ts
@@ -0,0 +1,46 @@
+import {
+    createAssetDelivery as createBaseAssetDelivery,
+    createAssetDeliveryConfig
+} from "@webiny/api-file-manager";
+import { S3 } from "@webiny/aws-sdk/client-s3";
+import { S3AssetResolver } from "~/assetDelivery/s3/S3AssetResolver";
+import { S3OutputStrategy } from "~/assetDelivery/s3/S3OutputStrategy";
+import { SharpTransform } from "~/assetDelivery/s3/SharpTransform";
+
+export type AssetDeliveryParams = Parameters<typeof createBaseAssetDelivery>[0] & {
+    imageResizeWidths?: number[];
+    presignedUrlTtl?: number;
+};
+
+export const assetDeliveryConfig = (params: AssetDeliveryParams) => {
+    const bucket = process.env.S3_BUCKET as string;
+    const region = process.env.AWS_REGION as string;
+
+    const {
+        presignedUrlTtl = 900,
+        imageResizeWidths = [100, 300, 500, 750, 1000, 1500, 2500],
+        ...baseParams
+    } = params;
+
+    return [
+        // Base asset delivery
+        createBaseAssetDelivery(baseParams),
+        // S3 plugins
+        createAssetDeliveryConfig(config => {
+            const s3 = new S3({ region });
+
+            config.decorateAssetResolver(() => {
+                // This resolver loads file information from the `.metadata` file.
+                return new S3AssetResolver(s3, bucket);
+            });
+
+            config.decorateAssetOutputStrategy(() => {
+                return new S3OutputStrategy(s3, bucket, presignedUrlTtl);
+            });
+
+            config.decorateAssetTransformationStrategy(() => {
+                return new SharpTransform({ s3, bucket, imageResizeWidths });
+            });
+        })
+    ];
+};
diff --git a/packages/api-file-manager-s3/src/assetDelivery/createAssetDelivery.ts b/packages/api-file-manager-s3/src/assetDelivery/createAssetDelivery.ts
@@ -0,0 +1,14 @@
+import { createAssetDeliveryPluginLoader } from "@webiny/api-file-manager";
+import { PluginFactory } from "@webiny/plugins/types";
+import type { AssetDeliveryParams } from "./assetDeliveryConfig";
+
+export const createAssetDelivery = (params: AssetDeliveryParams): PluginFactory => {
+    /**
+     * We only want to load this plugin in the context of the Asset Delivery Lambda function.
+     */
+    return createAssetDeliveryPluginLoader(() => {
+        return import(/* webpackChunkName: "s3AssetDelivery" */ "./assetDeliveryConfig").then(
+            ({ assetDeliveryConfig }) => assetDeliveryConfig(params)
+        );
+    });
+};
diff --git a/packages/api-file-manager-s3/src/assetDelivery/s3/S3AssetMetadataReader.ts b/packages/api-file-manager-s3/src/assetDelivery/s3/S3AssetMetadataReader.ts
@@ -0,0 +1,44 @@
+import type { S3 } from "@webiny/aws-sdk/client-s3";
+
+interface AssetMetadata {
+    id: string;
+    tenant: string;
+    locale: string;
+    size: number;
+    contentType: string;
+}
+
+export class S3AssetMetadataReader {
+    private readonly s3: S3;
+    private readonly bucket: string;
+
+    constructor(s3: S3, bucket: string) {
+        this.bucket = bucket;
+        this.s3 = s3;
+    }
+
+    async getMetadata(key: string): Promise<AssetMetadata> {
+        const metadataKey = `${key}.metadata`;
+
+        console.log("Reading metadata", metadataKey);
+
+        const { Body } = await this.s3.getObject({
+            Bucket: this.bucket,
+            Key: metadataKey
+        });
+
+        if (!Body) {
+            throw Error(`Missing or corrupted ${metadataKey} file!`);
+        }
+
+        const metadata = JSON.parse(await Body.transformToString());
+
+        return {
+            id: metadata.id,
+            tenant: metadata.tenant,
+            locale: metadata.locale,
+            size: metadata.size,
+            contentType: metadata.contentType
+        };
+    }
+}
diff --git a/packages/api-file-manager-s3/src/assetDelivery/s3/S3AssetResolver.ts b/packages/api-file-manager-s3/src/assetDelivery/s3/S3AssetResolver.ts
@@ -0,0 +1,37 @@
+import { S3 } from "@webiny/aws-sdk/client-s3";
+import { Asset, AssetRequest, AssetResolver } from "@webiny/api-file-manager";
+import { S3AssetMetadataReader } from "./S3AssetMetadataReader";
+import { S3ContentsReader } from "./S3ContentsReader";
+
+export class S3AssetResolver implements AssetResolver {
+    private readonly s3: S3;
+    private readonly bucket: string;
+
+    constructor(s3: S3, bucket: string) {
+        this.s3 = s3;
+        this.bucket = bucket;
+    }
+
+    async resolve(request: AssetRequest): Promise<Asset | undefined> {
+        try {
+            const metadataReader = new S3AssetMetadataReader(this.s3, this.bucket);
+            const metadata = await metadataReader.getMetadata(request.getKey());
+
+            const asset = new Asset({
+                id: metadata.id,
+                tenant: metadata.tenant,
+                locale: metadata.locale,
+                size: metadata.size,
+                contentType: metadata.contentType,
+                key: request.getKey()
+            });
+
+            asset.setContentsReader(new S3ContentsReader(this.s3, this.bucket));
+
+            return asset;
+        } catch (error) {
+            console.error(error);
+            return undefined;
+        }
+    }
+}
diff --git a/packages/api-file-manager-s3/src/assetDelivery/s3/S3ContentsReader.ts b/packages/api-file-manager-s3/src/assetDelivery/s3/S3ContentsReader.ts
@@ -0,0 +1,25 @@
+import { S3 } from "@webiny/aws-sdk/client-s3";
+import { Asset, AssetContentsReader } from "@webiny/api-file-manager";
+
+export class S3ContentsReader implements AssetContentsReader {
+    private s3: S3;
+    private readonly bucket: string;
+
+    constructor(s3: S3, bucket: string) {
+        this.s3 = s3;
+        this.bucket = bucket;
+    }
+
+    async read(asset: Asset): Promise<Buffer> {
+        const { Body } = await this.s3.getObject({
+            Bucket: this.bucket,
+            Key: asset.getKey()
+        });
+
+        if (!Body) {
+            throw Error(`Unable to read ${asset.getKey()}!`);
+        }
+
+        return Buffer.from(await Body.transformToByteArray());
+    }
+}
diff --git a/packages/api-file-manager-s3/src/assetDelivery/s3/S3ErrorAssetReply.ts b/packages/api-file-manager-s3/src/assetDelivery/s3/S3ErrorAssetReply.ts
@@ -0,0 +1,10 @@
+import { AssetReply } from "@webiny/api-file-manager";
+
+export class S3ErrorAssetReply extends AssetReply {
+    constructor(message: string) {
+        super({
+            code: 400,
+            body: () => ({ error: message })
+        });
+    }
+}