feat: introduce default robots.txt files (#4329)

webiny · Oct 10, 2024 · 9ce5e75 · 9ce5e75
1 parent 04061f0
commit 9ce5e75
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 0 deletions.
diff --git a/apps/admin/public/robots.txt b/apps/admin/public/robots.txt
@@ -0,0 +1,3 @@
+# No robots allowed.
+User-agent: *
+Disallow: /
diff --git a/apps/website/public/robots.txt b/apps/website/public/robots.txt
@@ -0,0 +1 @@
+# All robots allowed.
diff --git a/packages/cwp-template-aws/template/common/apps/admin/public/robots.txt b/packages/cwp-template-aws/template/common/apps/admin/public/robots.txt
@@ -0,0 +1,3 @@
+# No robots allowed.
+User-agent: *
+Disallow: /
diff --git a/packages/cwp-template-aws/template/common/apps/website/public/robots.txt b/packages/cwp-template-aws/template/common/apps/website/public/robots.txt
@@ -0,0 +1 @@
+# All robots allowed.
diff --git a/packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts b/packages/pulumi-aws/src/apps/website/createWebsitePulumiApp.ts
@@ -186,6 +186,23 @@ export const createWebsitePulumiApp = (projectAppParams: CreateWebsitePulumiAppP
                             minTtl: 0,
                             defaultTtl: 2592000, // 30 days
                             maxTtl: 2592000
+                        },
+                        // This forward is necessary for non-WCP projects. For WCP projects, the
+                        // forwarding is performed by the `website-router` Lambda@Edge function.
+                        {
+                            compress: true,
+                            allowedMethods: ["GET", "HEAD", "OPTIONS"],
+                            cachedMethods: ["GET", "HEAD", "OPTIONS"],
+                            forwardedValues: {
+                                cookies: {
+                                    forward: "none"
+                                },
+                                headers: [],
+                                queryString: false
+                            },
+                            pathPattern: "/robots.txt",
+                            viewerProtocolPolicy: "allow-all",
+                            targetOriginId: appBucket.origin.originId
                         }
                     ],
                     customErrorResponses: [

diff --git a/packages/serverless-cms-aws/src/react/plugins/uploadAppToS3.ts b/packages/serverless-cms-aws/src/react/plugins/uploadAppToS3.ts
@@ -57,6 +57,10 @@ export const uploadAppToS3 = ({ folder, ...config }: UploadAppToS3Config) => ({
                     pattern: /index\.html/,
                     value: "no-cache, no-store, must-revalidate"
                 },
+                {
+                    pattern: /robots\.txt/,
+                    value: "no-cache, no-store, must-revalidate"
+                },
                 {
                     pattern: /.*/,
                     value: "max-age=31536000"