target s3 bucket with docs refresh script (digital-asset#1287)

There is no simple way to configure GCS to serve the desired security headers, so instead the script will keep updating the existing s3 bucket. Consequent changes: - Add aws cli tool to dev-env - Remove docs bucket from Terraform
FlashSheridan · May 21, 2019 · c762d49 · c762d49
1 parent be2457c
commit c762d49
Show file tree

Hide file tree

Showing 9 changed files with 20 additions and 105 deletions.
diff --git a/azure-cron.yml b/azure-cron.yml
@@ -22,6 +22,8 @@ jobs:
       - bash: |
           set -euo pipefail
 
+          CUR_SHA=$(git rev-parse HEAD)
+
           robustly_download_nix_pkgs() {
             # In recent commits, this is part of the dev-env-install script.
             # However, we have to copy it here to apply it to older versions.
@@ -66,16 +68,23 @@ jobs:
             mkdir -p $DOCDIR/${version#v}
             tar xzf bazel-genfiles/docs/html.tar.gz --strip-components=1 -C $DOCDIR/${version#v} >$LOG 2>&1
           done
-          echo "Maintain proper 404 page"
-          curl -s https://docs.daml.com/not-found.html > $DOCDIR/not-found.html
-          echo "Pushing to GCS bucket..."
-          GCS_KEY=$(mktemp)
-          echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
-          gcloud auth activate-service-account --key-file=$GCS_KEY >$LOG 2>&1
-          BOTO_CONFIG=/dev/null gsutil rsync -d -r $DOCDIR gs://daml-docs >$LOG 2>&1
+
+          git checkout $CUR_SHA
+          echo "Pushing to S3 bucket..."
+          aws s3 sync $DOCDIR \
+                      s3://docs-daml-com/ \
+                      --delete \
+                      --acl public-read \
+                      --exclude '*.doctrees/*' \
+                      --exclude '*.buildinfo'
+          echo "Refreshing CloudFront cache..."
+          aws cloudfront create-invalidation \
+                         --distribution-id E1U753I56ERH55 \
+                         --paths '/*'
           echo "Done."
         env:
-          GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
+          AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
+          AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
       - task: PublishPipelineArtifact@0
         condition: always()
         inputs:

diff --git a/dev-env/bin/aws b/dev-env/bin/aws
@@ -0,0 +1 @@
+../lib/dade-exec-nix-tool
diff --git a/infra/docs_bucket.tf b/infra/docs_bucket.tf
diff --git a/infra/modules/gcp_cdn_bucket/README.md b/infra/modules/gcp_cdn_bucket/README.md
@@ -19,7 +19,6 @@ It also makes a few assumptions:
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
 | cache\_retention\_days | The number of days to keep the objects around | string | n/a | yes |
-| default\_file | The name of the file (from the files folder) to show for unknown paths. Currently one of cache or docs. | string | `"cache"` | no |
 | labels | Labels to apply on all the resources | map | `<map>` | no |
 | name | Name prefix for all the resources | string | n/a | yes |
 | project | GCP project name | string | n/a | yes |

diff --git a/infra/modules/gcp_cdn_bucket/files/docs.html b/infra/modules/gcp_cdn_bucket/files/docs.html
diff --git a/...a/modules/gcp_cdn_bucket/files/cache.html → ...a/modules/gcp_cdn_bucket/files/index.html b/...a/modules/gcp_cdn_bucket/files/cache.html → ...a/modules/gcp_cdn_bucket/files/index.html
diff --git a/infra/modules/gcp_cdn_bucket/google_storage.tf b/infra/modules/gcp_cdn_bucket/google_storage.tf
@@ -41,7 +41,6 @@ resource "google_storage_bucket" "default" {
 
   website {
     main_page_suffix = "index.html"
-    not_found_page   = "${var.default_file == "docs" ? "not-found.html" : ""}"
   }
 
   force_destroy = true
@@ -54,19 +53,9 @@ resource "google_storage_bucket_acl" "default" {
 }
 
 resource "google_storage_bucket_object" "default" {
-  count        = "${var.default_file == "cache" ? 1 : 0}"
   name         = "index.html"
   bucket       = "${google_storage_bucket.default.name}"
-  content      = "${file("${path.module}/files/${var.default_file}.html")}"
-  content_type = "text/html"
-  depends_on   = ["google_storage_bucket_acl.default"]
-}
-
-resource "google_storage_bucket_object" "not_found" {
-  count        = "${var.default_file == "docs" ? 1 : 0}"
-  name         = "not-found.html"
-  bucket       = "${google_storage_bucket.default.name}"
-  content      = "${file("${path.module}/files/${var.default_file}.html")}"
+  content      = "${file("${path.module}/files/index.html")}"
   content_type = "text/html"
   depends_on   = ["google_storage_bucket_acl.default"]
 }
diff --git a/infra/modules/gcp_cdn_bucket/variables.tf b/infra/modules/gcp_cdn_bucket/variables.tf
@@ -26,8 +26,3 @@ variable "ssl_certificate" {
 variable "cache_retention_days" {
   description = "The number of days to keep the objects around"
 }
-
-variable "default_file" {
-  description = "The name of the file (from the files folder) to show for unknown paths. Currently one of cache or docs."
-  default     = "cache"
-}
diff --git a/nix/packages.nix b/nix/packages.nix
@@ -245,6 +245,7 @@ in rec {
     jfrog = pkgs.callPackage ./tools/jfrog-cli {};
 
     # Cloud tools
+    aws = pkgs.awscli;
     gcloud = pkgs.google-cloud-sdk;
     bq     = gcloud;
     gsutil = gcloud;