Add deployment of export service to deploy.py (oppia#5773)

* Add deployment of export service in deployment script. * Change export folder name; Do not initiate real backup requests on nonprod apps. * Add error message. * Lint. * Lint. * Address review changes. * Use a local version of gcloud. * Change acl_decorator import. * Lint.
quantumferret · Oct 28, 2018 · 01170c4 · 01170c4
1 parent b4da477
commit 01170c4
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 3 deletions.
diff --git a/cron.yaml b/cron.yaml
@@ -21,6 +21,6 @@ cron:
   url: /cron/suggestions/notify_reviewers
   schedule: every day 9:00
 - description: "Weekly Cloud Datastore Export"
-  url: /cloud_datastore_export?namespace_id=&output_url_prefix=gs://oppia-test-export
+  url: /cloud_datastore_export?namespace_id=&output_url_prefix=gs://oppia-export-backups
   target: cloud-datastore-admin
   schedule: every thursday 9:00
diff --git a/export/cloud_datastore_admin.py b/export/cloud_datastore_admin.py
@@ -25,12 +25,14 @@
 import json
 import logging
 
-from oppia.export import acl_decorators
+import acl_decorators  # pylint: disable=relative-import
 
 from google.appengine.api import app_identity
 from google.appengine.api import urlfetch
 import webapp2
 
+APP_NAME_OPPIASERVER = 'oppiaserver'
+
 
 class ExportToCloudDatastoreHandler(webapp2.RequestHandler):
     """Request handler which supports triggering automatic exports of the
@@ -51,6 +53,13 @@ def get(self):
         access_token, _ = app_identity.get_access_token(
             'https://www.googleapis.com/auth/datastore')
         app_id = app_identity.get_application_id()
+
+        if app_id != APP_NAME_OPPIASERVER:
+            logging.error('Export service has been pinged. '
+                          'Since this is not production, a real export request '
+                          'has not been initiated.')
+            return
+
         timestamp = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S')
 
         output_url_prefix = self.request.get('output_url_prefix')

diff --git a/scripts/common.py b/scripts/common.py
@@ -17,6 +17,10 @@
 import os
 import subprocess
 
+GCLOUD_PATH = os.path.join(
+    '..', 'oppia_tools', 'google-cloud-sdk-222.0.0', 'google-cloud-sdk',
+    'bin', 'gcloud')
+
 
 def ensure_directory_exists(d):
     """Creates the given directory if it does not already exist."""
@@ -132,6 +136,15 @@ def ensure_release_scripts_folder_exists_and_is_up_to_date():
         subprocess.call(['git', 'pull', remote_alias])
 
 
+def require_gcloud_to_be_available():
+    try:
+        subprocess.check_output([GCLOUD_PATH, '--version'])
+    except Exception:
+        raise Exception(
+            'gcloud required, but could not be found. Please run '
+            'scripts/start.sh to install gcloud.')
+
+
 class CD(object):
     """Context manager for changing the current working directory."""
 

diff --git a/scripts/deploy.py b/scripts/deploy.py
@@ -201,6 +201,7 @@ def _execute_deployment():
     # Do prerequisite checks.
     common.require_cwd_to_be_oppia()
     common.ensure_release_scripts_folder_exists_and_is_up_to_date()
+    common.require_gcloud_to_be_available()
 
     current_git_revision = subprocess.check_output(
         ['git', 'rev-parse', 'HEAD']).strip()
@@ -247,7 +248,12 @@ def _execute_deployment():
         if build_process.returncode > 0:
             raise Exception('Build failed.')
 
-        # Deploy to GAE.
+        # Deploy export service to GAE.
+        subprocess.check_output([
+            common.GCLOUD_PATH, 'app', 'deploy', 'export/app.yaml',
+            '--project=%s' % APP_NAME])
+
+        # Deploy app to GAE.
         subprocess.check_output([APPCFG_PATH, 'update', '.'])
 
         # Writing log entry.

diff --git a/scripts/setup_gae.sh b/scripts/setup_gae.sh
@@ -23,6 +23,7 @@ if [ "$SETUP_GAE_DONE" ]; then
 fi
 
 export GOOGLE_APP_ENGINE_HOME=$TOOLS_DIR/google_appengine_1.9.67/google_appengine
+export GOOGLE_CLOUD_SDK_HOME=$TOOLS_DIR/google-cloud-sdk-222.0.0/google-cloud-sdk
 export COVERAGE_HOME=$TOOLS_DIR/coverage-4.5.1
 
 # Note that if the following line is changed so that it uses webob_1_1_1, PUT requests from the frontend fail.
@@ -48,4 +49,21 @@ if [ ! -d "$GOOGLE_APP_ENGINE_HOME" ]; then
   rm gae-download.zip
 fi
 
+echo Checking whether google-cloud-sdk is installed in $GOOGLE_CLOUD_SDK_HOME
+if [ ! -d "$GOOGLE_CLOUD_SDK_HOME" ]; then
+  echo "Downloading Google Cloud SDK (this may take a little while)..."
+  mkdir -p $GOOGLE_CLOUD_SDK_HOME
+  curl -o gcloud-sdk.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-222.0.0-linux-x86_64.tar.gz
+  # $? contains the (exit) status code of previous command.
+  # If curl was successful, $? will be 0 else non-zero.
+  if [ 0 -eq $? ]; then
+    echo "Download complete. Installing Google Cloud SDK..."
+  else
+    echo "Error downloading Google Cloud SDK. Exiting."
+    exit 1
+  fi
+  tar xzf gcloud-sdk.tar.gz -C $TOOLS_DIR/google-cloud-sdk-222.0.0/
+  rm gcloud-sdk.tar.gz
+fi
+
 export SETUP_GAE_DONE=true