diff --git a/cron.yaml b/cron.yaml index 9b13da6a82db..231a0b8ac47c 100644 --- a/cron.yaml +++ b/cron.yaml @@ -21,6 +21,6 @@ cron: url: /cron/suggestions/notify_reviewers schedule: every day 9:00 - description: "Weekly Cloud Datastore Export" - url: /cloud_datastore_export?namespace_id=&output_url_prefix=gs://oppia-test-export + url: /cloud_datastore_export?namespace_id=&output_url_prefix=gs://oppia-export-backups target: cloud-datastore-admin schedule: every thursday 9:00 diff --git a/export/cloud_datastore_admin.py b/export/cloud_datastore_admin.py index e1cf7eafba9c..441cec2084bd 100644 --- a/export/cloud_datastore_admin.py +++ b/export/cloud_datastore_admin.py @@ -25,12 +25,14 @@ import json import logging -from oppia.export import acl_decorators +import acl_decorators # pylint: disable=relative-import from google.appengine.api import app_identity from google.appengine.api import urlfetch import webapp2 +APP_NAME_OPPIASERVER = 'oppiaserver' + class ExportToCloudDatastoreHandler(webapp2.RequestHandler): """Request handler which supports triggering automatic exports of the @@ -51,6 +53,13 @@ def get(self): access_token, _ = app_identity.get_access_token( 'https://www.googleapis.com/auth/datastore') app_id = app_identity.get_application_id() + + if app_id != APP_NAME_OPPIASERVER: + logging.error('Export service has been pinged. ' + 'Since this is not production, a real export request ' + 'has not been initiated.') + return + timestamp = datetime.datetime.utcnow().strftime('%Y%m%d-%H%M%S') output_url_prefix = self.request.get('output_url_prefix') diff --git a/scripts/common.py b/scripts/common.py index ca1f8c69a5d9..a4012e388b3c 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -17,6 +17,10 @@ import os import subprocess +GCLOUD_PATH = os.path.join( + '..', 'oppia_tools', 'google-cloud-sdk-222.0.0', 'google-cloud-sdk', + 'bin', 'gcloud') + def ensure_directory_exists(d): """Creates the given directory if it does not already exist.""" @@ -132,6 +136,15 @@ def ensure_release_scripts_folder_exists_and_is_up_to_date(): subprocess.call(['git', 'pull', remote_alias]) +def require_gcloud_to_be_available(): + try: + subprocess.check_output([GCLOUD_PATH, '--version']) + except Exception: + raise Exception( + 'gcloud required, but could not be found. Please run ' + 'scripts/start.sh to install gcloud.') + + class CD(object): """Context manager for changing the current working directory.""" diff --git a/scripts/deploy.py b/scripts/deploy.py index 844e5e6a40c3..dec1a7b10cf4 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -201,6 +201,7 @@ def _execute_deployment(): # Do prerequisite checks. common.require_cwd_to_be_oppia() common.ensure_release_scripts_folder_exists_and_is_up_to_date() + common.require_gcloud_to_be_available() current_git_revision = subprocess.check_output( ['git', 'rev-parse', 'HEAD']).strip() @@ -247,7 +248,12 @@ def _execute_deployment(): if build_process.returncode > 0: raise Exception('Build failed.') - # Deploy to GAE. + # Deploy export service to GAE. + subprocess.check_output([ + common.GCLOUD_PATH, 'app', 'deploy', 'export/app.yaml', + '--project=%s' % APP_NAME]) + + # Deploy app to GAE. subprocess.check_output([APPCFG_PATH, 'update', '.']) # Writing log entry. diff --git a/scripts/setup_gae.sh b/scripts/setup_gae.sh index dae9848d3cfd..cb21ebf7fc36 100644 --- a/scripts/setup_gae.sh +++ b/scripts/setup_gae.sh @@ -23,6 +23,7 @@ if [ "$SETUP_GAE_DONE" ]; then fi export GOOGLE_APP_ENGINE_HOME=$TOOLS_DIR/google_appengine_1.9.67/google_appengine +export GOOGLE_CLOUD_SDK_HOME=$TOOLS_DIR/google-cloud-sdk-222.0.0/google-cloud-sdk export COVERAGE_HOME=$TOOLS_DIR/coverage-4.5.1 # Note that if the following line is changed so that it uses webob_1_1_1, PUT requests from the frontend fail. @@ -48,4 +49,21 @@ if [ ! -d "$GOOGLE_APP_ENGINE_HOME" ]; then rm gae-download.zip fi +echo Checking whether google-cloud-sdk is installed in $GOOGLE_CLOUD_SDK_HOME +if [ ! -d "$GOOGLE_CLOUD_SDK_HOME" ]; then + echo "Downloading Google Cloud SDK (this may take a little while)..." + mkdir -p $GOOGLE_CLOUD_SDK_HOME + curl -o gcloud-sdk.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-222.0.0-linux-x86_64.tar.gz + # $? contains the (exit) status code of previous command. + # If curl was successful, $? will be 0 else non-zero. + if [ 0 -eq $? ]; then + echo "Download complete. Installing Google Cloud SDK..." + else + echo "Error downloading Google Cloud SDK. Exiting." + exit 1 + fi + tar xzf gcloud-sdk.tar.gz -C $TOOLS_DIR/google-cloud-sdk-222.0.0/ + rm gcloud-sdk.tar.gz +fi + export SETUP_GAE_DONE=true