Don't zip the dependencies before deployment

appengine.ext.vendor doesn't support zip files and expects dirs. Appengine SDK limits imports of packages under "google" namespace unless they are provided via appengine.ext.vendor.
HLD · Jul 11, 2017 · 0002987 · 0002987
1 parent d8038a4
commit 0002987
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 40 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,7 +6,7 @@ cookbooks
 vagrant_ansible_inventory_*
 Cheffile.lock
 tmp
-src/packages.zip
+src/packages
 src/app.yaml
 *.pyc
 src/instance

diff --git a/Makefile b/Makefile
@@ -20,9 +20,8 @@ APPENGINE_ZIP_PATH=$(DEV_PREFIX)/opt/$(APPENGINE_ZIP_NAME)
 APPENGINE_SDK_PATH=$(DEV_PREFIX)/opt/google_appengine
 APPENGINE_NOAUTH_PATCH_PATH=$(PREFIX)/extras/google_appengine__force_noauth_local_webserver.diff
 
-APPENGINE_PACKAGES_ZIP=$(PREFIX)/src/packages.zip
-APPENGINE_PACKAGES_TEMP_ZIP=$(DEV_PREFIX)/opt/packages.zip
-APPENGINE_PACKAGES_DIR=$(DEV_PREFIX)/opt/gae_packages
+APPENGINE_PACKAGES_DIR=$(PREFIX)/src/packages
+APPENGINE_PACKAGES_TEMP_DIR=$(DEV_PREFIX)/opt/gae_packages
 
 APPENGINE_ENV_DIR=$(DEV_PREFIX)/opt/gae_virtualenv
 APPENGINE_REQUIREMENTS_TXT=$(PREFIX)/src/requirements.txt
@@ -57,9 +56,8 @@ $(APPENGINE_ZIP_PATH) :
 	mv "$(APPENGINE_ZIP_PATH).tmp" "$(APPENGINE_ZIP_PATH)"
 
 clean_appengine_packages :
-	rm -f -- "$(APPENGINE_PACKAGES_ZIP)"
-	rm -f -- "$(APPENGINE_PACKAGES_TEMP_ZIP)"
 	rm -rf -- "$(APPENGINE_PACKAGES_DIR)"
+	rm -rf -- "$(APPENGINE_PACKAGES_TEMP_DIR)"
 	rm -rf -- "$(APPENGINE_ENV_DIR)"
 
 $(APPENGINE_ENV_DIR) :
@@ -70,28 +68,23 @@ $(APPENGINE_ENV_DIR) :
 
 appengine_virtualenv : $(APPENGINE_ENV_DIR)
 
-$(APPENGINE_PACKAGES_DIR) : $(APPENGINE_ENV_DIR)
-	mkdir -p $(APPENGINE_PACKAGES_DIR)
+$(APPENGINE_PACKAGES_TEMP_DIR) : $(APPENGINE_ENV_DIR)
+	mkdir -p $(APPENGINE_PACKAGES_TEMP_DIR)
 	source "$(APPENGINE_ENV_DIR)/bin/activate"; \
-		pip install --no-deps -r "$(APPENGINE_REQUIREMENTS_TXT)" --target "$(APPENGINE_PACKAGES_DIR)"
-	cd "$(APPENGINE_PACKAGES_DIR)/webassets"; \
+		pip install --no-deps -r "$(APPENGINE_REQUIREMENTS_TXT)" --target "$(APPENGINE_PACKAGES_TEMP_DIR)"
+	cd "$(APPENGINE_PACKAGES_TEMP_DIR)/webassets"; \
 		patch -p3 < "${PREFIX}/extras/webassets__fix_builtin_filter_loading.diff"
 
-appengine_packages : $(APPENGINE_PACKAGES_DIR)
-
-$(APPENGINE_PACKAGES_TEMP_ZIP) : $(APPENGINE_PACKAGES_DIR)
-	cd "$(APPENGINE_PACKAGES_DIR)"; \
+$(APPENGINE_PACKAGES_DIR) : $(APPENGINE_PACKAGES_TEMP_DIR)
+	cd "$(APPENGINE_PACKAGES_TEMP_DIR)"; \
 		find . -name "*.pyc" -delete; \
-		find . -name "*.egg-info" | xargs rm -rf; \
-		zip -9rv "$(APPENGINE_PACKAGES_TEMP_ZIP)" .; \
-		touch "$(APPENGINE_PACKAGES_TEMP_ZIP)"
+		find . -name "*.egg-info" | xargs rm -rf
+	cp -r "$(APPENGINE_PACKAGES_TEMP_DIR)" "$(APPENGINE_PACKAGES_DIR)"
 
-$(APPENGINE_PACKAGES_ZIP) : $(APPENGINE_PACKAGES_TEMP_ZIP)
-	cp "$(APPENGINE_PACKAGES_TEMP_ZIP)" "$(APPENGINE_PACKAGES_ZIP)"
+appengine_packages : $(APPENGINE_PACKAGES_DIR)
 
-appengine_packages_zip : $(APPENGINE_PACKAGES_ZIP)
+appengine : appengine_sdk appengine_packages
 
-appengine : appengine_sdk appengine_packages appengine_packages_zip
 
 clean_appengine : clean_appengine_sdk clean_appengine_packages
 
@@ -172,7 +165,7 @@ bower_components : bower.json
 clean_bower_components :
 	rm -rf $(BOWER_PATH) $(FLASH_PATH) $(STATIC_PATH)/fonts
 
-deploy : appengine_packages_zip bower_components src/ggrc/assets/assets.manifest src/app.yaml
+deploy : appengine_packages bower_components src/ggrc/assets/assets.manifest src/app.yaml
 
 clean_deploy :
 	rm -f src/ggrc/assets/stylesheets/dashboard.css

diff --git a/bin/init_env b/bin/init_env
@@ -24,6 +24,6 @@ export GGRC_BOOTSTRAP_ADMIN_USERS="user@example.com"
 
 source "${DEV_PREFIX}/opt/dev_virtualenv/bin/activate"
 
-export PYTHONPATH="${PREFIX}/src:${PREFIX}/test/selenium/src:${DEV_PREFIX}/opt/google_appengine:${DEV_PREFIX}/opt/linked_packages:${PYTHONPATH}"
+export PYTHONPATH="${PREFIX}/src:${PREFIX}/src/packages:${PREFIX}/test/selenium/src:${DEV_PREFIX}/opt/google_appengine:${DEV_PREFIX}/opt/linked_packages:${PYTHONPATH}"
 
 export PATH="${PREFIX}/bin:${DEV_PREFIX}/opt/google_appengine:$PATH:${DEV_PREFIX}/node_modules/.bin/"
diff --git a/src/appengine_config.py b/src/appengine_config.py
@@ -0,0 +1,13 @@
+# Copyright (C) 2017 Google Inc.
+# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
+
+"""`appengine_config.py` is automatically loaded when Google App Engine starts
+a new instance of your application. This runs before any WSGI applications
+specified in app.yaml are loaded.
+"""
+
+from google.appengine.ext import vendor
+
+# Third-party libraries are stored in "packages", vendoring will make
+# sure that they are importable by the application.
+vendor.add('packages')
diff --git a/src/ggrc/bootstrap.py b/src/ggrc/bootstrap.py
@@ -1,24 +1,11 @@
 # Copyright (C) 2017 Google Inc.
 # Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>
 
-import os
-import sys
-
-
-from ggrc import settings
-
-
-def set_appengine_packages_path():
-  if getattr(settings, 'APP_ENGINE', False):
-    if os.path.exists(os.path.join(settings.BASE_DIR, 'packages')):
-      sys.path.insert(0, os.path.join(settings.BASE_DIR, 'packages'))
-    else:
-      sys.path.insert(0, os.path.join(settings.BASE_DIR, 'packages.zip'))
+from flask.ext.sqlalchemy import SQLAlchemy
 
 
 def get_db():
   """Get modified db object."""
-  from flask.ext.sqlalchemy import SQLAlchemy
   db = SQLAlchemy()
 
   class String(db.String):
@@ -33,6 +20,3 @@ def __init__(self, length=None, *args, **kwargs):
       return super(String, self).__init__(length, *args, **kwargs)
   db.String = String
   return db
-
-
-set_appengine_packages_path()