Fixes #5310: Added a size check to third party folder to prevent brea…

…ching the 10k … (#5743) * Added a size check to third party folder to prevent braching the 10k files limit on AppEngine * Changed third party size check from .travis.yaml to scripts.run_presubmit_checks.sh and solved documentation mistakes as suggested by kevinlee12 * Deleted run_third_party_size_check.sh and directly call third_party_size_check.py from run_presubmit_checks.py as suggested by kevinlee12 * Deleted unnecessary disable pylint import order on scripts.third_party_size_check.py * Moved third_party_size_check.py to .travis.yaml to execute in the same container as lint checks * Remove raises from _check_third_party_size method docstring as sugested by apb7 in scripts.third_party_size_check.py * Made some scripts.third_party_size_check docstrings more specific as suggested by kevinlee12 * Changed check results to print the third party size check on scripts.third_party_size_check.py as suggested by kevinlee12 * Made some improvements on scripts.third_party_size_check.py as suggested by seanlip * Made some improvements on scripts.third_party_size_check.py as suggested by seanlip
oppia · Oct 24, 2018 · aef8883 · aef8883
1 parent d316ca8
commit aef8883
Show file tree

Hide file tree

Showing 2 changed files with 78 additions and 1 deletion.
diff --git a/.travis.yml b/.travis.yml
@@ -86,7 +86,7 @@ install:
 - source scripts/setup_gae.sh || exit 1
 
 script:
-- if [ "$RUN_LINT" == 'true' ]; then bash scripts/install_third_party.sh; python scripts/pre_commit_linter.py --path=.; fi
+- if [ "$RUN_LINT" == 'true' ]; then bash scripts/install_third_party.sh; python scripts/third_party_size_check.py; python scripts/pre_commit_linter.py --path=.; fi
 # Travis aborts test run if nothing is printed back to STDOUT for some time.
 # -x is used to avoid that.
 - if [ "$RUN_FRONTEND_TESTS" == 'true' ]; then travis_retry bash -x scripts/run_frontend_tests.sh --run-minified-tests=true; fi

diff --git a/scripts/third_party_size_check.py b/scripts/third_party_size_check.py
@@ -0,0 +1,77 @@
+# coding: utf-8
+#
+# Copyright 2018 The Oppia Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script that runs a size check in third-party folder and errors if
+size limit is exceeded. The aim of this is to prevent us accidentally
+breaching the 10k file limit on App Engine.
+"""
+
+import os
+import sys
+
+THIRD_PARTY_PATH = os.path.join(os.getcwd(), 'third_party')
+THIRD_PARTY_SIZE_LIMIT = 7000
+
+
+def _check_size_in_dir(dir_path):
+    """Recursive method that checks the number of files inside the given
+    directory.
+
+    Args:
+         dir_path. str. The directory which files will be counted.
+
+    Returns:
+        The number of files inside the given directory.
+    """
+    number_of_files_in_dir = 0
+    for name in os.listdir(dir_path):
+        if os.path.isfile(os.path.join(dir_path, name)):
+            number_of_files_in_dir += 1
+        else:
+            if os.path.isdir(os.path.join(dir_path, name)):
+                number_of_files_in_dir += _check_size_in_dir(
+                    os.path.join(dir_path, name))
+    return number_of_files_in_dir
+
+
+def _check_third_party_size():
+    """Checks if the third-party size limit has been exceeded."""
+    number_of_files_in_third_party = _check_size_in_dir(THIRD_PARTY_PATH)
+    print ''
+    print '------------------------------------------------------'
+    print '    Number of files in third-party folder: %d' % (
+        number_of_files_in_third_party)
+    print ''
+    if number_of_files_in_third_party > THIRD_PARTY_SIZE_LIMIT:
+        print(
+            '    ERROR: The third-party folder size exceeded the %d files'
+            ' limit.' % THIRD_PARTY_SIZE_LIMIT)
+        print '------------------------------------------------------'
+        print ''
+        sys.exit(1)
+    else:
+        print '    The size of third-party folder is within the limits.'
+        print '------------------------------------------------------'
+        print ''
+        print 'Done!'
+        print ''
+
+
+if __name__ == '__main__':
+    print 'Running third-party size check'
+    _check_third_party_size()
+    print 'Third-party folder size check passed.'
+    print ''