chore: Enable basic mypy typechecking

aws · Nov 18, 2020 · 8f46e04 · hoffa · Nov 18, 2020 · 8f46e04
1 parent b820e9d
commit 8f46e04
Show file tree

Hide file tree

Showing 23 changed files with 120 additions and 66 deletions.
diff --git a/Makefile b/Makefile
@@ -35,6 +35,8 @@ smoke-test:
 lint:
 	# Linter performs static analysis to catch latent bugs
 	pylint --rcfile .pylintrc samcli
+	# mypy performs type check
+	mypy setup.py samcli tests
 
 # Command to run everytime you make changes to verify everything works
 dev: lint test

diff --git a/appveyor.yml b/appveyor.yml
@@ -76,6 +76,7 @@ for:
       - "venv\\Scripts\\activate"
       - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
       - "pylint --rcfile .pylintrc samcli"
+      - "mypy setup.py samcli tests"
       - "pytest -n 4 tests/functional"
 
   - 
@@ -129,6 +130,7 @@ for:
     test_script:
       - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
       - "pylint --rcfile .pylintrc samcli"
+      - "mypy setup.py samcli tests"
       - "pytest -n 4 tests/functional"
 
       # Runs only in Linux

diff --git a/mypy.ini b/mypy.ini
@@ -0,0 +1,56 @@
+# https://mypy.readthedocs.io/en/stable/config_file.html#config-file-format
+
+[mypy]
+warn_return_any=True
+warn_unused_configs=True
+no_implicit_optional=True
+warn_redundant_casts=True
+warn_unused_ignores=True
+warn_unreachable=True
+
+#
+# ignore errors in testdata
+#
+
+[mypy-tests.integration.testdata.*]
+ignore_errors=True
+
+#
+# below are packages/modules that do not have stubs available
+#
+
+[mypy-pywintypes]
+ignore_missing_imports=True
+
+[mypy-botocore,botocore.*]
+ignore_missing_imports=True
+
+[mypy-docker,docker.*]
+ignore_missing_imports=True
+
+[mypy-aws_lambda_builders,aws_lambda_builders.*]
+ignore_missing_imports=True
+
+[mypy-cookiecutter,cookiecutter.*]
+ignore_missing_imports=True
+
+[mypy-serverlessrepo,serverlessrepo.*]
+ignore_missing_imports=True
+
+[mypy-tomlkit]
+ignore_missing_imports=True
+
+[mypy-samtranslator,samtranslator.*]
+ignore_missing_imports=True
+
+[mypy-jmespath]
+ignore_missing_imports=True
+
+[mypy-chevron]
+ignore_missing_imports=True
+
+[mypy-parameterized]
+ignore_missing_imports=True
+
+[mypy-setuptools]
+ignore_missing_imports=True
diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -2,6 +2,10 @@ coverage==5.3
 pytest-cov==2.10.1
 pylint~=2.6.0
 
+# type checking and related stubs
+mypy~=0.790
+boto3-stubs[essential]~=1.14.23 # maintain version consistent with boto3
+
 # Test requirements
 pytest==6.1.1
 parameterized==0.7.4

diff --git a/samcli/lib/generated_sample_events/events.py b/samcli/lib/generated_sample_events/events.py
@@ -5,7 +5,7 @@
 import os
 import json
 import base64
-from requests.utils import quote as url_quote
+from urllib.parse import quote as url_quote
 from chevron import renderer
 from samcli.lib.utils.hash import str_checksum
 

diff --git a/samcli/lib/logs/formatter.py b/samcli/lib/logs/formatter.py
@@ -5,14 +5,8 @@
 import json
 import functools
 
-try:
-    # Python2
-    from itertools import imap
-except ImportError:
-    # Python3 already has `map` defined, alias to imap
-    # We do this to prevent accidentally using the built-in ``map`` in Python2. In Python2, ``map`` does a full
-    # evaluation of the iterator whereas imap does a lazy evaluation. For performance reasons, we need to use ``imap``.
-    from builtins import map as imap
+
+from builtins import map
 
 
 class LogsFormatter:
@@ -106,7 +100,7 @@ def do_format(self, event_iterable):
 
             # Make sure the operation has access to certain basic objects like colored
             partial_op = functools.partial(operation, colored=self.colored)
-            event_iterable = imap(partial_op, event_iterable)
+            event_iterable = map(partial_op, event_iterable)
 
         return event_iterable
 

diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py
@@ -24,6 +24,7 @@
 import contextlib
 from contextlib import contextmanager
 import uuid
+from typing import Optional
 from urllib.parse import urlparse, parse_qs
 import shutil
 from botocore.utils import set_value_from_jmespath
@@ -251,8 +252,8 @@ class Resource:
     Base class representing a CloudFormation resource that can be exported
     """
 
-    RESOURCE_TYPE = None
-    PROPERTY_NAME = None
+    RESOURCE_TYPE: Optional[str] = None
+    PROPERTY_NAME: Optional[str] = None
     PACKAGE_NULL_PROPERTY = True
     # Set this property to True in base class if you want the exporter to zip
     # up the file before uploading This is useful for Lambda functions.
@@ -311,9 +312,9 @@ class ResourceWithS3UrlDict(Resource):
     an dict like {Bucket: "", Key: "", Version: ""}
     """
 
-    BUCKET_NAME_PROPERTY = None
-    OBJECT_KEY_PROPERTY = None
-    VERSION_PROPERTY = None
+    BUCKET_NAME_PROPERTY: Optional[str] = None
+    OBJECT_KEY_PROPERTY: Optional[str] = None
+    VERSION_PROPERTY: Optional[str] = None
 
     def do_export(self, resource_id, resource_dict, parent_dir):
         """

diff --git a/samcli/lib/providers/provider.py b/samcli/lib/providers/provider.py
@@ -194,6 +194,10 @@ def name(self):
     def codeuri(self):
         return self._codeuri
 
+    @codeuri.setter
+    def codeuri(self, codeuri):
+        self._codeuri = codeuri
+
     @property
     def version(self):
         return self._version
@@ -203,10 +207,6 @@ def layer_arn(self):
         layer_arn, _ = self.arn.rsplit(":", 1)
         return layer_arn
 
-    @codeuri.setter
-    def codeuri(self, codeuri):
-        self._codeuri = codeuri
-
     @property
     def build_method(self):
         return self._build_method
@@ -251,7 +251,7 @@ def binary_media_types(self):
 
 _CorsTuple = namedtuple("Cors", ["allow_origin", "allow_methods", "allow_headers", "max_age"])
 
-_CorsTuple.__new__.__defaults__ = (
+_CorsTuple.__new__.__defaults__ = (  # type: ignore
     None,  # Allow Origin defaults to None
     None,  # Allow Methods is optional and defaults to empty
     None,  # Allow Headers is optional and defaults to empty

diff --git a/samcli/lib/schemas/schemas_aws_config.py b/samcli/lib/schemas/schemas_aws_config.py
@@ -2,7 +2,7 @@
 
 import click
 
-from boto3 import Session
+from boto3.session import Session
 from samcli.commands.local.cli_common.user_exceptions import ResourceNotFound
 
 

diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py
@@ -5,6 +5,7 @@
 import itertools
 import os
 import pathlib
+from typing import Set
 
 _init_path = str(pathlib.Path(os.path.dirname(__file__)).parent)
 _templates = os.path.join(_init_path, "init", "templates")
@@ -85,14 +86,16 @@
     "java8.al2": ["maven", "gradle"],
 }
 
-SUPPORTED_DEP_MANAGERS = {
-    c["dependency_manager"]
+SUPPORTED_DEP_MANAGERS: Set[str] = {
+    c["dependency_manager"]  # type: ignore
     for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))
     if c["dependency_manager"]
 }
 
-RUNTIMES = set(
-    itertools.chain(*[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))])
+RUNTIMES: Set[str] = set(
+    itertools.chain(
+        *[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))]  # type: ignore
+    )
 )
 
 # When adding new Lambda runtimes, please update SAM_RUNTIME_TO_SCHEMAS_CODE_LANG_MAPPING

diff --git a/samcli/local/lambdafn/env_vars.py b/samcli/local/lambdafn/env_vars.py
@@ -131,6 +131,10 @@ def memory(self, value):
     def handler(self):
         return self._function["handler"]
 
+    @handler.setter
+    def handler(self, value):
+        self._function["handler"] = value
+
     @property
     def name(self):
         return self._function["name"]
@@ -139,10 +143,6 @@ def name(self):
     def name(self, value):
         self._function["name"] = value
 
-    @handler.setter
-    def handler(self, value):
-        self._function["handler"] = value
-
     def _get_aws_variables(self):
         """
         Returns the AWS specific environment variables that should be available in the Lambda runtime.

diff --git a/samcli/settings/__init__.py b/samcli/settings/__init__.py
@@ -17,7 +17,7 @@
 
 import os
 
-if "__SAM_CLI_TELEMETRY_ENDPOINT_URL" not in os.environ:
-    telemetry_endpoint_url = "https://aws-serverless-tools-telemetry.us-west-2.amazonaws.com/metrics"
-else:
-    telemetry_endpoint_url = os.getenv("__SAM_CLI_TELEMETRY_ENDPOINT_URL")
+
+telemetry_endpoint_url = os.getenv(
+    "__SAM_CLI_TELEMETRY_ENDPOINT_URL", "https://aws-serverless-tools-telemetry.us-west-2.amazonaws.com/metrics"
+)
diff --git a/samcli/yamlhelper.py b/samcli/yamlhelper.py
@@ -21,7 +21,10 @@
 from botocore.compat import OrderedDict
 
 import yaml
-from yaml.resolver import ScalarNode, SequenceNode
+
+# ScalarNode and SequenceNode are not declared in __all__,
+# TODO: we need to double check whether they are public and stable
+from yaml.resolver import ScalarNode, SequenceNode  # type: ignore
 
 TAG_STR = "tag:yaml.org,2002:str"
 

diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py
@@ -149,7 +149,6 @@ def _verify_invoke_built_function(self, template_path, function_logical_id, over
 
 
 class BuildIntegRubyBase(BuildIntegBase):
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"app.rb"}
     EXPECTED_RUBY_GEM = "aws-record"
 

diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py
@@ -23,7 +23,6 @@
     "Skip build tests on windows when running in CI unless overridden",
 )
 class TestBuildCommand_PythonFunctions(BuildIntegBase):
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {
         "__init__.py",
         "main.py",
@@ -135,7 +134,6 @@ def test_unsupported_runtime(self):
     "Skip build tests on windows when running in CI unless overridden",
 )
 class TestBuildCommand_NodeFunctions(BuildIntegBase):
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"node_modules", "main.js"}
     EXPECTED_NODE_MODULES = {"minimal-request-promise"}
 
@@ -598,7 +596,6 @@ def _verify_built_artifact(self, build_dir, function_logical_id, expected_files)
 class TestBuildCommand_SingleFunctionBuilds(BuildIntegBase):
     template = "many-functions-template.yaml"
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {
         "__init__.py",
         "main.py",
@@ -671,7 +668,6 @@ def _get_python_version(self):
 class TestBuildCommand_LayerBuilds(BuildIntegBase):
     template = "layers-functions-template.yaml"
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"__init__.py", "main.py", "requirements.txt"}
     EXPECTED_LAYERS_FILES_PROJECT_MANIFEST = {"__init__.py", "layer.py", "numpy", "requirements.txt"}
 
@@ -818,7 +814,6 @@ class TestBuildCommand_ProvidedFunctions(BuildIntegBase):
     # Test Suite for runtime: provided and where selection of the build workflow is implicitly makefile builder
     # if the makefile is present.
 
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"__init__.py", "main.py", "requests", "requirements.txt"}
 
     FUNCTION_LOGICAL_ID = "Function"
@@ -888,7 +883,6 @@ class TestBuildWithBuildMethod(BuildIntegBase):
     # Test Suite where `BuildMethod` is explicitly specified.
 
     template = "custom-build-function.yaml"
-    EXPECTED_FILES_GLOBAL_MANIFEST = set()
     EXPECTED_FILES_PROJECT_MANIFEST = {"__init__.py", "main.py", "requests", "requirements.txt"}
 
     FUNCTION_LOGICAL_ID = "Function"

diff --git a/tests/integration/init/schemas/schemas_test_data_setup.py b/tests/integration/init/schemas/schemas_test_data_setup.py
@@ -4,7 +4,7 @@
 import tempfile
 from unittest import TestCase
 
-from boto3 import Session
+from boto3.session import Session
 from botocore.exceptions import ClientError
 from click.testing import CliRunner
 from samcli.commands.init import cli as init_cmd

diff --git a/tests/integration/local/invoke/invoke_integ_base.py b/tests/integration/local/invoke/invoke_integ_base.py
@@ -1,10 +1,11 @@
 import os
+from typing import Optional
 from unittest import TestCase
 from pathlib import Path
 
 
 class InvokeIntegBase(TestCase):
-    template = None
+    template: Optional[Path] = None
 
     @classmethod
     def setUpClass(cls):

diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py
@@ -439,7 +439,7 @@ def test_invoke_returns_expected_results_from_git_function_with_parameters(self)
 
 
 class TestSamInstrinsicsAndPlugins(InvokeIntegBase):
-    template = "template-pseudo-params.yaml"
+    template = Path("template-pseudo-params.yaml")
 
     @pytest.mark.flaky(reruns=3)
     def test_resolve_instrincs_which_runs_plugins(self):

diff --git a/tests/integration/local/start_api/start_api_integ_base.py b/tests/integration/local/start_api/start_api_integ_base.py
@@ -1,3 +1,4 @@
+from typing import Optional
 from unittest import TestCase
 import threading
 from subprocess import Popen
@@ -8,8 +9,8 @@
 
 
 class StartApiIntegBaseClass(TestCase):
-    template = None
-    binary_data_file = None
+    template: Optional[str] = None
+    binary_data_file: Optional[str] = None
     integration_dir = str(Path(__file__).resolve().parents[2])
 
     @classmethod

diff --git a/tests/unit/commands/deploy/test_guided_config.py b/tests/unit/commands/deploy/test_guided_config.py
@@ -3,7 +3,9 @@
 from unittest.mock import patch
 
 import click
-from click.globals import _local
+
+# TODO: importing private module property, to investigate alternative for stability
+from click.globals import _local  # type: ignore
 from click import Context
 
 from samcli.commands.deploy.exceptions import GuidedDeployFailedError