fix: --base-dir is disregarded in sam build (#2718)

* fix: build_dir is disregarded in sam build * fix unit tests * Add debug log * Add more comments * Add integ test * Add integ test for docker context
aws · Mar 16, 2021 · f459d50 · f459d50
1 parent 9aff4ee
commit f459d50
Show file tree

Hide file tree

Showing 21 changed files with 315 additions and 38 deletions.
diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py
@@ -49,6 +49,10 @@ def __init__(
         self._resource_identifier = resource_identifier
         self._template_file = template_file
         self._base_dir = base_dir
+
+        # Note(xinhol): use_raw_codeuri is temporary to fix a bug, and will be removed for a permanent solution.
+        self._use_raw_codeuri = bool(self._base_dir)
+
         self._build_dir = build_dir
         self._cache_dir = cache_dir
         self._manifest_path = manifest_path
@@ -80,8 +84,11 @@ def __enter__(self) -> "BuildContext":
                 "\n".join([f"- {full_path}" for full_path in remote_stack_full_paths]),
             )
 
-        self._function_provider = SamFunctionProvider(self.stacks)
-        self._layer_provider = SamLayerProvider(self.stacks)
+        # Note(xinhol): self._use_raw_codeuri is added temporarily to fix issue #2717
+        # when base_dir is provided, codeuri should not be resolved based on template file path.
+        # we will refactor to make all path resolution inside providers intead of in multiple places
+        self._function_provider = SamFunctionProvider(self.stacks, self._use_raw_codeuri)
+        self._layer_provider = SamLayerProvider(self.stacks, self._use_raw_codeuri)
 
         if not self._base_dir:
             # Base directory, if not provided, is the directory containing the template

diff --git a/samcli/lib/providers/sam_function_provider.py b/samcli/lib/providers/sam_function_provider.py
@@ -23,7 +23,9 @@ class SamFunctionProvider(SamBaseProvider):
     It may or may not contain a function.
     """
 
-    def __init__(self, stacks: List[Stack], ignore_code_extraction_warnings: bool = False) -> None:
+    def __init__(
+        self, stacks: List[Stack], use_raw_codeuri: bool = False, ignore_code_extraction_warnings: bool = False
+    ) -> None:
         """
         Initialize the class with SAM template data. The SAM template passed to this provider is assumed
         to be valid, normalized and a dictionary. It should be normalized by running all pre-processing
@@ -35,6 +37,8 @@ def __init__(self, stacks: List[Stack], ignore_code_extraction_warnings: bool =
         You need to explicitly update the class with new template, if necessary.
 
         :param dict stacks: List of stacks functions are extracted from
+        :param bool use_raw_codeuri: Do not resolve adjust core_uri based on the template path, use the raw uri.
+            Note(xinhol): use_raw_codeuri is temporary to fix a bug, and will be removed for a permanent solution.
         :param bool ignore_code_extraction_warnings: Ignores Log warnings
         """
 
@@ -44,7 +48,9 @@ def __init__(self, stacks: List[Stack], ignore_code_extraction_warnings: bool =
             LOG.debug("%d resources found in the stack %s", len(stack.resources), stack.stack_path)
 
         # Store a map of function full_path to function information for quick reference
-        self.functions = SamFunctionProvider._extract_functions(self.stacks, ignore_code_extraction_warnings)
+        self.functions = SamFunctionProvider._extract_functions(
+            self.stacks, use_raw_codeuri, ignore_code_extraction_warnings
+        )
 
         self._deprecated_runtimes = {"nodejs4.3", "nodejs6.10", "nodejs8.10", "dotnetcore2.0"}
         self._colored = Colored()
@@ -98,12 +104,15 @@ def get_all(self) -> Iterator[Function]:
             yield function
 
     @staticmethod
-    def _extract_functions(stacks: List[Stack], ignore_code_extraction_warnings: bool = False) -> Dict[str, Function]:
+    def _extract_functions(
+        stacks: List[Stack], use_raw_codeuri: bool = False, ignore_code_extraction_warnings: bool = False
+    ) -> Dict[str, Function]:
         """
         Extracts and returns function information from the given dictionary of SAM/CloudFormation resources. This
         method supports functions defined with AWS::Serverless::Function and AWS::Lambda::Function
 
         :param stacks: List of SAM/CloudFormation stacks to extract functions from
+        :param bool use_raw_codeuri: Do not resolve adjust core_uri based on the template path, use the raw uri.
         :param bool ignore_code_extraction_warnings: suppress log statements on code extraction from resources.
         :return dict(string : samcli.commands.local.lib.provider.Function): Dictionary of function full_path to the
             Function configuration object
@@ -124,13 +133,15 @@ def _extract_functions(stacks: List[Stack], ignore_code_extraction_warnings: boo
                     layers = SamFunctionProvider._parse_layer_info(
                         stack,
                         resource_properties.get("Layers", []),
+                        use_raw_codeuri,
                         ignore_code_extraction_warnings=ignore_code_extraction_warnings,
                     )
                     function = SamFunctionProvider._convert_sam_function_resource(
                         stack,
                         name,
                         resource_properties,
                         layers,
+                        use_raw_codeuri,
                         ignore_code_extraction_warnings=ignore_code_extraction_warnings,
                     )
                     result[function.full_path] = function
@@ -139,10 +150,11 @@ def _extract_functions(stacks: List[Stack], ignore_code_extraction_warnings: boo
                     layers = SamFunctionProvider._parse_layer_info(
                         stack,
                         resource_properties.get("Layers", []),
+                        use_raw_codeuri,
                         ignore_code_extraction_warnings=ignore_code_extraction_warnings,
                     )
                     function = SamFunctionProvider._convert_lambda_function_resource(
-                        stack, name, resource_properties, layers
+                        stack, name, resource_properties, layers, use_raw_codeuri
                     )
                     result[function.full_path] = function
 
@@ -156,6 +168,7 @@ def _convert_sam_function_resource(
         name: str,
         resource_properties: Dict,
         layers: List[LayerVersion],
+        use_raw_codeuri: bool = False,
         ignore_code_extraction_warnings: bool = False,
     ) -> Function:
         """
@@ -196,18 +209,12 @@ def _convert_sam_function_resource(
             LOG.debug("Found Serverless function with name='%s' and ImageUri='%s'", name, imageuri)
 
         return SamFunctionProvider._build_function_configuration(
-            stack,
-            name,
-            codeuri,
-            resource_properties,
-            layers,
-            inlinecode,
-            imageuri,
+            stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, use_raw_codeuri
         )
 
     @staticmethod
     def _convert_lambda_function_resource(
-        stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion]
+        stack: Stack, name: str, resource_properties: Dict, layers: List[LayerVersion], use_raw_codeuri: bool = False
     ) -> Function:
         """
         Converts a AWS::Lambda::Function resource to a Function configuration usable by the provider.
@@ -220,6 +227,8 @@ def _convert_lambda_function_resource(
             Properties of this resource
         layers List(samcli.commands.local.lib.provider.Layer)
             List of the Layer objects created from the template and layer list defined on the function.
+        use_raw_codeuri
+            Do not resolve adjust core_uri based on the template path, use the raw uri.
 
         Returns
         -------
@@ -250,13 +259,7 @@ def _convert_lambda_function_resource(
             LOG.debug("Found Lambda function with name='%s' and Imageuri='%s'", name, imageuri)
 
         return SamFunctionProvider._build_function_configuration(
-            stack,
-            name,
-            codeuri,
-            resource_properties,
-            layers,
-            inlinecode,
-            imageuri,
+            stack, name, codeuri, resource_properties, layers, inlinecode, imageuri, use_raw_codeuri
         )
 
     @staticmethod
@@ -268,6 +271,7 @@ def _build_function_configuration(
         layers: List,
         inlinecode: Optional[str],
         imageuri: Optional[str],
+        use_raw_codeuri: bool = False,
     ) -> Function:
         """
         Builds a Function configuration usable by the provider.
@@ -282,18 +286,29 @@ def _build_function_configuration(
             Properties of this resource
         layers List(samcli.commands.local.lib.provider.Layer)
             List of the Layer objects created from the template and layer list defined on the function.
+        use_raw_codeuri
+            Do not resolve adjust core_uri based on the template path, use the raw uri.
 
         Returns
         -------
         samcli.commands.local.lib.provider.Function
             Function configuration
         """
         metadata = resource_properties.get("Metadata", None)
-        if metadata and "DockerContext" in metadata:
+        if metadata and "DockerContext" in metadata and not use_raw_codeuri:
+            LOG.debug(
+                "--base-dir is presented not, adjusting uri %s relative to %s",
+                metadata["DockerContext"],
+                stack.location,
+            )
             metadata["DockerContext"] = SamLocalStackProvider.normalize_resource_path(
                 stack.location, metadata["DockerContext"]
             )
 
+        if codeuri and not use_raw_codeuri:
+            LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
+            codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)
+
         return Function(
             stack_path=stack.stack_path,
             name=name,
@@ -303,7 +318,7 @@ def _build_function_configuration(
             memory=resource_properties.get("MemorySize"),
             timeout=resource_properties.get("Timeout"),
             handler=resource_properties.get("Handler"),
-            codeuri=SamLocalStackProvider.normalize_resource_path(stack.location, codeuri) if codeuri else None,
+            codeuri=codeuri,
             imageuri=imageuri if imageuri else resource_properties.get("ImageUri"),
             imageconfig=resource_properties.get("ImageConfig"),
             environment=resource_properties.get("Environment"),
@@ -319,6 +334,7 @@ def _build_function_configuration(
     def _parse_layer_info(
         stack: Stack,
         list_of_layers: List[Any],
+        use_raw_codeuri: bool = False,
         ignore_code_extraction_warnings: bool = False,
     ) -> List[LayerVersion]:
         """
@@ -331,6 +347,8 @@ def _parse_layer_info(
         list_of_layers : List[Any]
             List of layers that are defined within the Layers Property on a function,
             layer can be defined as string or Dict, in case customers define it in other types, use "Any" here.
+        use_raw_codeuri : bool
+            Do not resolve adjust core_uri based on the template path, use the raw uri.
         ignore_code_extraction_warnings : bool
             Whether to print warning when codeuri is not a local pth
 
@@ -390,10 +408,14 @@ def _parse_layer_info(
                         layer_logical_id, layer_properties, "ContentUri", ignore_code_extraction_warnings
                     )
 
+                if codeuri and not use_raw_codeuri:
+                    LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
+                    codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)
+
                 layers.append(
                     LayerVersion(
                         layer_logical_id,
-                        SamLocalStackProvider.normalize_resource_path(stack.location, codeuri) if codeuri else None,
+                        codeuri,
                         compatible_runtimes,
                         layer_resource.get("Metadata", None),
                         stack_path=stack.stack_path,

diff --git a/samcli/lib/providers/sam_layer_provider.py b/samcli/lib/providers/sam_layer_provider.py
@@ -20,7 +20,7 @@ class SamLayerProvider(SamBaseProvider):
     It may or may not contain a layer.
     """
 
-    def __init__(self, stacks: List[Stack]) -> None:
+    def __init__(self, stacks: List[Stack], use_raw_codeuri: bool = False) -> None:
         """
         Initialize the class with SAM template data. The SAM template passed to this provider is assumed
         to be valid, normalized and a dictionary. It should be normalized by running all pre-processing
@@ -34,8 +34,11 @@ def __init__(self, stacks: List[Stack]) -> None:
         Parameters
         ----------
         :param dict stacks: List of stacks layers are extracted from
+        :param bool use_raw_codeuri: Do not resolve adjust core_uri based on the template path, use the raw uri.
+            Note(xinhol): use_raw_codeuri is temporary to fix a bug, and will be removed for a permanent solution.
         """
         self._stacks = stacks
+        self._use_raw_codeuri = use_raw_codeuri
 
         self._layers = self._extract_layers()
 
@@ -102,9 +105,13 @@ def _convert_lambda_layer_resource(self, stack: Stack, layer_logical_id: str, la
         if resource_type == self.LAMBDA_LAYER:
             codeuri = SamLayerProvider._extract_lambda_function_code(layer_properties, "Content")
 
+        if codeuri and not self._use_raw_codeuri:
+            LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
+            codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)
+
         return LayerVersion(
             layer_logical_id,
-            SamLocalStackProvider.normalize_resource_path(stack.location, codeuri) if codeuri else None,
+            codeuri,
             compatible_runtimes,
             layer_resource.get("Metadata", None),
             stack_path=stack.stack_path,

diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py
@@ -1588,6 +1588,13 @@ def test_nested_build(self, use_container, cached, parallel):
             )
 
 
+@parameterized_class(
+    ("template", "use_base_dir"),
+    [
+        (os.path.join("deep-nested", "template.yaml"), False),
+        (os.path.join("base-dir", "template", "template.yaml"), "use_base_dir"),
+    ],
+)
 class TestBuildWithNestedStacks3Level(NestedBuildIntegBase):
     """
     In this template, it has the same structure as .aws-sam/build
@@ -1610,7 +1617,12 @@ def test_nested_build(self):
         if SKIP_DOCKER_TESTS:
             self.skipTest(SKIP_DOCKER_MESSAGE)
 
-        cmdlist = self.get_command_list(use_container=True, cached=True, parallel=True)
+        cmdlist = self.get_command_list(
+            use_container=True,
+            cached=True,
+            parallel=True,
+            base_dir=(os.path.join(self.test_data_path, "base-dir") if self.use_base_dir else None),
+        )
 
         LOG.info("Running Command: %s", cmdlist)
         LOG.info(self.working_dir)
@@ -1712,8 +1724,14 @@ def test_nested_build(self):
             )
 
 
+@parameterized_class(
+    ("template", "use_base_dir"),
+    [
+        (os.path.join("nested-parent", "nested-root-template-image.yaml"), False),
+        (os.path.join("base-dir-image", "template", "template.yaml"), "use_base_dir"),
+    ],
+)
 class TestBuildWithNestedStacksImage(NestedBuildIntegBase):
-    template = os.path.join("nested-parent", "nested-root-template-image.yaml")
 
     EXPECTED_FILES_PROJECT_MANIFEST = {
         "__init__.py",
@@ -1757,7 +1775,11 @@ def test_nested_build(self, use_container, cached, parallel):
             "LocalNestedFuncHandler": "main.handler",
         }
         cmdlist = self.get_command_list(
-            use_container=use_container, parameter_overrides=overrides, cached=cached, parallel=parallel
+            use_container=use_container,
+            parameter_overrides=overrides,
+            cached=cached,
+            parallel=parallel,
+            base_dir=(os.path.join(self.test_data_path, "base-dir-image") if self.use_base_dir else None),
         )
 
         LOG.info("Running Command: %s", cmdlist)

diff --git a/tests/integration/testdata/buildcmd/base-dir-image/template/child/template.yaml b/tests/integration/testdata/buildcmd/base-dir-image/template/child/template.yaml
@@ -0,0 +1,65 @@
+AWSTemplateFormatVersion : '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+
+Parameteres:
+  Runtime:
+    Type: String
+  Handler1:
+    Type: String
+  Handler2:
+    Type: String
+  DockerFile:
+    Type: String
+  Tag:
+    Type: String
+
+
+Resources:
+
+  Function1:
+    Type: AWS::Serverless::Function
+    Properties:
+      PackageType: Image
+      ImageConfig:
+        Command:
+          - !Ref Handler1
+      Timeout: 600
+    Metadata:
+      DockerTag: !Ref Tag
+      # this is relative to base-dir-image
+      DockerContext: ../PythonImage
+      Dockerfile: !Ref DockerFile
+      DockerBuildArgs:
+        BASE_RUNTIME: !Ref Runtime
+
+  Function2:
+    Type: AWS::Serverless::Function
+    Properties:
+      PackageType: Image
+      ImageConfig:
+        Command:
+          - !Ref Handler2
+      Timeout: 600
+    Metadata:
+      DockerTag: !Ref Tag
+      # this is relative to base-dir-image
+      DockerContext: ../PythonImage
+      Dockerfile: !Ref DockerFile
+      DockerBuildArgs:
+        BASE_RUNTIME: !Ref Runtime
+
+  OtherRelativePathResource:
+    Type: AWS::ApiGateway::RestApi
+    Properties:
+      BodyS3Location: SomeRelativePath
+
+  GlueResource:
+    Type: AWS::Glue::Job
+    Properties:
+      Command:
+        ScriptLocation: SomeRelativePath
+
+  ExampleNestedStack:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      TemplateURL: https://s3.amazonaws.com/examplebucket/exampletemplate.yml