Skip to content

Commit

Permalink
fix: Skip functions/layers pointing to S3 location in providers (#2756)
Browse files Browse the repository at this point in the history 
* refactor: Extract logics into _locate_layer_from_ref()

* providers no longer extract functions/layers with s3 uri
  • Loading branch information
@aahung
aahung authored Mar 30, 2021
1 parent 9838d05 commit fd25093
Show file tree
Hide file tree
Showing 7 changed files with 286 additions and 203 deletions.
76 changes: 33 additions & 43 deletions samcli/lib/providers/sam_base_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import logging

from typing import Any, Dict, Optional, cast, Iterable
from typing import Any, Dict, Optional, cast, Iterable, Union
from samcli.commands._utils.resources import AWS_SERVERLESS_APPLICATION, AWS_CLOUDFORMATION_STACK
from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver
from samcli.lib.intrinsic_resolver.intrinsics_symbol_table import IntrinsicsSymbolTable
Expand All @@ -27,6 +27,13 @@ class SamBaseProvider:
CLOUDFORMATION_STACK = AWS_CLOUDFORMATION_STACK
DEFAULT_CODEURI = "."

CODE_PROPERTY_KEYS = {
LAMBDA_FUNCTION: "Code",
SERVERLESS_FUNCTION: "CodeUri",
LAMBDA_LAYER: "Content",
SERVERLESS_LAYER: "ContentUri",
}

def get(self, name: str) -> Optional[Any]:
"""
Given name of the function, this method must return the Function object
Expand All @@ -45,9 +52,9 @@ def get_all(self) -> Iterable:
raise NotImplementedError("not implemented")

@staticmethod
def _extract_lambda_function_code(resource_properties: Dict, code_property_key: str) -> str:
def _extract_codeuri(resource_properties: Dict, code_property_key: str) -> str:
"""
Extracts the Lambda Function Code from the Resource Properties
Extracts the Function/Layer code path from the Resource Properties
Parameters
----------
Expand All @@ -61,14 +68,36 @@ def _extract_lambda_function_code(resource_properties: Dict, code_property_key:
str
Representing the local code path
"""

codeuri = resource_properties.get(code_property_key, SamBaseProvider.DEFAULT_CODEURI)

if isinstance(codeuri, dict):
return SamBaseProvider.DEFAULT_CODEURI

return cast(str, codeuri)

@staticmethod
def _is_s3_location(location: Optional[Union[str, Dict]]) -> bool:
"""
the input could be:
- CodeUri of Serverless::Function
- Code of Lambda::Function
- ContentUri of Serverless::LayerVersion
- Content of Lambda::LayerVersion
"""
return (isinstance(location, dict) and ("S3Bucket" in location or "Bucket" in location)) or (
isinstance(location, str) and location.startswith("s3://")
)

@staticmethod
def _warn_code_extraction(resource_type: str, resource_name: str, code_property: str) -> None:
LOG.warning(
"The resource %s '%s' has specified S3 location for %s. "
"It will not be built and SAM CLI does not support invoking it locally.",
resource_type,
resource_name,
code_property,
)

@staticmethod
def _extract_lambda_function_imageuri(resource_properties: Dict, code_property_key: str) -> Optional[str]:
"""
Expand Down Expand Up @@ -107,45 +136,6 @@ def _extract_sam_function_imageuri(resource_properties: Dict, code_property_key:
"""
return resource_properties.get(code_property_key, None)

@staticmethod
def _extract_sam_function_codeuri(
name: str,
resource_properties: Dict[str, Any],
code_property_key: str,
ignore_code_extraction_warnings: bool = False,
) -> str:
"""
Extracts the SAM Function CodeUri from the Resource Properties
Parameters
----------
name str
LogicalId of the resource
resource_properties dict
Dictionary representing the Properties of the Resource
code_property_key str
Property Key of the code on the Resource
ignore_code_extraction_warnings
Boolean to ignore log statements on code extraction from Resources.
Returns
-------
str
Representing the local code path
"""
codeuri = resource_properties.get(code_property_key, SamBaseProvider.DEFAULT_CODEURI)
# CodeUri can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported
if isinstance(codeuri, dict) or (isinstance(codeuri, str) and codeuri.startswith("s3://")):
if not ignore_code_extraction_warnings:
LOG.warning(
"Lambda function '%s' has specified S3 location for CodeUri which is unsupported. "
"Using default value of '%s' instead",
name,
SamBaseProvider.DEFAULT_CODEURI,
)
return SamBaseProvider.DEFAULT_CODEURI
return cast(str, codeuri)

@staticmethod
def get_template(template_dict: Dict, parameter_overrides: Optional[Dict[str, str]] = None) -> Dict:
"""
Expand Down
98 changes: 54 additions & 44 deletions samcli/lib/providers/sam_function_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,14 @@ def _extract_functions(
if resource_metadata:
resource_properties["Metadata"] = resource_metadata

if resource_type in [SamFunctionProvider.SERVERLESS_FUNCTION, SamFunctionProvider.LAMBDA_FUNCTION]:
code_property_key = SamBaseProvider.CODE_PROPERTY_KEYS[resource_type]
if SamBaseProvider._is_s3_location(resource_properties.get(code_property_key)):
# CodeUri can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported
if not ignore_code_extraction_warnings:
SamFunctionProvider._warn_code_extraction(resource_type, name, code_property_key)
continue

if resource_type == SamFunctionProvider.SERVERLESS_FUNCTION:
layers = SamFunctionProvider._parse_layer_info(
stack,
Expand All @@ -142,7 +150,6 @@ def _extract_functions(
resource_properties,
layers,
use_raw_codeuri,
ignore_code_extraction_warnings=ignore_code_extraction_warnings,
)
result[function.full_path] = function

Expand All @@ -169,7 +176,6 @@ def _convert_sam_function_resource(
resource_properties: Dict,
layers: List[LayerVersion],
use_raw_codeuri: bool = False,
ignore_code_extraction_warnings: bool = False,
) -> Function:
"""
Converts a AWS::Serverless::Function resource to a Function configuration usable by the provider.
Expand Down Expand Up @@ -197,12 +203,7 @@ def _convert_sam_function_resource(
LOG.debug("Found Serverless function with name='%s' and InlineCode", name)
codeuri = None
else:
codeuri = SamFunctionProvider._extract_sam_function_codeuri(
name,
resource_properties,
"CodeUri",
ignore_code_extraction_warnings=ignore_code_extraction_warnings,
)
codeuri = SamBaseProvider._extract_codeuri(resource_properties, "CodeUri")
LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri)
elif packagetype == IMAGE:
imageuri = SamFunctionProvider._extract_sam_function_imageuri(resource_properties, "ImageUri")
Expand Down Expand Up @@ -252,7 +253,7 @@ def _convert_lambda_function_resource(
LOG.debug("Found Lambda function with name='%s' and Code ZipFile", name)
codeuri = None
else:
codeuri = SamFunctionProvider._extract_lambda_function_code(resource_properties, "Code")
codeuri = SamBaseProvider._extract_codeuri(resource_properties, "Code")
LOG.debug("Found Lambda function with name='%s' and CodeUri='%s'", name, codeuri)
elif packagetype == IMAGE:
imageuri = SamFunctionProvider._extract_lambda_function_imageuri(resource_properties, "Code")
Expand Down Expand Up @@ -297,7 +298,7 @@ def _build_function_configuration(
metadata = resource_properties.get("Metadata", None)
if metadata and "DockerContext" in metadata and not use_raw_codeuri:
LOG.debug(
"--base-dir is presented not, adjusting uri %s relative to %s",
"--base-dir is not presented, adjusting uri %s relative to %s",
metadata["DockerContext"],
stack.location,
)
Expand All @@ -306,7 +307,7 @@ def _build_function_configuration(
)

if codeuri and not use_raw_codeuri:
LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location)
codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)

return Function(
Expand Down Expand Up @@ -387,40 +388,11 @@ def _parse_layer_info(
# In the list of layers that is defined within a template, you can reference a LayerVersion resource.
# When running locally, we need to follow that Ref so we can extract the local path to the layer code.
if isinstance(layer, dict) and layer.get("Ref"):
layer_logical_id = cast(str, layer.get("Ref"))
layer_resource = stack.resources.get(layer_logical_id)
if not layer_resource or layer_resource.get("Type", "") not in (
SamFunctionProvider.SERVERLESS_LAYER,
SamFunctionProvider.LAMBDA_LAYER,
):
raise InvalidLayerReference()

layer_properties = layer_resource.get("Properties", {})
resource_type = layer_resource.get("Type")
compatible_runtimes = layer_properties.get("CompatibleRuntimes")
codeuri: Optional[str] = None

if resource_type == SamFunctionProvider.LAMBDA_LAYER:
codeuri = SamFunctionProvider._extract_lambda_function_code(layer_properties, "Content")

if resource_type == SamFunctionProvider.SERVERLESS_LAYER:
codeuri = SamFunctionProvider._extract_sam_function_codeuri(
layer_logical_id, layer_properties, "ContentUri", ignore_code_extraction_warnings
)

if codeuri and not use_raw_codeuri:
LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)

layers.append(
LayerVersion(
layer_logical_id,
codeuri,
compatible_runtimes,
layer_resource.get("Metadata", None),
stack_path=stack.stack_path,
)
found_layer = SamFunctionProvider._locate_layer_from_ref(
stack, layer, use_raw_codeuri, ignore_code_extraction_warnings
)
if found_layer:
layers.append(found_layer)
else:
LOG.debug(
'layer "%s" is not recognizable, '
Expand All @@ -430,6 +402,44 @@ def _parse_layer_info(

return layers

@staticmethod
def _locate_layer_from_ref(
stack: Stack, layer: Dict, use_raw_codeuri: bool = False, ignore_code_extraction_warnings: bool = False
) -> Optional[LayerVersion]:
layer_logical_id = cast(str, layer.get("Ref"))
layer_resource = stack.resources.get(layer_logical_id)
if not layer_resource or layer_resource.get("Type", "") not in (
SamFunctionProvider.SERVERLESS_LAYER,
SamFunctionProvider.LAMBDA_LAYER,
):
raise InvalidLayerReference()

layer_properties = layer_resource.get("Properties", {})
resource_type = layer_resource.get("Type")
compatible_runtimes = layer_properties.get("CompatibleRuntimes")
codeuri: Optional[str] = None

if resource_type in [SamFunctionProvider.LAMBDA_LAYER, SamFunctionProvider.SERVERLESS_LAYER]:
code_property_key = SamBaseProvider.CODE_PROPERTY_KEYS[resource_type]
if SamBaseProvider._is_s3_location(layer_properties.get(code_property_key)):
# Content can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported
if not ignore_code_extraction_warnings:
SamFunctionProvider._warn_code_extraction(resource_type, layer_logical_id, code_property_key)
return None
codeuri = SamBaseProvider._extract_codeuri(layer_properties, code_property_key)

if codeuri and not use_raw_codeuri:
LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location)
codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)

return LayerVersion(
layer_logical_id,
codeuri,
compatible_runtimes,
layer_resource.get("Metadata", None),
stack_path=stack.stack_path,
)

def get_resources_by_stack_path(self, stack_path: str) -> Dict:
candidates = [stack.resources for stack in self.stacks if stack.stack_path == stack_path]
if not candidates:
Expand Down
60 changes: 41 additions & 19 deletions samcli/lib/providers/sam_layer_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,38 +81,60 @@ def _extract_layers(self) -> List[LayerVersion]:
layers = []
for stack in self._stacks:
for name, resource in stack.resources.items():
if resource.get("Type") in [self.LAMBDA_LAYER, self.SERVERLESS_LAYER]:
layers.append(self._convert_lambda_layer_resource(stack, name, resource))
# In the list of layers that is defined within a template, you can reference a LayerVersion resource.
# When running locally, we need to follow that Ref so we can extract the local path to the layer code.
resource_type = resource.get("Type")
resource_properties = resource.get("Properties", {})

if resource_type in [SamBaseProvider.LAMBDA_LAYER, SamBaseProvider.SERVERLESS_LAYER]:
code_property_key = SamBaseProvider.CODE_PROPERTY_KEYS[resource_type]
if SamBaseProvider._is_s3_location(resource_properties.get(code_property_key)):
# Content can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported
SamBaseProvider._warn_code_extraction(resource_type, name, code_property_key)
continue
codeuri = SamBaseProvider._extract_codeuri(resource_properties, code_property_key)

compatible_runtimes = resource_properties.get("CompatibleRuntimes")
metadata = resource.get("Metadata", None)
layers.append(
self._convert_lambda_layer_resource(stack, name, codeuri, compatible_runtimes, metadata)
)
return layers

def _convert_lambda_layer_resource(self, stack: Stack, layer_logical_id: str, layer_resource: Dict) -> LayerVersion:
def _convert_lambda_layer_resource(
self,
stack: Stack,
layer_logical_id: str,
codeuri: str,
compatible_runtimes: Optional[List[str]],
metadata: Optional[Dict],
) -> LayerVersion:
"""
Convert layer resource into {LayerVersion} object.
Parameters
----------
layer_logical_id: LogicalID of resource.
layer_resource: resource in template.
stack
layer_logical_id
LogicalID of resource.
codeuri
codeuri of the layer
compatible_runtimes
list of compatible runtimes
metadata
dictionary of layer metadata
Returns
-------
LayerVersion
The layer object
"""
# In the list of layers that is defined within a template, you can reference a LayerVersion resource.
# When running locally, we need to follow that Ref so we can extract the local path to the layer code.
layer_properties = layer_resource.get("Properties", {})
resource_type = layer_resource.get("Type")
compatible_runtimes = layer_properties.get("CompatibleRuntimes")
codeuri = None

if resource_type == self.SERVERLESS_LAYER:
codeuri = SamLayerProvider._extract_sam_function_codeuri(layer_logical_id, layer_properties, "ContentUri")
if resource_type == self.LAMBDA_LAYER:
codeuri = SamLayerProvider._extract_lambda_function_code(layer_properties, "Content")

if codeuri and not self._use_raw_codeuri:
LOG.debug("--base-dir is presented not, adjusting uri %s relative to %s", codeuri, stack.location)
LOG.debug("--base-dir is not presented, adjusting uri %s relative to %s", codeuri, stack.location)
codeuri = SamLocalStackProvider.normalize_resource_path(stack.location, codeuri)

return LayerVersion(
layer_logical_id,
codeuri,
compatible_runtimes,
layer_resource.get("Metadata", None),
metadata,
stack_path=stack.stack_path,
)
Loading

0 comments on commit fd25093

Please sign in to comment.