Merge remote-tracking branch 'upstream/develop' into release-v1.58.0

Author: aahung

.cfnlintrc.yaml

@@ -108,6 +108,7 @@ ignore_templates:
   - tests/translator/output/**/s3_intrinsics.json
   - tests/translator/output/**/schema_validation_1.json
   - tests/translator/output/**/self_managed_kafka_with_intrinsics.json
+  - tests/translator/output/**/sqs_with_scaling_config.json # TODO: remove once SQS scaling config is available
   - tests/translator/output/**/state_machine_with_condition.json
   - tests/translator/output/**/state_machine_with_condition_and_events.json
   - tests/translator/output/**/state_machine_with_eb_dlq_target_id.json

.coveragerc

@@ -5,5 +5,4 @@ omit =
     samtranslator/schema/*
 [report]
 exclude_lines =
-    pragma: no cover
-    raise NotImplementedError.*
+    pragma: no cover

.gitignore

@@ -119,4 +119,4 @@ venv.bak/
 # Companion stack config
 integration/config/file_to_s3_map_modified.json
 
-.tmp_schema.json
+.tmp

.pylintrc

@@ -76,30 +76,19 @@ disable=
     R0401, # Cyclic import
     C0411, # import ordering
     W9015, # missing parameter in doc strings
-    R0205, # useless-object-inheritanc
     C0301, # line to long
     C0114, # missing-module-docstring
     W1202, # Use lazy % formatting in logging functions (logging-format-interpolation)
-    E1101, # No member 
-    W0622, # Redefining built-in 'property' (redefined-builtin)
+    E1101, # No member
     W0212, # protected-access
     W0201, # attribute-defined-outside-init
-    R1725, # Consider using Python 3 style super() without arguments (super-with-arguments)
     C2001, # Avoid comparisons to zero (compare-to-zero)
     R0912, # too many branches
-    W0235, # Useless super delegation in method '__init__' (useless-super-delegation)
     C0412, # Imports from package samtranslator are not grouped (ungrouped-imports)
-    W0223, # abstract-method
-    W0107, # unnecessary-pass
-    W0707, # raise-missing-from
     E0203, # access-member-before-definition
     W0221, # arguments-differ
-    R1710, # inconsistent-return-statements
     R1702, # too-many-nested-blocks
-    C0123, # Use isinstance() rather than type() for a typecheck. (unidiomatic-typecheck)
     W0105, # String statement has no effect (pointless-string-statement)
-    C0206, # Consider iterating with .items() (consider-using-dict-items)
-    W9008, # Redundant returns documentation (redundant-returns-doc)
     C0112, # empty-docstring
     C0116, # missing-function-docstring
     W9017, # differing-param-doc

MANIFEST.in

@@ -7,6 +7,8 @@ include samtranslator/policy_templates_data/schema.json
 include samtranslator/model/connector_profiles/profiles.json
 include samtranslator/schema/docs.json
 include samtranslator/schema/schema.json
+include samtranslator/schema/sam.schema.json
+include samtranslator/schema/cloudformation.schema.json
 include README.md
 include THIRD_PARTY_LICENSES
 

Makefile

@@ -25,9 +25,10 @@ black:
 
 black-check:
 	# Checking latest schema was generated (run `make schema` if this fails)
-	python samtranslator/schema/schema.py > .tmp_schema.json
-	diff -u samtranslator/schema/schema.json .tmp_schema.json
-	rm .tmp_schema.json
+	mkdir -p .tmp
+	python samtranslator/schema/schema.py --sam-schema .tmp/sam.schema.json --cfn-schema samtranslator/schema/cloudformation.schema.json --unified-schema .tmp/schema.json
+	diff -u samtranslator/schema/sam.schema.json .tmp/sam.schema.json
+	diff -u samtranslator/schema/schema.json .tmp/schema.json
 	black --check setup.py samtranslator/* tests/* integration/* bin/*.py
 	bin/json-format.py --check tests integration samtranslator/policy_templates_data
 	bin/yaml-format.py --check tests
@@ -44,15 +45,21 @@ lint:
 prepare-companion-stack:
 	pytest -v --no-cov integration/setup -m setup
 
+update-cfn-schema:
+	curl -o samtranslator/schema/cloudformation.schema.json https://raw.githubusercontent.com/awslabs/goformation/master/schema/cloudformation.schema.json
+
 schema:
-	python samtranslator/schema/schema.py > samtranslator/schema/schema.json
+	python samtranslator/schema/schema.py --sam-schema samtranslator/schema/sam.schema.json --cfn-schema samtranslator/schema/cloudformation.schema.json --unified-schema samtranslator/schema/schema.json
 
 # Command to run everytime you make changes to verify everything works
 dev: test
 
 # Verifications to run before sending a pull request
 pr: black-check lint init dev
 
+clean:
+	rm -r .tmp
+
 define HELP_MESSAGE
 
 Usage: $ make [TARGETS]

bin/run_cfn_lint.sh

@@ -7,7 +7,7 @@ VENV=.venv_cfn_lint
 # See https://github.com/aws/serverless-application-model/issues/1042
 if [ ! -d "${VENV}" ]; then
     python3 -m venv "${VENV}"
-    "${VENV}/bin/python" -m pip install cfn-lint==0.72.2 --quiet
+    "${VENV}/bin/python" -m pip install cfn-lint --quiet
 fi
 
 "${VENV}/bin/cfn-lint" --format parseable

integration/resources/templates/combination/api_with_authorizer_apikey.yaml

@@ -104,7 +104,7 @@ Outputs:
   ApiKeyId:
     Description: API Key ID
     Value:
-      Fn::GetAtt: MyFirstApiKey.APIKeyId
+      Ref: MyFirstApiKey
   ApiUrl:
     Description: API endpoint URL for Prod environment
     Value:

requirements/base.txt

@@ -1,4 +1,4 @@
 boto3>=1.19.5,==1.*
 jsonschema<5,>=3.2  # TODO: evaluate risk of removing jsonschema 3.x support
-pydantic~=1.10.2
+pydantic~=1.8
 typing_extensions~=4.4.0 # 3.7 doesn't have Literal

samtranslator/feature_toggle/dialup.py

@@ -1,19 +1,20 @@
 import hashlib
+from abc import ABC, abstractmethod
 
 
-class BaseDialup(object):
+class BaseDialup(ABC):
     """BaseDialup class to provide an interface for all dialup classes"""
 
     def __init__(self, region_config, **kwargs):  # type: ignore[no-untyped-def]
         self.region_config = region_config
 
-    def is_enabled(self):  # type: ignore[no-untyped-def]
+    @abstractmethod
+    def is_enabled(self) -> bool:
         """
         Returns a bool on whether this dialup is enabled or not
         """
-        raise NotImplementedError
 
-    def __str__(self):  # type: ignore[no-untyped-def]
+    def __str__(self) -> str:
         return self.__class__.__name__
 
 
@@ -23,7 +24,7 @@ class DisabledDialup(BaseDialup):
     """
 
     def __init__(self, region_config, **kwargs):  # type: ignore[no-untyped-def]
-        super(DisabledDialup, self).__init__(region_config)  # type: ignore[no-untyped-call]
+        super().__init__(region_config)  # type: ignore[no-untyped-call]
 
     def is_enabled(self) -> bool:
         return False
@@ -36,7 +37,7 @@ class ToggleDialup(BaseDialup):
     """
 
     def __init__(self, region_config, **kwargs):  # type: ignore[no-untyped-def]
-        super(ToggleDialup, self).__init__(region_config)  # type: ignore[no-untyped-call]
+        super().__init__(region_config)  # type: ignore[no-untyped-call]
         self.region_config = region_config
 
     def is_enabled(self):  # type: ignore[no-untyped-def]
@@ -50,11 +51,11 @@ class SimpleAccountPercentileDialup(BaseDialup):
     """
 
     def __init__(self, region_config, account_id, feature_name, **kwargs):  # type: ignore[no-untyped-def]
-        super(SimpleAccountPercentileDialup, self).__init__(region_config)  # type: ignore[no-untyped-call]
+        super().__init__(region_config)  # type: ignore[no-untyped-call]
         self.account_id = account_id
         self.feature_name = feature_name
 
-    def _get_account_percentile(self):  # type: ignore[no-untyped-def]
+    def _get_account_percentile(self) -> int:
         """
         Get account percentile based on sha256 hash of account ID and feature_name
 
@@ -65,10 +66,10 @@ def _get_account_percentile(self):  # type: ignore[no-untyped-def]
         m.update(self.feature_name.encode())
         return int(m.hexdigest(), 16) % 100
 
-    def is_enabled(self):  # type: ignore[no-untyped-def]
+    def is_enabled(self) -> bool:
         """
         Enable when account_percentile falls within target_percentile
         Meaning only (target_percentile)% of accounts will be enabled
         """
-        target_percentile = self.region_config.get("enabled-%", 0)
-        return self._get_account_percentile() < target_percentile  # type: ignore[no-untyped-call]
+        target_percentile: int = self.region_config.get("enabled-%", 0)
+        return self._get_account_percentile() < target_percentile

samtranslator/feature_toggle/feature_toggle.py

@@ -1,4 +1,7 @@
 import json
+from abc import ABC, abstractmethod
+from typing import Any, Dict, cast
+
 import boto3
 import logging
 
@@ -87,15 +90,16 @@ def is_enabled(self, feature_name: str) -> bool:
         return is_enabled
 
 
-class FeatureToggleConfigProvider:
+class FeatureToggleConfigProvider(ABC):
     """Interface for all FeatureToggle config providers"""
 
     def __init__(self) -> None:
         pass
 
     @property
-    def config(self):  # type: ignore[no-untyped-def]
-        raise NotImplementedError
+    @abstractmethod
+    def config(self) -> Dict[str, Any]:
+        pass
 
 
 class FeatureToggleDefaultConfigProvider(FeatureToggleConfigProvider):
@@ -105,7 +109,7 @@ def __init__(self) -> None:
         FeatureToggleConfigProvider.__init__(self)
 
     @property
-    def config(self):  # type: ignore[no-untyped-def]
+    def config(self) -> Dict[str, Any]:
         return {}
 
 
@@ -116,10 +120,10 @@ def __init__(self, local_config_path):  # type: ignore[no-untyped-def]
         FeatureToggleConfigProvider.__init__(self)
         with open(local_config_path, "r", encoding="utf-8") as f:
             config_json = f.read()
-        self.feature_toggle_config = json.loads(config_json)
+        self.feature_toggle_config = cast(Dict[str, Any], json.loads(config_json))
 
     @property
-    def config(self):  # type: ignore[no-untyped-def]
+    def config(self) -> Dict[str, Any]:
         return self.feature_toggle_config
 
 
@@ -147,13 +151,13 @@ def __init__(self, application_id, environment_id, configuration_profile_id, app
                 ClientId="FeatureToggleAppConfigConfigProvider",
             )
             binary_config_string = response["Content"].read()
-            self.feature_toggle_config = json.loads(binary_config_string.decode("utf-8"))
+            self.feature_toggle_config = cast(Dict[str, Any], json.loads(binary_config_string.decode("utf-8")))
             LOG.info("Finished loading feature toggle config from AppConfig.")
         except Exception as ex:
             LOG.error("Failed to load config from AppConfig: {}. Using empty config.".format(ex))
             # There is chance that AppConfig is not available in a particular region.
-            self.feature_toggle_config = json.loads("{}")
+            self.feature_toggle_config = {}
 
     @property
-    def config(self):  # type: ignore[no-untyped-def]
+    def config(self) -> Dict[str, Any]:
         return self.feature_toggle_config