feat: allow concurrent artifact uploads

docs/sam-config-docs.md

@@ -28,6 +28,18 @@ region="us-east-1"
 profile="srirammv"
 ```
 
+### Specifying a Boolean deployment option
+
+```
+[default.deploy.parameters]
+parallel_upload=true
+```
+
+Setting `parallel_upload` to `true` is equivalent to passing `--parallel-upload` on
+`sam deploy`, enabling concurrent S3/ECR uploads during the packaging phase.
+Up to 8 uploads run at once; set the `SAM_CLI_PARALLEL_UPLOAD_WORKERS` environment variable
+to change the number of concurrent uploads.
+
 Version
 -------
 

samcli/commands/deploy/command.py

@@ -159,6 +159,13 @@
 @image_repository_option
 @image_repositories_option
 @force_upload_option
+@click.option(
+    "--parallel-upload",
+    is_flag=True,
+    default=False,
+    help="Enable parallel upload of artifacts to S3/ECR during packaging before deployment. "
+    "Runs up to 8 uploads at once; set the SAM_CLI_PARALLEL_UPLOAD_WORKERS environment variable to change this.",
+)
 @s3_prefix_option
 @kms_key_id_option
 @role_arn_option
@@ -192,6 +199,7 @@ def cli(
     image_repository,
     image_repositories,
     force_upload,
+    parallel_upload,
     no_progressbar,
     s3_prefix,
     kms_key_id,
@@ -230,6 +238,7 @@ def cli(
         image_repository,
         image_repositories,
         force_upload,
+        parallel_upload,
         no_progressbar,
         s3_prefix,
         kms_key_id,
@@ -267,6 +276,7 @@ def do_cli(
     image_repository,
     image_repositories,
     force_upload,
+    parallel_upload,
     no_progressbar,
     s3_prefix,
     kms_key_id,
@@ -341,6 +351,7 @@ def do_cli(
                 config_file=config_file,
                 disable_rollback=disable_rollback,
                 language_extensions_enabled=language_extensions_enabled,
+                parallel_upload=parallel_upload,
             )
             guided_context.run()
         else:
@@ -379,6 +390,7 @@ def do_cli(
                 kms_key_id=kms_key_id,
                 use_json=use_json,
                 force_upload=force_upload,
+                parallel_upload=guided_context.guided_parallel_upload if guided else parallel_upload,
                 no_progressbar=no_progressbar if output_mode is not OutputOption.json else True,
                 metadata=metadata,
                 on_deploy=True,
@@ -407,6 +419,7 @@ def do_cli(
                 image_repository=guided_context.guided_image_repository if guided else image_repository,
                 image_repositories=guided_context.guided_image_repositories if guided else image_repositories,
                 force_upload=force_upload,
+                parallel_upload=guided_context.guided_parallel_upload if guided else parallel_upload,
                 no_progressbar=no_progressbar,
                 s3_prefix=guided_context.guided_s3_prefix if guided else s3_prefix,
                 kms_key_id=kms_key_id,

samcli/commands/deploy/core/options.py

@@ -37,6 +37,7 @@
     "disable_rollback",
     "on_failure",
     "force_upload",
+    "parallel_upload",
     "max_wait_duration",
     "express",
 ]

samcli/commands/deploy/deploy_context.py

@@ -81,13 +81,15 @@ def __init__(
         language_extensions: Optional[bool] = None,
         express: bool = False,
         output: str = "text",
+        parallel_upload: bool = False,
     ):
         self.template_file = template_file
         self.stack_name = stack_name
         self.s3_bucket = s3_bucket
         self.image_repository = image_repository
         self.image_repositories = image_repositories
         self.force_upload = force_upload
+        self.parallel_upload = parallel_upload
         self.no_progressbar = no_progressbar
         self.s3_prefix = s3_prefix
         self.kms_key_id = kms_key_id
@@ -182,6 +184,7 @@ def run(self):
                 self.signing_profiles,
                 self.use_changeset,
                 self.disable_rollback,
+                self.parallel_upload,
             )
         return self.deploy(
             self.stack_name,

samcli/commands/deploy/guided_context.py

@@ -63,6 +63,7 @@ def __init__(
         config_file=None,
         disable_rollback=None,
         language_extensions_enabled: bool = False,
+        parallel_upload=False,
     ):
         self.template_file = template_file
         self.stack_name = stack_name
@@ -97,6 +98,8 @@ def __init__(
         self.function_provider: Optional[SamFunctionProvider] = None
         self.disable_rollback = disable_rollback
         self._language_extensions_enabled = language_extensions_enabled
+        self.parallel_upload = parallel_upload
+        self.guided_parallel_upload = None
 
     @property
     def guided_capabilities(self):
@@ -207,6 +210,8 @@ def guided_prompts(self, parameter_override_keys):
         self.guided_s3_prefix = stack_name
         self.guided_region = region
         self.guided_profile = self.profile
+        # Not prompted, so scripted guided deploys keep their answer order; the flag value is saved.
+        self.guided_parallel_upload = self.parallel_upload
         self._capabilities = input_capabilities if input_capabilities else default_capabilities
         self._parameter_overrides = (
             input_parameter_overrides if input_parameter_overrides else self.parameter_overrides_from_cmdline
@@ -590,6 +595,7 @@ def run(self):
                 capabilities=self._capabilities,
                 signing_profiles=self.signing_profiles,
                 disable_rollback=self.disable_rollback,
+                parallel_upload=self.guided_parallel_upload,
             )
 
     @staticmethod

samcli/commands/deploy/utils.py

@@ -20,6 +20,7 @@ def print_deploy_args(
     signing_profiles,
     use_changeset,
     disable_rollback,
+    parallel_upload,
 ):
     """
     Print a table of the values that are used during a sam deploy.
@@ -48,6 +49,7 @@ def print_deploy_args(
     :param signing_profiles: Signing profile details which will be used to sign functions/layers
     :param use_changeset: Flag to use or skip the usage of changesets
     :param disable_rollback: Preserve the state of previously provisioned resources when an operation fails.
+    :param parallel_upload: Whether artifact uploads run in parallel prior to deployment.
     """
     _parameters = parameter_overrides.copy()
 
@@ -70,6 +72,7 @@ def print_deploy_args(
     if use_changeset:
         click.echo(f"\tConfirm changeset            : {confirm_changeset}")
     click.echo(f"\tDisable rollback             : {disable_rollback}")
+    click.echo(f"\tParallel uploads             : {parallel_upload}")
     if image_repository:
         msg = "Deployment image repository  : "
         # NOTE(sriram-mv): tab length is 8 spaces.

samcli/commands/package/package_context.py

@@ -86,6 +86,7 @@ def __init__(
         resolve_image_repos=False,
         language_extensions=None,
         output="text",
+        parallel_upload=False,
     ):
         self.template_file = template_file
         self.s3_bucket = s3_bucket
@@ -96,6 +97,7 @@ def __init__(
         self.output_template_file = output_template_file
         self.use_json = use_json
         self.force_upload = force_upload
+        self.parallel_upload = parallel_upload
         self.no_progressbar = no_progressbar
         self.metadata = metadata
         self.region = region
@@ -152,13 +154,15 @@ def run(self):
         # Pass None instead of validating Docker client upfront - ECRUploader will validate only when needed
         docker_client = None
 
+        # Progress bars redraw the terminal in place and interleave badly across threads.
+        no_progressbar = self.no_progressbar or self.parallel_upload
         s3_uploader = S3Uploader(
-            s3_client, self.s3_bucket, self.s3_prefix, self.kms_key_id, self.force_upload, self.no_progressbar
+            s3_client, self.s3_bucket, self.s3_prefix, self.kms_key_id, self.force_upload, no_progressbar
         )
         # attach the given metadata to the artifacts to be uploaded
         s3_uploader.artifact_metadata = self.metadata
         ecr_uploader = ECRUploader(
-            docker_client, ecr_client, self.image_repository, self.image_repositories, self.no_progressbar
+            docker_client, ecr_client, self.image_repository, self.image_repositories, no_progressbar
         )
 
         self.uploaders = Uploaders(s3_uploader, ecr_uploader)
@@ -218,6 +222,7 @@ def _export_without_language_extensions(self, template_path, original_template_d
             normalize_parameters=True,
             template_dict=original_template_dict,
             language_extensions_enabled=False,
+            parallel_upload=self.parallel_upload,
         )
         return template.export()
 
@@ -265,6 +270,7 @@ def _export_with_language_extensions(self, template_path, original_template_dict
             template_dict=copy.deepcopy(result.expanded_template),
             parameter_values=parameter_values,
             language_extensions_enabled=True,
+            parallel_upload=self.parallel_upload,
         )
         exported_template = template.export()