diff --git a/wdl/HardCloneWithExternalBucket/HardCloneWithExternalBucket.wdl b/wdl/HardCloneWithExternalBucket/HardCloneWithExternalBucket.wdl index 48c3e03..9013632 100644 --- a/wdl/HardCloneWithExternalBucket/HardCloneWithExternalBucket.wdl +++ b/wdl/HardCloneWithExternalBucket/HardCloneWithExternalBucket.wdl @@ -88,7 +88,7 @@ task HardCloneWithExternalBucketTask { ~{if metadata_only then "--metadata_only" else ""} \ ~{if do_not_update_acls then "--do_not_update_acls" else ""} \ ~{if check_and_wait_for_permissions then "--check_and_wait_for_permissions" else ""} \ - ~{"--max_permissions_wait_time " + max_permissions_wait_time} \ + ~{"--max_permissions_wait_time " + max_permissions_wait_time} >>> output { diff --git a/wdl/HardCloneWithExternalBucket/README.md b/wdl/HardCloneWithExternalBucket/README.md index 7a02f2c..00d70d1 100644 --- a/wdl/HardCloneWithExternalBucket/README.md +++ b/wdl/HardCloneWithExternalBucket/README.md @@ -19,12 +19,12 @@ This WDL script creates a new workspace that is nearly identical to the source w | **allow_already_created** | If `true`, allows the script to proceed without failing if the destination workspace already exists. | Boolean | Yes | N/A | | **rsync_workspace** | If you would like to use rsync for copy instead of gcloud libraries. Can be quicker if large copies. | Boolean | Yes | N/A | | **do_not_update_acls** | If you want to skip updating ACLs to match source workspace. If you don't have OWNER access of source workspace you HAVE to use this option or it will fail. | Boolean | Yes | N/A | +| **check_and_wait_for_permissions** | When used, workflow will check write permissions on destination bucket every 30 minutes for 5 hours total before exiting. Useful for when permissions were newly added and could take some time to propagate | Boolean | Yes | N/A | | **workers** | The number of worker threads to use for the file transfer. Only used if not rsyncing. Optional. | Int | No | 10 | | **extensions_to_ignore** | A comma-separated list of file extensions to ignore during the file copy process. Only used if not rsyncing. Optional. Do not include spaces (i.e. ".txt,.tsv") | String | No | N/A | | **docker** | Specifies a custom Docker image to use. Optional. | String | No | us-central1-docker.pkg.dev/operations-portal-427515/ops-toolbox/ops_terra_utils_slim:latest | | **memory_gb** | How much memory given to task | Int | No | 8 | | **batch_size** | Can be used if you want to batch up how much files at a time to copy over. If not used will do it all in one batch. Only used if not rsyncing | Int | No | N/A | -| **check_and_wait_for_permissions** | When used, workflow will check write permissions on destination bucket every 30 minutes for 5 hours total before exiting. Useful for when permissions were newly added and could take some time to propagate | Boolean | Yes | N/A | | **max_permissions_wait_time** | Optional total time to wait for permissions to propagate. Defaults to 5 hours. Won't run for more than 5 hours total. | Int | No | 5 | diff --git a/wdl/HardCloneWorkspace/HardCloneWorkspace.wdl b/wdl/HardCloneWorkspace/HardCloneWorkspace.wdl index 7d1dff6..ac7c77f 100644 --- a/wdl/HardCloneWorkspace/HardCloneWorkspace.wdl +++ b/wdl/HardCloneWorkspace/HardCloneWorkspace.wdl @@ -16,6 +16,8 @@ workflow HardCloneTerraWorkspace { String? docker Int? memory_gb Int? batch_size + Boolean check_and_wait_for_permissions + Int? max_permissions_wait_time } String docker_name = select_first([docker, "us-central1-docker.pkg.dev/operations-portal-427515/ops-toolbox/ops_terra_utils_slim:latest"]) @@ -36,7 +38,9 @@ workflow HardCloneTerraWorkspace { memory_gb=memory, batch_size=batch_size, metadata_only=rsync_workspace, - do_not_update_acls=do_not_update_acls + do_not_update_acls=do_not_update_acls, + check_and_wait_for_permissions=check_and_wait_for_permissions, + max_permissions_wait_time=max_permissions_wait_time } if (rsync_workspace) { @@ -63,6 +67,8 @@ task HardCloneTerraWorkspaceTask { Boolean metadata_only Boolean do_not_update_acls Int? batch_size + Boolean check_and_wait_for_permissions + Int? max_permissions_wait_time } command <<< @@ -76,7 +82,9 @@ task HardCloneTerraWorkspaceTask { ~{"--extensions_to_ignore " + extensions_to_ignore} \ ~{"--batch_size " + batch_size} \ ~{if metadata_only then "--metadata_only" else ""} \ - ~{if do_not_update_acls then "--do_not_update_acls" else ""} + ~{if do_not_update_acls then "--do_not_update_acls" else ""} \ + ~{if check_and_wait_for_permissions then "--check_and_wait_for_permissions" else ""} \ + ~{"--max_permissions_wait_time " + max_permissions_wait_time} >>> output { diff --git a/wdl/HardCloneWorkspace/README.md b/wdl/HardCloneWorkspace/README.md index 90126dd..10b4e31 100644 --- a/wdl/HardCloneWorkspace/README.md +++ b/wdl/HardCloneWorkspace/README.md @@ -6,20 +6,23 @@ This WDL script creates a new workspace that is nearly identical to the source w ## Inputs Table: -| Input Name | Description | Type | Required | Default | -|----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|----------|---------------------------------------------------------------------------------------------| -| **source_billing_project** | The billing project for the source workspace. | String | Yes | N/A | -| **source_workspace_name** | The name of the source workspace that will be copied. | String | Yes | N/A | -| **dest_billing_project** | The billing project for the new destination workspace. | String | Yes | N/A | -| **dest_workspace_name** | The name of the new workspace to be created. | String | Yes | N/A | -| **allow_already_created** | If `true`, allows the script to proceed without failing if the destination workspace already exists. | Boolean | Yes | N/A | -| **rsync_workspace** | If you would like to use rsync for copy instead of gcloud libraries. Can be quicker if large copies. | Boolean | Yes | N/A | -| **do_not_update_acls** | If you want to skip updating ACLs to match source workspace. If you don't have OWNER access of source workspace you HAVE to use this option or it will fail. | Boolean | Yes | N/A | -| **workers** | The number of worker threads to use for the file transfer. Only used if not rsyncing. Optional. | Int | No | 10 | -| **extensions_to_ignore** | A comma-separated list of file extensions to ignore during the file copy process. Only used if not rsyncing. Optional. Do not include spaces (i.e. ".txt,.tsv") | String | No | N/A | -| **docker** | Specifies a custom Docker image to use. Optional. | String | No | us-central1-docker.pkg.dev/operations-portal-427515/ops-toolbox/ops_terra_utils_slim:latest | -| **memory_gb** | How much memory given to task | Int | No | 8 | -| **batch_size** | Can be used if you want to batch up how much files at a time to copy over. If not used will do it all in one batch. Only used if not rsyncing | Int | No | N/A | +| Input Name | Description | Type | Required | Default | +|------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|----------|-----------------------------------------------------------------------------------------------| +| **source_billing_project** | The billing project for the source workspace. | String | Yes | N/A | +| **source_workspace_name** | The name of the source workspace that will be copied. | String | Yes | N/A | +| **dest_billing_project** | The billing project for the new destination workspace. | String | Yes | N/A | +| **dest_workspace_name** | The name of the new workspace to be created. | String | Yes | N/A | +| **allow_already_created** | If `true`, allows the script to proceed without failing if the destination workspace already exists. | Boolean | Yes | N/A | +| **rsync_workspace** | If you would like to use rsync for copy instead of gcloud libraries. Can be quicker if large copies. | Boolean | Yes | N/A | +| **do_not_update_acls** | If you want to skip updating ACLs to match source workspace. If you don't have OWNER access of source workspace you HAVE to use this option or it will fail. | Boolean | Yes | N/A | +| **check_and_wait_for_permissions** | When used, workflow will check write permissions on destination bucket every 30 minutes for 5 hours total before exiting. Useful for when permissions were newly added and could take some time to propagate | Boolean | Yes | N/A | +| **workers** | The number of worker threads to use for the file transfer. Only used if not rsyncing. Optional. | Int | No | 10 | +| **extensions_to_ignore** | A comma-separated list of file extensions to ignore during the file copy process. Only used if not rsyncing. Optional. Do not include spaces (i.e. ".txt,.tsv") | String | No | N/A | +| **docker** | Specifies a custom Docker image to use. Optional. | String | No | us-central1-docker.pkg.dev/operations-portal-427515/ops-toolbox/ops_terra_utils_slim:latest | +| **memory_gb** | How much memory given to task | Int | No | 8 | +| **batch_size** | Can be used if you want to batch up how much files at a time to copy over. If not used will do it all in one batch. Only used if not rsyncing | Int | No | N/A | +| **max_permissions_wait_time** | Optional total time to wait for permissions to propagate. Defaults to 5 hours. Won't run for more than 5 hours total. | Int | No | 5 | + ## Outputs Table: This script does not generate direct outputs. However, logs will track the progress of workspace creation, file transfer, and metadata updates. The logs, which include any errors or warnings, can be reviewed in the stderr file for additional details about the process.