From f6447d840834d8598ff9b3dcb10a6a657728f93a Mon Sep 17 00:00:00 2001 From: Greg Lin Date: Mon, 29 Aug 2022 15:53:29 -0500 Subject: [PATCH] Increase ECS_CONTAINER_CREATE_TIMEOUT to 10m to mitigate Docker timeouts Many jobs were failing to start from an error: DockerTimeoutError: Could not transition to created; timed out after waiting 4m0s Try to mitigate this by increasing the Docker timeout from the default 4m to 10m. ECS_CONTAINER_CREATE_TIMEOUT is not documented in the AWS developer guide, but it does seem to be a real setting from the ECS agent repo README: https://github.com/aws/amazon-ecs-agent/blob/a5f8d25f66fbf4f8839d5e7e5fcbc635a4d75c5e/README.md --- serverless-resources.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/serverless-resources.yml b/serverless-resources.yml index d62a9ee..09a69fa 100644 --- a/serverless-resources.yml +++ b/serverless-resources.yml @@ -84,6 +84,11 @@ rBuildsEcsLaunchTemplate: Ebs: VolumeType: gp3 VolumeSize: 96 + UserData: + Fn::Base64: | + #!/bin/bash + # Mitigate DockerTimeoutError errors from the default timeout of 4m + echo "ECS_CONTAINER_CREATE_TIMEOUT=10m" >> /etc/ecs/ecs.config rBuildsBatchComputeEnvironment: Type: AWS::Batch::ComputeEnvironment