Merge pull request #129 from fujiwara/retry-update-function-code

retry UpdateFunctionCode when ErrCodeResourceConflictException

.github/workflows/go.yml

@@ -38,6 +38,22 @@ jobs:
           make clean
           make test
 
+      - name: create test
+        run: |
+          make
+          ./cmd/lambroll/lambroll deploy \
+            --function test/function.json \
+            --src test/src \
+            --endpoint http://127.0.0.1:4566 \
+            --tfstate test/terraform.tfstate \
+            --log-level debug
+        env:
+          FUNCTION_NAME: hello
+          AWS_REGION: ap-northeast-1
+          AWS_ACCESS_KEY_ID: dummy
+          AWS_SECRET_ACCESS_KEY: dummy
+          WORLD: a
+
       - name: deploy test
         run: |
           make

deploy.go

@@ -2,6 +2,7 @@ package lambroll
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"io/ioutil"
 	"log"
@@ -143,9 +144,9 @@ func (app *App) Deploy(opt DeployOption) error {
 	}
 	log.Printf("[debug]\n%s", codeIn.String())
 
-	res, err := app.lambda.UpdateFunctionCode(codeIn)
+	res, err := app.updateFunctionCodeWithRetry(context.Background(), codeIn)
 	if err != nil {
-		return errors.Wrap(err, "failed to update function code")
+		return err
 	}
 	if res.Version != nil {
 		newerVersion = *res.Version
@@ -160,6 +161,26 @@ func (app *App) Deploy(opt DeployOption) error {
 	return app.updateAliases(*fn.FunctionName, versionAlias{newerVersion, *opt.AliasName})
 }
 
+func (app *App) updateFunctionCodeWithRetry(ctx context.Context, in *lambda.UpdateFunctionCodeInput) (*lambda.FunctionConfiguration, error) {
+	retrier := retryPolicy.Start(ctx)
+	for retrier.Continue() {
+		res, err := app.lambda.UpdateFunctionCode(in)
+		if err != nil {
+			if aerr, ok := err.(awserr.Error); ok {
+				switch aerr.Code() {
+				case lambda.ErrCodeResourceConflictException:
+					log.Println("[debug] retrying", aerr.Message())
+					continue
+				}
+			} else {
+				return nil, errors.Wrap(err, "failed to update function code")
+			}
+		}
+		return res, nil
+	}
+	return nil, errors.New("failed to update function code (max retries)")
+}
+
 func (app *App) updateAliases(functionName string, vs ...versionAlias) error {
 	for _, v := range vs {
 		log.Printf("[info] updating alias set %s to version %s", v.Name, v.Version)

go.mod

@@ -13,4 +13,5 @@ require (
 	github.com/kayac/go-config v0.5.1
 	github.com/kylelemons/godebug v1.1.0
 	github.com/pkg/errors v0.9.1
+	github.com/shogo82148/go-retry v1.0.0
 )

go.sum

@@ -74,6 +74,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/shogo82148/go-retry v1.0.0 h1:c487Qe+QYUffpUpPxrUN5fGJq6WVHSzGS4N3MNuR2OU=
+github.com/shogo82148/go-retry v1.0.0/go.mod h1:5jiw5yPWW6K+pMyimtNoaQSDD08RMEsJbhDwFrui5rc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -105,6 +107,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=

lambroll.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/endpoints"
@@ -14,8 +15,15 @@ import (
 	"github.com/hashicorp/go-envparse"
 	"github.com/kayac/go-config"
 	"github.com/pkg/errors"
+	"github.com/shogo82148/go-retry"
 )
 
+var retryPolicy = retry.Policy{
+	MinDelay: time.Second,
+	MaxDelay: 5 * time.Second,
+	MaxCount: 10,
+}
+
 // Function represents configuration of Lambda function
 type Function = lambda.CreateFunctionInput