feat(compute/metadata): retry error when talking to metadata service

compute/metadata/metadata.go

@@ -32,6 +32,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/googleapis/gax-go/v2"
 )
 
 const (
@@ -304,9 +306,21 @@ func (c *Client) getETag(suffix string) (value, etag string, err error) {
 	}
 	req.Header.Set("Metadata-Flavor", "Google")
 	req.Header.Set("User-Agent", userAgent)
-	res, err := c.hc.Do(req)
-	if err != nil {
-		return "", "", err
+	var res *http.Response
+	retryer := newRetryer()
+	for {
+		var err error
+		res, err = c.hc.Do(req)
+		if err != nil {
+			if delay, shouldRetry :=retryer.Retry(res.StatusCode, err); shouldRetry {
+				if err := gax.Sleep(context.Background(), delay); err != nil {
+					return "", "", err
+				}
+				continue
+			}
+			return "", "", err
+		}
+		break
 	}
 	defer res.Body.Close()
 	if res.StatusCode == http.StatusNotFound {

compute/metadata/retry.go

@@ -0,0 +1,73 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadata
+
+import (
+	"io"
+	"time"
+
+	"github.com/googleapis/gax-go/v2"
+)
+
+var (
+	syscallRetryable = func(err error) bool { return false }
+)
+
+func newRetryer() *metadataRetryer {
+	return &metadataRetryer{bo: &gax.Backoff{Initial: 100 * time.Millisecond}}
+}
+
+type backoff interface {
+	Pause() time.Duration
+}
+
+type metadataRetryer struct {
+	bo       backoff
+	attempts int
+}
+
+func (r *metadataRetryer) Retry(status int, err error) (time.Duration, bool) {
+	retryOk := shouldRetry(status, err)
+	if !retryOk {
+		return 0, false
+	}
+	r.attempts++
+	if r.attempts == 6 {
+		return 0, false
+	}
+	return r.bo.Pause(), true
+}
+
+func shouldRetry(status int, err error) bool {
+	if 500 <= status && status <= 599 {
+		return true
+	}
+	if err == io.ErrUnexpectedEOF {
+		return true
+	}
+	// Transient network errors should be retried.
+	if syscallRetryable(err) {
+		return true
+	}
+	if err, ok := err.(interface{ Temporary() bool }); ok {
+		if err.Temporary() {
+			return true
+		}
+	}
+	if err, ok := err.(interface{ Unwrap() error }); ok {
+		return shouldRetry(status, err.Unwrap())
+	}
+	return false
+}

compute/metadata/retry_linux.go

@@ -0,0 +1,25 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package metadata
+
+import "syscall"
+
+func init() {
+	// Initialize syscallRetryable to return true on transient socket-level
+	// errors. These errors are specific to Linux.
+	syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED }
+}

compute/metadata/retry_linux_test.go

@@ -0,0 +1,39 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package metadata
+
+import (
+	"syscall"
+	"testing"
+)
+
+func TestMetadataRetryerLinux(t *testing.T) {
+	retryer := metadataRetryer{bo: constantBackoff{}}
+
+	t.Run("retry on syscall.ECONNRESET", func(t *testing.T) {
+		delay, shouldRetry := retryer.Retry(400, syscall.ECONNRESET)
+		if !shouldRetry {
+			t.Fatal("retryer.Retry(500, nil) = false, want true")
+		}
+	})
+	t.Run("retry on syscall.ECONNREFUSED", func(t *testing.T) {
+		delay, shouldRetry := retryer.Retry(400, syscall.ECONNREFUSED)
+		if !shouldRetry {
+			t.Fatal("retryer.Retry(500, nil) = false, want true")
+		}
+	})
+}

compute/metadata/retry_test.go

@@ -0,0 +1,108 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadata
+
+import (
+	"io"
+	"testing"
+	"time"
+)
+
+type constantBackoff struct{}
+
+func (b constantBackoff) Pause() time.Duration { return 100 }
+
+type errTemp struct{}
+
+func (e errTemp) Error() string { return "temporary error" }
+
+func (e errTemp) Temporary() bool { return true }
+
+type errWrapped struct {
+	e error
+}
+
+func (e errWrapped) Error() string { return "unwrap me to get more context" }
+
+func (e errWrapped) Unwrap() error { return e.e }
+
+func TestMetadataRetryer(t *testing.T) {
+	t.Run("retry on 500", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		delay, shouldRetry := retryer.Retry(500, nil)
+		if !shouldRetry {
+			t.Fatal("retryer.Retry(500, nil) = false, want true")
+		}
+		if delay != 100 {
+			t.Fatalf("retryer.Retry(500, nil) = %d, want 100", delay)
+		}
+	})
+	t.Run("don't retry 400", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		delay, shouldRetry := retryer.Retry(400, io.EOF)
+		if shouldRetry {
+			t.Fatal("retryer.Retry(400, io.EOF) = true, want false")
+		}
+		if delay != 0 {
+			t.Fatalf("retryer.Retry(400, io.EOF) = %d, want 0", delay)
+		}
+	})
+	t.Run("retry on io.ErrUnexpectedEOF", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		_, shouldRetry := retryer.Retry(400, io.ErrUnexpectedEOF)
+		if !shouldRetry {
+			t.Fatal("retryer.Retry(400, io.ErrUnexpectedEOF) = false, want true")
+		}
+	})
+	t.Run("retry on temporary error", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		err := errTemp{}
+		_, shouldRetry := retryer.Retry(400, err)
+		if !shouldRetry {
+			t.Fatal("retryer.Retry(400, err) = false, want true")
+		}
+	})
+	t.Run("retry on wrapped temporary error", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		err := errWrapped{errTemp{}}
+		_, shouldRetry := retryer.Retry(400, err)
+		if !shouldRetry {
+			t.Fatal("retryer.Retry(400, err) = false, want true")
+		}
+	})
+	t.Run("don't retry on wrapped io.EOF", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		err := errWrapped{io.EOF}
+		_, shouldRetry := retryer.Retry(400, err)
+		if shouldRetry {
+			t.Fatal("retryer.Retry(400, err) = true, want false")
+		}
+	})
+	t.Run("stop retry after 5 attempts", func(t *testing.T) {
+		retryer := metadataRetryer{bo: constantBackoff{}}
+		for i := 1; i <= 6; i++ {
+			_, shouldRetry := retryer.Retry(500, nil)
+			if i == 6 {
+				if shouldRetry {
+					t.Fatal("an error should only be retried 5 times")
+				}
+				break
+			}
+			if !shouldRetry {
+				t.Fatalf("retryer.Retry(500, nil) = false, want true")
+			}
+		}
+	})
+}