diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java index f3dcdfdee..d621d1492 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/CopyWriter.java @@ -97,10 +97,11 @@ public long getTotalBytesCopied() { */ public void copyChunk() { if (!isDone()) { + RewriteRequest rewriteRequest = rewriteResponse.rewriteRequest; this.rewriteResponse = Retrying.run( serviceOptions, - serviceOptions.getRetryAlgorithmManager().getForObjectsCopy(), + serviceOptions.getRetryAlgorithmManager().getForObjectsRewrite(rewriteRequest), () -> storageRpc.continueRewrite(rewriteResponse), Function.identity()); } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java index fd88c3333..f6ba9d381 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/LegacyRetryAlgorithmManager.java @@ -219,14 +219,9 @@ public ExceptionHandler getForObjectsRewrite(RewriteRequest pb) { return BaseService.EXCEPTION_HANDLER; } - @Override - public ExceptionHandler getForObjectsCopy() { - return BaseService.EXCEPTION_HANDLER; - } - @Override public ExceptionHandler getForObjectsCompose( - List sources, StorageObject target, Map targetOptions) { + List sources, StorageObject target, Map optionsMap) { return BaseService.EXCEPTION_HANDLER; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java index 0c3e1f8e7..8f84990f2 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/NewRetryAlgorithmManager.java @@ -16,5 +16,341 @@ package com.google.cloud.storage; -final class NewRetryAlgorithmManager extends LegacyRetryAlgorithmManager - implements RetryAlgorithmManager {} +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.BucketAccessControl; +import com.google.api.services.storage.model.HmacKeyMetadata; +import com.google.api.services.storage.model.ObjectAccessControl; +import com.google.api.services.storage.model.Policy; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.BaseServiceException; +import com.google.cloud.ExceptionHandler; +import com.google.cloud.ExceptionHandler.Interceptor; +import com.google.cloud.storage.spi.v1.StorageRpc; +import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +final class NewRetryAlgorithmManager implements RetryAlgorithmManager { + + private static final Interceptor INTERCEPTOR_IDEMPOTENT = + new InterceptorImpl( + true, + ImmutableSet.builder() + .add(408) + .add(429) + .add(500) + .add(502) + .add(503) + .add(504) + .build()); + private static final Interceptor INTERCEPTOR_IDEMPOTENT_RESUMABLE = + new InterceptorImpl( + true, + ImmutableSet.builder().add(408).add(500).add(502).add(503).add(504).build()); + private static final Interceptor INTERCEPTOR_NON_IDEMPOTENT = + new InterceptorImpl(false, ImmutableSet.builder().build()); + + private static final ExceptionHandler IDEMPOTENT_HANDLER = + ExceptionHandler.newBuilder() + .retryOn(RuntimeException.class) + .addInterceptors(INTERCEPTOR_IDEMPOTENT) + .build(); + + private static final ExceptionHandler IDEMPOTENT_RESUMABLE_UPLOAD_HANDLER = + ExceptionHandler.newBuilder() + .retryOn(RuntimeException.class) + .addInterceptors(INTERCEPTOR_IDEMPOTENT_RESUMABLE) + .build(); + + private static final ExceptionHandler NON_IDEMPOTENT_HANDLER = + ExceptionHandler.newBuilder() + .retryOn(RuntimeException.class) + .addInterceptors(INTERCEPTOR_NON_IDEMPOTENT) + .build(); + + @Override + public ExceptionHandler getForBucketAclCreate( + BucketAccessControl pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclDelete(String pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclGet(String pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclUpdate( + BucketAccessControl pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketAclList(String pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsCreate(Bucket pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsDelete(Bucket pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsGet(Bucket pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsUpdate(Bucket pb, Map optionsMap) { + // TODO: Include etag when it is supported by the library + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsList(Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsLockRetentionPolicy( + Bucket pb, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsGetIamPolicy( + String bucket, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsSetIamPolicy( + String bucket, Policy pb, Map optionsMap) { + // TODO: Include etag when it is supported by the library + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForBucketsTestIamPermissions( + String bucket, List permissions, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclCreate(ObjectAccessControl pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclDelete(String pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclGet(String pb) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclUpdate(ObjectAccessControl pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForDefaultObjectAclList(String pb) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyCreate(String pb, Map optionsMap) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyDelete( + HmacKeyMetadata pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyGet(String accessId, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyUpdate( + HmacKeyMetadata pb, Map optionsMap) { + // TODO: Include etag when it is supported by the library + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForHmacKeyList(Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclCreate(ObjectAccessControl aclPb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclDelete( + String bucket, String name, Long generation, String pb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclList(String bucket, String name, Long generation) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclGet( + String bucket, String name, Long generation, String pb) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectAclUpdate(ObjectAccessControl aclPb) { + return NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCreate( + StorageObject pb, Map optionsMap) { + if (pb.getGeneration() != null && pb.getGeneration() == 0) { + return IDEMPOTENT_HANDLER; + } + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsDelete( + StorageObject pb, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsGet(StorageObject pb, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsUpdate( + StorageObject pb, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsList(String bucket, Map optionsMap) { + return IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsRewrite(RewriteRequest pb) { + return (pb.sourceOptions.containsKey(StorageRpc.Option.IF_SOURCE_GENERATION_MATCH) + || pb.sourceOptions.containsKey(StorageRpc.Option.IF_GENERATION_MATCH)) + && pb.targetOptions.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForObjectsCompose( + List sources, StorageObject target, Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_METAGENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForResumableUploadSessionCreate(Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForResumableUploadSessionWrite(Map optionsMap) { + return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH) + ? IDEMPOTENT_RESUMABLE_UPLOAD_HANDLER + : NON_IDEMPOTENT_HANDLER; + } + + @Override + public ExceptionHandler getForServiceAccountGet(String pb) { + return IDEMPOTENT_HANDLER; + } + + private static class InterceptorImpl implements Interceptor { + + private final boolean idempotent; + private final ImmutableSet retryableCodes; + + private InterceptorImpl(boolean idempotent, ImmutableSet retryableCodes) { + this.idempotent = idempotent; + this.retryableCodes = retryableCodes; + } + + @Override + public RetryResult afterEval(Exception exception, RetryResult retryResult) { + return RetryResult.CONTINUE_EVALUATION; + } + + @Override + public RetryResult beforeEval(Exception exception) { + + // first check if an IO exception has been wrapped by a StorageException, fallback to + // a general BaseServiceException to check status code + if (exception instanceof StorageException) { + StorageException storageException = (StorageException) exception; + Throwable cause = storageException.getCause(); + //noinspection StatementWithEmptyBody + if (cause instanceof GoogleJsonResponseException) { + // this is handled by the case for BaseServiceException below + } else if (cause instanceof IOException) { + IOException ioException = (IOException) cause; + return BaseServiceException.isRetryable(idempotent, ioException) + ? RetryResult.RETRY + : RetryResult.NO_RETRY; + } + } + + if (exception instanceof BaseServiceException) { + int code = ((BaseServiceException) exception).getCode(); + if (retryableCodes.contains(code)) { + return RetryResult.RETRY; + } else { + return RetryResult.NO_RETRY; + } + } + return RetryResult.CONTINUE_EVALUATION; + } + } +} diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java index 884c4d4eb..aa72abe03 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/RetryAlgorithmManager.java @@ -105,10 +105,8 @@ ExceptionHandler getForBucketsTestIamPermissions( ExceptionHandler getForObjectsRewrite(RewriteRequest pb); - ExceptionHandler getForObjectsCopy(); - ExceptionHandler getForObjectsCompose( - List sources, StorageObject target, Map targetOptions); + List sources, StorageObject target, Map optionsMap); ExceptionHandler getForResumableUploadSessionCreate(Map optionsMap); /** Resumable upload has differing 429 handling */ diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java index 0cf60a787..8450df6f6 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageOptions.java @@ -146,7 +146,7 @@ protected StorageRpc getStorageRpcV1() { return (StorageRpc) getRpc(); } - protected RetryAlgorithmManager getRetryAlgorithmManager() { + RetryAlgorithmManager getRetryAlgorithmManager() { return retryAlgorithmManager; } diff --git a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt index 2dcb832dd..c631fba3b 100644 --- a/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt +++ b/google-cloud-storage/src/test/resources/com/google/cloud/storage/conformance/retry/testNamesWhichShouldSucceed.txt @@ -167,12 +167,91 @@ TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.insert-47 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-57 TestRetryConformance/2-[return-reset-connection_return-reset-connection]-storage.objects.patch-80 +TestRetryConformance/3-[return-503]-storage.buckets.patch-17 +TestRetryConformance/3-[return-503]-storage.buckets.setIamPolicy-18 +TestRetryConformance/3-[return-503]-storage.hmacKey.update-29 +TestRetryConformance/3-[return-503]-storage.objects.compose-35 +TestRetryConformance/3-[return-503]-storage.objects.delete-36 +TestRetryConformance/3-[return-503]-storage.objects.delete-67 +TestRetryConformance/3-[return-503]-storage.objects.insert-108 +TestRetryConformance/3-[return-503]-storage.objects.insert-109 TestRetryConformance/3-[return-503]-storage.objects.insert-110 TestRetryConformance/3-[return-503]-storage.objects.insert-111 +TestRetryConformance/3-[return-503]-storage.objects.insert-112 +TestRetryConformance/3-[return-503]-storage.objects.insert-113 TestRetryConformance/3-[return-503]-storage.objects.insert-114 +TestRetryConformance/3-[return-503]-storage.objects.insert-115 +TestRetryConformance/3-[return-503]-storage.objects.insert-116 +TestRetryConformance/3-[return-503]-storage.objects.insert-117 +TestRetryConformance/3-[return-503]-storage.objects.patch-56 +TestRetryConformance/3-[return-503]-storage.objects.patch-79 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-58 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-81 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-82 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-83 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-84 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-85 +TestRetryConformance/3-[return-503]-storage.objects.rewrite-86 +TestRetryConformance/3-[return-reset-connection]-storage.buckets.patch-17 +TestRetryConformance/3-[return-reset-connection]-storage.buckets.setIamPolicy-18 +TestRetryConformance/3-[return-reset-connection]-storage.hmacKey.update-29 +TestRetryConformance/3-[return-reset-connection]-storage.objects.compose-35 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-108 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-109 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-110 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-111 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-112 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-113 TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-114 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-115 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-116 +TestRetryConformance/3-[return-reset-connection]-storage.objects.insert-117 +TestRetryConformance/3-[return-reset-connection]-storage.objects.patch-56 +TestRetryConformance/3-[return-reset-connection]-storage.objects.patch-79 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-58 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-81 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-82 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-83 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-84 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-85 +TestRetryConformance/3-[return-reset-connection]-storage.objects.rewrite-86 +TestRetryConformance/4-[return-503]-storage.bucket_acl.delete-1 +TestRetryConformance/4-[return-503]-storage.bucket_acl.delete-2 +TestRetryConformance/4-[return-503]-storage.bucket_acl.delete-87 +TestRetryConformance/4-[return-503]-storage.bucket_acl.insert-5 +TestRetryConformance/4-[return-503]-storage.bucket_acl.insert-6 +TestRetryConformance/4-[return-503]-storage.bucket_acl.insert-89 +TestRetryConformance/4-[return-503]-storage.bucket_acl.patch-10 +TestRetryConformance/4-[return-503]-storage.bucket_acl.patch-9 +TestRetryConformance/4-[return-503]-storage.bucket_acl.patch-91 +TestRetryConformance/4-[return-503]-storage.default_object_acl.delete-102 +TestRetryConformance/4-[return-503]-storage.default_object_acl.delete-20 +TestRetryConformance/4-[return-503]-storage.default_object_acl.insert-104 +TestRetryConformance/4-[return-503]-storage.default_object_acl.insert-22 +TestRetryConformance/4-[return-503]-storage.default_object_acl.patch-106 +TestRetryConformance/4-[return-503]-storage.default_object_acl.patch-24 +TestRetryConformance/4-[return-503]-storage.hmacKey.create-25 +TestRetryConformance/4-[return-503]-storage.object_acl.delete-30 +TestRetryConformance/4-[return-503]-storage.object_acl.delete-62 +TestRetryConformance/4-[return-503]-storage.object_acl.insert-32 +TestRetryConformance/4-[return-503]-storage.object_acl.insert-64 +TestRetryConformance/4-[return-503]-storage.object_acl.patch-34 +TestRetryConformance/4-[return-503]-storage.object_acl.patch-66 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.insert-5 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.insert-6 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.insert-89 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.patch-10 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.patch-9 +TestRetryConformance/4-[return-reset-connection]-storage.bucket_acl.patch-91 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.insert-104 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.insert-22 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.patch-106 +TestRetryConformance/4-[return-reset-connection]-storage.default_object_acl.patch-24 +TestRetryConformance/4-[return-reset-connection]-storage.hmacKey.create-25 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.insert-32 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.insert-64 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.patch-34 +TestRetryConformance/4-[return-reset-connection]-storage.object_acl.patch-66 TestRetryConformance/5-[return-400]-storage.bucket_acl.delete-1 TestRetryConformance/5-[return-400]-storage.bucket_acl.delete-2 TestRetryConformance/5-[return-400]-storage.bucket_acl.delete-87 @@ -388,7 +467,6 @@ TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-31 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.get-63 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-33 TestRetryConformance/6-[return-503_return-400]-storage.object_acl.list-65 -TestRetryConformance/6-[return-503_return-400]-storage.objects.compose-35 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-37 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-38 TestRetryConformance/6-[return-503_return-400]-storage.objects.delete-68 @@ -412,23 +490,9 @@ TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-118 TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-119 TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-46 TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-47 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-49 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-50 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-51 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-52 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-53 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-77 -TestRetryConformance/6-[return-503_return-400]-storage.objects.insert-78 TestRetryConformance/6-[return-503_return-400]-storage.objects.list-55 TestRetryConformance/6-[return-503_return-400]-storage.objects.patch-57 TestRetryConformance/6-[return-503_return-400]-storage.objects.patch-80 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-58 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-81 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-82 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-83 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-84 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-85 -TestRetryConformance/6-[return-503_return-400]-storage.objects.rewrite-86 TestRetryConformance/6-[return-503_return-400]-storage.serviceaccount.get-59 TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-3 TestRetryConformance/6-[return-reset-connection_return-401]-storage.bucket_acl.get-4 @@ -459,7 +523,6 @@ TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.g TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.get-63 TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-33 TestRetryConformance/6-[return-reset-connection_return-401]-storage.object_acl.list-65 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.compose-35 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-37 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-38 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.delete-68 @@ -486,18 +549,8 @@ TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.inse TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-49 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-50 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-51 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-52 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-53 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-77 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.insert-78 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.list-55 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.patch-57 TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.patch-80 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-58 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-81 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-82 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-83 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-84 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-85 -TestRetryConformance/6-[return-reset-connection_return-401]-storage.objects.rewrite-86 TestRetryConformance/6-[return-reset-connection_return-401]-storage.serviceaccount.get-59