Skip to content

Commit

Permalink
chore: better error message when metric publish failed (#2214)
Browse files Browse the repository at this point in the history
Also reduces logging when there are continuous failures (like permission denied).

Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly:
- [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigtable/issues/new/choose) before writing your code!  That way we can discuss the change, evaluate designs, and agree on the general idea
- [ ] Ensure the tests and linter pass
- [ ] Code coverage does not decrease (if any source code was changed)
- [ ] Appropriate docs were updated (if necessary)
- [ ] Rollback plan is reviewed and LGTMed
- [ ] All new data plane features have a completed end to end testing plan

Fixes #<issue_number_goes_here> ☕️

If you write sample code, please follow the [samples format](
https://togithub.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md).
  • Loading branch information
mutianf committed Apr 26, 2024
1 parent 38043ba commit 4d92e1c
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 9 deletions.
2 changes: 1 addition & 1 deletion README.md
Expand Up @@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>26.37.0</version>
<version>26.25.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
Expand Down
Expand Up @@ -34,6 +34,7 @@
import com.google.api.gax.core.CredentialsProvider;
import com.google.api.gax.core.FixedCredentialsProvider;
import com.google.api.gax.core.NoCredentialsProvider;
import com.google.api.gax.rpc.PermissionDeniedException;
import com.google.auth.Credentials;
import com.google.cloud.monitoring.v3.MetricServiceClient;
import com.google.cloud.monitoring.v3.MetricServiceSettings;
Expand Down Expand Up @@ -96,6 +97,9 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter {

private CompletableResultCode lastExportCode;

private final AtomicBoolean bigtableExportFailureLogged = new AtomicBoolean(false);
private final AtomicBoolean applicationExportFailureLogged = new AtomicBoolean(false);

private static final ImmutableList<String> BIGTABLE_TABLE_METRICS =
ImmutableSet.of(
OPERATION_LATENCIES_NAME,
Expand Down Expand Up @@ -227,15 +231,24 @@ private CompletableResultCode exportBigtableResourceMetrics(Collection<MetricDat
new ApiFutureCallback<Empty>() {
@Override
public void onFailure(Throwable throwable) {
logger.log(
Level.WARNING,
"createServiceTimeSeries request failed for bigtable metrics. ",
throwable);
if (bigtableExportFailureLogged.compareAndSet(false, true)) {
String msg = "createServiceTimeSeries request failed for bigtable metrics.";
if (throwable instanceof PermissionDeniedException) {
msg +=
String.format(
" Need monitoring metric writer permission on project=%s. Follow https://cloud.google.com/bigtable/docs/client-side-metrics-setup to set up permissions.",
projectName.getProject());
}
logger.log(Level.WARNING, msg, throwable);
}
bigtableExportCode.fail();
}

@Override
public void onSuccess(Empty empty) {
// When an export succeeded reset the export failure flag to false so if there's a
// transient failure it'll be logged.
bigtableExportFailureLogged.set(false);
bigtableExportCode.succeed();
}
},
Expand Down Expand Up @@ -295,15 +308,24 @@ private CompletableResultCode exportApplicationResourceMetrics(
new ApiFutureCallback<Empty>() {
@Override
public void onFailure(Throwable throwable) {
logger.log(
Level.WARNING,
"createServiceTimeSeries request failed for per connection error metrics.",
throwable);
if (applicationExportFailureLogged.compareAndSet(false, true)) {
String msg = "createServiceTimeSeries request failed for bigtable metrics.";
if (throwable instanceof PermissionDeniedException) {
msg +=
String.format(
" Need monitoring metric writer permission on project=%s. Follow https://cloud.google.com/bigtable/docs/client-side-metrics-setup to set up permissions.",
projectName.getProject());
}
logger.log(Level.WARNING, msg, throwable);
}
exportCode.fail();
}

@Override
public void onSuccess(Empty empty) {
// When an export succeeded reset the export failure flag to false so if there's a
// transient failure it'll be logged.
applicationExportFailureLogged.set(false);
exportCode.succeed();
}
},
Expand Down

0 comments on commit 4d92e1c

Please sign in to comment.