Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(bigtable): Client side merics #10046

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+2,076 −54
Open

feat(bigtable): Client side merics #10046

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e6c4fd6
feat(bigtable): Built-in client side metrics
bhshkh May 31, 2024
0accc87
refactor(bigtable): refactored code, corrected units and method names
bhshkh Jun 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
209 changes: 164 additions & 45 deletions bigtable/bigtable.go
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 3 additions & 2 deletions bigtable/bigtable_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,8 +253,9 @@ func TestApplyErrors(t *testing.T) {
ctx := context.Background()
table := &Table{
c: &Client{
project: "P",
instance: "I",
project: "P",
instance: "I",
metricsTracerFactory: &builtinMetricsTracerFactory{},
},
table: "t",
}
Expand Down
15 changes: 10 additions & 5 deletions bigtable/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ require (
github.com/google/go-cmp v0.6.0
github.com/googleapis/cloud-bigtable-clients-test v0.0.2
github.com/googleapis/gax-go/v2 v2.12.4
go.opentelemetry.io/otel v1.24.0 // Use older version compatible with Go 1.20
go.opentelemetry.io/otel/metric v1.24.0 // Use older version compatible with Go 1.20
go.opentelemetry.io/otel/sdk v1.24.0 // Use older version compatible with Go 1.20
go.opentelemetry.io/otel/sdk/metric v1.24.0 // Use older version compatible with Go 1.20
google.golang.org/api v0.182.0
google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6
google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157
Expand All @@ -18,6 +22,12 @@ require (
rsc.io/binaryregexp v0.2.0
)

require (
cloud.google.com/go/monitoring v1.18.2
github.com/google/uuid v1.6.0
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8
)

require (
cloud.google.com/go/auth v0.4.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
Expand All @@ -33,14 +43,10 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/metric v1.24.0 // indirect
go.opentelemetry.io/otel/sdk v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/net v0.25.0 // indirect
Expand All @@ -49,5 +55,4 @@ require (
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/time v0.5.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect
)
4 changes: 4 additions & 0 deletions bigtable/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0=
cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE=
cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU=
cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng=
cloud.google.com/go/monitoring v1.18.2 h1:nIQdZdf1/9M0cEmXSlLB2jMq/k3CRh9p3oUzS06VDG8=
cloud.google.com/go/monitoring v1.18.2/go.mod h1:MuL95M6d9HtXQOaWP9JxhFZJKP+fdTF0Gt5xl4IDsew=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g=
Expand Down Expand Up @@ -98,6 +100,8 @@ go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGX
go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8=
go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0=
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
Expand Down
63 changes: 63 additions & 0 deletions bigtable/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ import (
"cloud.google.com/go/internal/optional"
"cloud.google.com/go/internal/testutil"
"cloud.google.com/go/internal/uid"
monitoring "cloud.google.com/go/monitoring/apiv3/v2"
"cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
"github.com/google/go-cmp/cmp"
gax "github.com/googleapis/gax-go/v2"
"google.golang.org/api/iterator"
Expand All @@ -46,6 +48,7 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/timestamppb"
)

const (
Expand Down Expand Up @@ -274,6 +277,7 @@ func TestIntegration_ReadRowList(t *testing.T) {
t.Fatalf("bulk read: wrong reads.\n got %q\nwant %q", got, want)
}
}

func TestIntegration_ReadRowListReverse(t *testing.T) {
ctx := context.Background()
_, _, _, table, _, cleanup, err := setupIntegration(ctx, t)
Expand Down Expand Up @@ -748,6 +752,18 @@ func TestIntegration_HighlyConcurrentReadsAndWrites(t *testing.T) {

func TestIntegration_LargeReadsWritesAndScans(t *testing.T) {
ctx := context.Background()
origSamplePeriod := defaultSamplePeriod
defaultSamplePeriod = time.Minute
defer func() {
defaultSamplePeriod = origSamplePeriod
}()
testStartTime := time.Now()

tsListStart := &timestamppb.Timestamp{
Seconds: testStartTime.Unix(),
Nanos: int32(testStartTime.Nanosecond()),
}

testEnv, _, adminClient, table, tableName, cleanup, err := setupIntegration(ctx, t)
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -899,6 +915,53 @@ func TestIntegration_LargeReadsWritesAndScans(t *testing.T) {
} else if status[0] == nil || status[1] == nil {
t.Fatalf("No error for bad bulk mutation")
}

if !testEnv.Config().UseProd {
t.Logf("Skipping metrics validation while using emulator")
return
}

// Validate that metrics are exported
elapsedTime := time.Since(testStartTime)
if elapsedTime < 2*defaultSamplePeriod {
// Ensure at least 2 datapoints are recorded
time.Sleep(2*defaultSamplePeriod - elapsedTime)
}

timeListEnd := time.Now()
tsListEnd := &timestamppb.Timestamp{
Seconds: timeListEnd.Unix(),
Nanos: int32(timeListEnd.Nanosecond()),
}

monitoringClient, err := monitoring.NewMetricClient(ctx)
if err != nil {
t.Errorf("Failed to create metric client: %v", err)
}

metricNamesValidate := []string{
metricNameOperationLatencies,
metricNameAttemptLatencies,
metricNameServerLatencies,
}
for _, metricName := range metricNamesValidate {
// ListTimeSeries can list only one metric type at a time.
// So, call ListTimeSeries with different metric names
iter := monitoringClient.ListTimeSeries(ctx, &monitoringpb.ListTimeSeriesRequest{
Name: fmt.Sprintf("projects/%s", testEnv.Config().Project),
Interval: &monitoringpb.TimeInterval{
StartTime: tsListStart,
EndTime: tsListEnd,
},
Filter: fmt.Sprintf("metric.type = starts_with(\"bigtable.googleapis.com/client/%v\")", metricName),
})

// Assert at least 1 datapoint was exported
_, err := iter.Next()
if err != nil {
t.Errorf("%v not exported\n", metricName)
}
}
}

func TestIntegration_Read(t *testing.T) {
Expand Down