Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add support for Dataset property maxTimeTravelHours #3015

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
Expand Up @@ -53,20 +53,20 @@ If you are using Maven without the BOM, add this to your dependencies:
If you are using Gradle 5.x or later, add this to your dependencies:

```Groovy
implementation platform('com.google.cloud:libraries-bom:26.25.0')
implementation platform('com.google.cloud:libraries-bom:26.27.0')

implementation 'com.google.cloud:google-cloud-bigquery'
```
If you are using Gradle without BOM, add this to your dependencies:

```Groovy
implementation 'com.google.cloud:google-cloud-bigquery:2.33.2'
implementation 'com.google.cloud:google-cloud-bigquery:2.34.2'
```

If you are using SBT, add this to your dependencies:

```Scala
libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.33.2"
libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.34.2"
```
<!-- {x-version-update-end} -->

Expand Down Expand Up @@ -351,7 +351,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
[kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquery/java11.html
[stability-image]: https://img.shields.io/badge/stability-stable-green
[maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquery.svg
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.33.2
[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquery/2.34.2
[authentication]: https://github.com/googleapis/google-cloud-java#authentication
[auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
[predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles
Expand Down
5 changes: 5 additions & 0 deletions google-cloud-bigquery/clirr-ignored-differences.xml
Expand Up @@ -119,4 +119,9 @@
<className>com/google/cloud/bigquery/StandardTableDefinition*</className>
<method>*BigLakeConfiguration(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/DatasetInfo*</className>
<method>*setMaxTimeTravelHours(*)</method>
</difference>
</differences>
Expand Up @@ -164,6 +164,12 @@ public Builder setStorageBillingModel(String storageBillingModel) {
return this;
}

@Override
public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) {
infoBuilder.setMaxTimeTravelHours(maxTimeTravelHours);
return this;
}

@Override
public Dataset build() {
return new Dataset(bigquery, infoBuilder);
Expand Down
Expand Up @@ -75,6 +75,7 @@ public Dataset apply(DatasetInfo datasetInfo) {
private final String defaultCollation;
private final ExternalDatasetReference externalDatasetReference;
private final String storageBillingModel;
private final Long maxTimeTravelHours;

/** A builder for {@code DatasetInfo} objects. */
public abstract static class Builder {
Expand Down Expand Up @@ -142,6 +143,13 @@ public abstract Builder setExternalDatasetReference(
*/
public abstract Builder setStorageBillingModel(String storageBillingModel);

/**
* Optional. Defines the time travel window in hours. The value can be from 48 to 168 hours (2
* to 7 days). Value must in multiple of 24 hours (48, 72, 96, 120, 144, 168) The default value
* is 168 hours if this is not set.
*/
public abstract Builder setMaxTimeTravelHours(Long maxTimeTravelHours);

/**
* The default encryption key for all tables in the dataset. Once this property is set, all
* newly-created partitioned tables in the dataset will have encryption key set to this value,
Expand Down Expand Up @@ -200,6 +208,7 @@ static final class BuilderImpl extends Builder {
private String defaultCollation;
private ExternalDatasetReference externalDatasetReference;
private String storageBillingModel;
private Long maxTimeTravelHours;

BuilderImpl() {}

Expand All @@ -221,6 +230,7 @@ static final class BuilderImpl extends Builder {
this.defaultCollation = datasetInfo.defaultCollation;
this.externalDatasetReference = datasetInfo.externalDatasetReference;
this.storageBillingModel = datasetInfo.storageBillingModel;
this.maxTimeTravelHours = datasetInfo.maxTimeTravelHours;
}

BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) {
Expand Down Expand Up @@ -260,6 +270,7 @@ public Acl apply(Dataset.Access accessPb) {
ExternalDatasetReference.fromPb(datasetPb.getExternalDatasetReference());
}
this.storageBillingModel = datasetPb.getStorageBillingModel();
this.maxTimeTravelHours = datasetPb.getMaxTimeTravelHours();
}

@Override
Expand Down Expand Up @@ -372,6 +383,12 @@ public Builder setStorageBillingModel(String storageBillingModel) {
return this;
}

@Override
public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) {
this.maxTimeTravelHours = maxTimeTravelHours;
return this;
}

@Override
public DatasetInfo build() {
return new DatasetInfo(this);
Expand All @@ -396,6 +413,7 @@ public DatasetInfo build() {
defaultCollation = builder.defaultCollation;
externalDatasetReference = builder.externalDatasetReference;
storageBillingModel = builder.storageBillingModel;
maxTimeTravelHours = builder.maxTimeTravelHours;
}

/** Returns the dataset identity. */
Expand Down Expand Up @@ -529,6 +547,10 @@ public String getStorageBillingModel() {
return storageBillingModel;
}

public Long getMaxTimeTravelHours() {
return maxTimeTravelHours;
}

/**
* Returns information about the external metadata storage where the dataset is defined. Filled
* out when the dataset type is EXTERNAL.
Expand Down Expand Up @@ -562,6 +584,7 @@ public String toString() {
.add("defaultCollation", defaultCollation)
.add("externalDatasetReference", externalDatasetReference)
.add("storageBillingModel", storageBillingModel)
.add("maxTimeTravelHours", maxTimeTravelHours)
.toString();
}

Expand Down Expand Up @@ -646,6 +669,9 @@ public Dataset.Access apply(Acl acl) {
if (storageBillingModel != null) {
datasetPb.setStorageBillingModel(storageBillingModel);
}
if (maxTimeTravelHours != null) {
datasetPb.setMaxTimeTravelHours(maxTimeTravelHours);
}
return datasetPb;
}

Expand Down
Expand Up @@ -59,6 +59,7 @@ public class DatasetInfoTest {
private static final EncryptionConfiguration DATASET_ENCRYPTION_CONFIGURATION =
EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build();
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
private static final Long MAX_TIME_TRAVEL_HOURS = 48L;

private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE =
ExternalDatasetReference.newBuilder()
Expand All @@ -81,6 +82,7 @@ public class DatasetInfoTest {
.setDefaultEncryptionConfiguration(DATASET_ENCRYPTION_CONFIGURATION)
.setDefaultPartitionExpirationMs(DEFAULT_PARTITION__EXPIRATION)
.setStorageBillingModel(STORAGE_BILLING_MODEL)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.build();
private static final DatasetInfo DATASET_INFO_COMPLETE =
DATASET_INFO
Expand Down Expand Up @@ -173,6 +175,7 @@ public void testBuilder() {
EXTERNAL_DATASET_REFERENCE,
DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.getExternalDatasetReference());
assertEquals(STORAGE_BILLING_MODEL, DATASET_INFO_COMPLETE.getStorageBillingModel());
assertEquals(MAX_TIME_TRAVEL_HOURS, DATASET_INFO_COMPLETE.getMaxTimeTravelHours());
}

@Test
Expand All @@ -194,6 +197,7 @@ public void testOf() {
assertTrue(datasetInfo.getLabels().isEmpty());
assertNull(datasetInfo.getExternalDatasetReference());
assertNull(datasetInfo.getStorageBillingModel());
assertNull(datasetInfo.getMaxTimeTravelHours());

datasetInfo = DatasetInfo.of(DATASET_ID);
assertEquals(DATASET_ID, datasetInfo.getDatasetId());
Expand All @@ -212,6 +216,7 @@ public void testOf() {
assertTrue(datasetInfo.getLabels().isEmpty());
assertNull(datasetInfo.getExternalDatasetReference());
assertNull(datasetInfo.getStorageBillingModel());
assertNull(datasetInfo.getMaxTimeTravelHours());
}

@Test
Expand Down Expand Up @@ -249,5 +254,6 @@ private void compareDatasets(DatasetInfo expected, DatasetInfo value) {
expected.getDefaultPartitionExpirationMs(), value.getDefaultPartitionExpirationMs());
assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference());
assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel());
assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours());
}
}
Expand Up @@ -67,6 +67,7 @@ public class DatasetTest {
private static final DatasetInfo DATASET_INFO = DatasetInfo.newBuilder(DATASET_ID).build();
private static final Field FIELD = Field.of("FieldName", LegacySQLTypeName.INTEGER);
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
private static final Long MAX_TIME_TRAVEL_HOURS = 48L;
private static final StandardTableDefinition TABLE_DEFINITION =
StandardTableDefinition.of(Schema.of(FIELD));
private static final ViewDefinition VIEW_DEFINITION = ViewDefinition.of("QUERY");
Expand Down Expand Up @@ -122,6 +123,7 @@ public void testBuilder() {
.setSelfLink(SELF_LINK)
.setLabels(LABELS)
.setStorageBillingModel(STORAGE_BILLING_MODEL)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.build();
assertEquals(DATASET_ID, builtDataset.getDatasetId());
assertEquals(ACCESS_RULES, builtDataset.getAcl());
Expand All @@ -136,6 +138,7 @@ public void testBuilder() {
assertEquals(SELF_LINK, builtDataset.getSelfLink());
assertEquals(LABELS, builtDataset.getLabels());
assertEquals(STORAGE_BILLING_MODEL, builtDataset.getStorageBillingModel());
assertEquals(MAX_TIME_TRAVEL_HOURS, builtDataset.getMaxTimeTravelHours());
}

@Test
Expand Down Expand Up @@ -344,6 +347,7 @@ public void testExternalDatasetReference() {
.setLabels(LABELS)
.setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE)
.setStorageBillingModel(STORAGE_BILLING_MODEL)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.build();
assertEquals(
EXTERNAL_DATASET_REFERENCE,
Expand Down Expand Up @@ -374,5 +378,6 @@ private void compareDatasetInfo(DatasetInfo expected, DatasetInfo value) {
assertEquals(expected.getLastModified(), value.getLastModified());
assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference());
assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel());
assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours());
}
}
Expand Up @@ -203,6 +203,7 @@ public class ITBigQueryTest {
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8);
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
private static final Long MAX_TIME_TRAVEL_HOURS = 48L;
private static final String CLOUD_SAMPLES_DATA =
Optional.fromNullable(System.getenv("CLOUD_SAMPLES_DATA_BUCKET")).or("cloud-samples-data");
private static final Map<String, String> LABELS =
Expand Down Expand Up @@ -967,6 +968,7 @@ public void testGetDatasetWithSelectedFields() {
assertNull(dataset.getLocation());
assertNull(dataset.getSelfLink());
assertNull(dataset.getStorageBillingModel());
assertNull(dataset.getMaxTimeTravelHours());
}

@Test
Expand All @@ -983,6 +985,7 @@ public void testUpdateDataset() {
assertThat(dataset.getDescription()).isEqualTo("Some Description");
assertThat(dataset.getLabels()).containsExactly("a", "b");
assertThat(dataset.getStorageBillingModel()).isNull();
assertThat(dataset.getMaxTimeTravelHours()).isNull();

Map<String, String> updateLabels = new HashMap<>();
updateLabels.put("x", "y");
Expand All @@ -994,10 +997,12 @@ public void testUpdateDataset() {
.setDescription("Updated Description")
.setLabels(updateLabels)
.setStorageBillingModel("LOGICAL")
.setMaxTimeTravelHours(48L)
.build());
assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description");
assertThat(updatedDataset.getLabels()).containsExactly("x", "y");
assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL");
assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(48L);

updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build());
assertThat(updatedDataset.getLabels()).isEmpty();
Expand Down Expand Up @@ -1028,6 +1033,7 @@ public void testUpdateDatasetWithSelectedFields() {
assertNull(updatedDataset.getLocation());
assertNull(updatedDataset.getSelfLink());
assertNull(updatedDataset.getStorageBillingModel());
assertNull(updatedDataset.getMaxTimeTravelHours());
assertTrue(dataset.delete());
}

Expand Down Expand Up @@ -1306,6 +1312,23 @@ public void testCreateDatasetWithSpecifiedStorageBillingModel() {
RemoteBigQueryHelper.forceDelete(bigquery, billingModelDataset);
}

@Test
public void testCreateDatasetWithSpecifiedMaxTimeTravelHours() {
String timeTravelDataset = RemoteBigQueryHelper.generateDatasetName();
DatasetInfo info =
DatasetInfo.newBuilder(timeTravelDataset)
.setDescription(DESCRIPTION)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.setLabels(LABELS)
.build();
bigquery.create(info);

Dataset dataset = bigquery.getDataset(DatasetId.of(timeTravelDataset));
assertEquals(MAX_TIME_TRAVEL_HOURS, dataset.getMaxTimeTravelHours());

RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset);
}

@Test
public void testCreateDatasetWithDefaultCollation() {
String collationDataset = RemoteBigQueryHelper.generateDatasetName();
Expand Down