Skip to content

Commit

Permalink
feat: Add support for Dataset property maxTimeTravelHours
Browse files Browse the repository at this point in the history
  • Loading branch information
@brendangreenley
brendangreenley committed Nov 27, 2023
1 parent 0c8fb65 commit 406e167
Show file tree
Hide file tree
Showing 6 changed files with 71 additions and 0 deletions.
5 changes: 5 additions & 0 deletions google-cloud-bigquery/clirr-ignored-differences.xml
View file
Expand Up @@ -119,4 +119,9 @@
<className>com/google/cloud/bigquery/StandardTableDefinition*</className>
<method>*BigLakeConfiguration(*)</method>
</difference>
<difference>
<differenceType>7013</differenceType>
<className>com/google/cloud/bigquery/DatasetInfo*</className>
<method>*setMaxTimeTravelHours(*)</method>
</difference>
</differences>
6 changes: 6 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/Dataset.java
View file
Expand Up @@ -164,6 +164,12 @@ public Builder setStorageBillingModel(String storageBillingModel) {
return this;
}

@Override
public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) {
infoBuilder.setMaxTimeTravelHours(maxTimeTravelHours);
return this;
}

@Override
public Dataset build() {
return new Dataset(bigquery, infoBuilder);
Expand Down
26 changes: 26 additions & 0 deletions google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/DatasetInfo.java
View file
Expand Up @@ -75,6 +75,7 @@ public Dataset apply(DatasetInfo datasetInfo) {
private final String defaultCollation;
private final ExternalDatasetReference externalDatasetReference;
private final String storageBillingModel;
private final Long maxTimeTravelHours;

/** A builder for {@code DatasetInfo} objects. */
public abstract static class Builder {
Expand Down Expand Up @@ -142,6 +143,13 @@ public abstract Builder setExternalDatasetReference(
*/
public abstract Builder setStorageBillingModel(String storageBillingModel);

/**
* Optional. Defines the time travel window in hours. The value can be from 48 to 168 hours (2
* to 7 days). Value must in multiple of 24 hours (48, 72, 96, 120, 144, 168) The default value
* is 168 hours if this is not set.
*/
public abstract Builder setMaxTimeTravelHours(Long maxTimeTravelHours);

/**
* The default encryption key for all tables in the dataset. Once this property is set, all
* newly-created partitioned tables in the dataset will have encryption key set to this value,
Expand Down Expand Up @@ -200,6 +208,7 @@ static final class BuilderImpl extends Builder {
private String defaultCollation;
private ExternalDatasetReference externalDatasetReference;
private String storageBillingModel;
private Long maxTimeTravelHours;

BuilderImpl() {}

Expand All @@ -221,6 +230,7 @@ static final class BuilderImpl extends Builder {
this.defaultCollation = datasetInfo.defaultCollation;
this.externalDatasetReference = datasetInfo.externalDatasetReference;
this.storageBillingModel = datasetInfo.storageBillingModel;
this.maxTimeTravelHours = datasetInfo.maxTimeTravelHours;
}

BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) {
Expand Down Expand Up @@ -260,6 +270,7 @@ public Acl apply(Dataset.Access accessPb) {
ExternalDatasetReference.fromPb(datasetPb.getExternalDatasetReference());
}
this.storageBillingModel = datasetPb.getStorageBillingModel();
this.maxTimeTravelHours = datasetPb.getMaxTimeTravelHours();
}

@Override
Expand Down Expand Up @@ -372,6 +383,12 @@ public Builder setStorageBillingModel(String storageBillingModel) {
return this;
}

@Override
public Builder setMaxTimeTravelHours(Long maxTimeTravelHours) {
this.maxTimeTravelHours = maxTimeTravelHours;
return this;
}

@Override
public DatasetInfo build() {
return new DatasetInfo(this);
Expand All @@ -396,6 +413,7 @@ public DatasetInfo build() {
defaultCollation = builder.defaultCollation;
externalDatasetReference = builder.externalDatasetReference;
storageBillingModel = builder.storageBillingModel;
maxTimeTravelHours = builder.maxTimeTravelHours;
}

/** Returns the dataset identity. */
Expand Down Expand Up @@ -529,6 +547,10 @@ public String getStorageBillingModel() {
return storageBillingModel;
}

public Long getMaxTimeTravelHours() {
return maxTimeTravelHours;
}

/**
* Returns information about the external metadata storage where the dataset is defined. Filled
* out when the dataset type is EXTERNAL.
Expand Down Expand Up @@ -562,6 +584,7 @@ public String toString() {
.add("defaultCollation", defaultCollation)
.add("externalDatasetReference", externalDatasetReference)
.add("storageBillingModel", storageBillingModel)
.add("maxTimeTravelHours", maxTimeTravelHours)
.toString();
}

Expand Down Expand Up @@ -646,6 +669,9 @@ public Dataset.Access apply(Acl acl) {
if (storageBillingModel != null) {
datasetPb.setStorageBillingModel(storageBillingModel);
}
if (maxTimeTravelHours != null) {
datasetPb.setMaxTimeTravelHours(maxTimeTravelHours);
}
return datasetPb;
}

Expand Down
6 changes: 6 additions & 0 deletions google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetInfoTest.java
View file
Expand Up @@ -59,6 +59,7 @@ public class DatasetInfoTest {
private static final EncryptionConfiguration DATASET_ENCRYPTION_CONFIGURATION =
EncryptionConfiguration.newBuilder().setKmsKeyName("KMS_KEY_1").build();
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
private static final Long MAX_TIME_TRAVEL_HOURS = 48L;

private static final ExternalDatasetReference EXTERNAL_DATASET_REFERENCE =
ExternalDatasetReference.newBuilder()
Expand All @@ -81,6 +82,7 @@ public class DatasetInfoTest {
.setDefaultEncryptionConfiguration(DATASET_ENCRYPTION_CONFIGURATION)
.setDefaultPartitionExpirationMs(DEFAULT_PARTITION__EXPIRATION)
.setStorageBillingModel(STORAGE_BILLING_MODEL)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.build();
private static final DatasetInfo DATASET_INFO_COMPLETE =
DATASET_INFO
Expand Down Expand Up @@ -173,6 +175,7 @@ public void testBuilder() {
EXTERNAL_DATASET_REFERENCE,
DATASET_INFO_COMPLETE_WITH_EXTERNAL_DATASET_REFERENCE.getExternalDatasetReference());
assertEquals(STORAGE_BILLING_MODEL, DATASET_INFO_COMPLETE.getStorageBillingModel());
assertEquals(MAX_TIME_TRAVEL_HOURS, DATASET_INFO_COMPLETE.getMaxTimeTravelHours());
}

@Test
Expand All @@ -194,6 +197,7 @@ public void testOf() {
assertTrue(datasetInfo.getLabels().isEmpty());
assertNull(datasetInfo.getExternalDatasetReference());
assertNull(datasetInfo.getStorageBillingModel());
assertNull(datasetInfo.getMaxTimeTravelHours());

datasetInfo = DatasetInfo.of(DATASET_ID);
assertEquals(DATASET_ID, datasetInfo.getDatasetId());
Expand All @@ -212,6 +216,7 @@ public void testOf() {
assertTrue(datasetInfo.getLabels().isEmpty());
assertNull(datasetInfo.getExternalDatasetReference());
assertNull(datasetInfo.getStorageBillingModel());
assertNull(datasetInfo.getMaxTimeTravelHours());
}

@Test
Expand Down Expand Up @@ -249,5 +254,6 @@ private void compareDatasets(DatasetInfo expected, DatasetInfo value) {
expected.getDefaultPartitionExpirationMs(), value.getDefaultPartitionExpirationMs());
assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference());
assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel());
assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours());
}
}
5 changes: 5 additions & 0 deletions google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/DatasetTest.java
View file
Expand Up @@ -67,6 +67,7 @@ public class DatasetTest {
private static final DatasetInfo DATASET_INFO = DatasetInfo.newBuilder(DATASET_ID).build();
private static final Field FIELD = Field.of("FieldName", LegacySQLTypeName.INTEGER);
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
private static final Long MAX_TIME_TRAVEL_HOURS = 48L;
private static final StandardTableDefinition TABLE_DEFINITION =
StandardTableDefinition.of(Schema.of(FIELD));
private static final ViewDefinition VIEW_DEFINITION = ViewDefinition.of("QUERY");
Expand Down Expand Up @@ -122,6 +123,7 @@ public void testBuilder() {
.setSelfLink(SELF_LINK)
.setLabels(LABELS)
.setStorageBillingModel(STORAGE_BILLING_MODEL)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.build();
assertEquals(DATASET_ID, builtDataset.getDatasetId());
assertEquals(ACCESS_RULES, builtDataset.getAcl());
Expand All @@ -136,6 +138,7 @@ public void testBuilder() {
assertEquals(SELF_LINK, builtDataset.getSelfLink());
assertEquals(LABELS, builtDataset.getLabels());
assertEquals(STORAGE_BILLING_MODEL, builtDataset.getStorageBillingModel());
assertEquals(MAX_TIME_TRAVEL_HOURS, builtDataset.getMaxTimeTravelHours());
}

@Test
Expand Down Expand Up @@ -344,6 +347,7 @@ public void testExternalDatasetReference() {
.setLabels(LABELS)
.setExternalDatasetReference(EXTERNAL_DATASET_REFERENCE)
.setStorageBillingModel(STORAGE_BILLING_MODEL)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.build();
assertEquals(
EXTERNAL_DATASET_REFERENCE,
Expand Down Expand Up @@ -374,5 +378,6 @@ private void compareDatasetInfo(DatasetInfo expected, DatasetInfo value) {
assertEquals(expected.getLastModified(), value.getLastModified());
assertEquals(expected.getExternalDatasetReference(), value.getExternalDatasetReference());
assertEquals(expected.getStorageBillingModel(), value.getStorageBillingModel());
assertEquals(expected.getMaxTimeTravelHours(), value.getMaxTimeTravelHours());
}
}
23 changes: 23 additions & 0 deletions google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java
View file
Expand Up @@ -203,6 +203,7 @@ public class ITBigQueryTest {
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();
private static final String RANDOM_ID = UUID.randomUUID().toString().substring(0, 8);
private static final String STORAGE_BILLING_MODEL = "LOGICAL";
private static final Long MAX_TIME_TRAVEL_HOURS = 48L;
private static final String CLOUD_SAMPLES_DATA =
Optional.fromNullable(System.getenv("CLOUD_SAMPLES_DATA_BUCKET")).or("cloud-samples-data");
private static final Map<String, String> LABELS =
Expand Down Expand Up @@ -967,6 +968,7 @@ public void testGetDatasetWithSelectedFields() {
assertNull(dataset.getLocation());
assertNull(dataset.getSelfLink());
assertNull(dataset.getStorageBillingModel());
assertNull(dataset.getMaxTimeTravelHours());
}

@Test
Expand All @@ -983,6 +985,7 @@ public void testUpdateDataset() {
assertThat(dataset.getDescription()).isEqualTo("Some Description");
assertThat(dataset.getLabels()).containsExactly("a", "b");
assertThat(dataset.getStorageBillingModel()).isNull();
assertThat(dataset.getMaxTimeTravelHours()).isNull();

Map<String, String> updateLabels = new HashMap<>();
updateLabels.put("x", "y");
Expand All @@ -994,10 +997,12 @@ public void testUpdateDataset() {
.setDescription("Updated Description")
.setLabels(updateLabels)
.setStorageBillingModel("LOGICAL")
.setMaxTimeTravelHours(48L)
.build());
assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description");
assertThat(updatedDataset.getLabels()).containsExactly("x", "y");
assertThat(updatedDataset.getStorageBillingModel()).isEqualTo("LOGICAL");
assertThat(updatedDataset.getMaxTimeTravelHours()).isEqualTo(48L);

updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build());
assertThat(updatedDataset.getLabels()).isEmpty();
Expand Down Expand Up @@ -1028,6 +1033,7 @@ public void testUpdateDatasetWithSelectedFields() {
assertNull(updatedDataset.getLocation());
assertNull(updatedDataset.getSelfLink());
assertNull(updatedDataset.getStorageBillingModel());
assertNull(updatedDataset.getMaxTimeTravelHours());
assertTrue(dataset.delete());
}

Expand Down Expand Up @@ -1306,6 +1312,23 @@ public void testCreateDatasetWithSpecifiedStorageBillingModel() {
RemoteBigQueryHelper.forceDelete(bigquery, billingModelDataset);
}

@Test
public void testCreateDatasetWithSpecifiedMaxTimeTravelHours() {
String timeTravelDataset = RemoteBigQueryHelper.generateDatasetName();
DatasetInfo info =
DatasetInfo.newBuilder(timeTravelDataset)
.setDescription(DESCRIPTION)
.setMaxTimeTravelHours(MAX_TIME_TRAVEL_HOURS)
.setLabels(LABELS)
.build();
bigquery.create(info);

Dataset dataset = bigquery.getDataset(DatasetId.of(timeTravelDataset));
assertEquals(MAX_TIME_TRAVEL_HOURS, dataset.getMaxTimeTravelHours());

RemoteBigQueryHelper.forceDelete(bigquery, timeTravelDataset);
}

@Test
public void testCreateDatasetWithDefaultCollation() {
String collationDataset = RemoteBigQueryHelper.generateDatasetName();
Expand Down

0 comments on commit 406e167

Please sign in to comment.