From f9bd8b6f20f323a5fefbc6ea8a9b154ccff410ac Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Mon, 23 Aug 2021 15:17:19 -0600 Subject: [PATCH 1/5] test: Add test of datetime and time pandas load --- tests/system/test_pandas.py | 40 +++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 371dcea71..59cdb8c5d 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -798,3 +798,43 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client): dataframe = row_iterator.to_dataframe(bqstorage_client=bqstorage_client) assert len(dataframe.index) == 100 + + +def test_upload_time_and_datetime(bigquery_client, dataset_id): + df = pandas.DataFrame( + dict(dt=[datetime.datetime(2020, 1, 8, 8, 0, 0), + datetime.datetime( + 2020, 1, 8, 8, 0, 0, + tzinfo=datetime.timezone(datetime.timedelta(hours=-7))), + ], + t=[datetime.time(0, 0, 10, 100001), None], + )) + table = f"{dataset_id}.test_upload_time_and_datetime" + bigquery_client.load_table_from_dataframe(df, table).result() + data = list(map(list, bigquery_client.list_rows(table))) + assert data == [ + [datetime.datetime(2020, 1, 8, 8, 0, tzinfo=datetime.timezone.utc), + datetime.time(0, 0, 10, 100001), + ], + [datetime.datetime(2020, 1, 8, 15, 0, tzinfo=datetime.timezone.utc), + None, + ], + ] + + from google.cloud.bigquery import job, schema + + table = f"{dataset_id}.test_upload_time_and_datetime_dt" + config = job.LoadJobConfig( + schema=[schema.SchemaField('dt', 'DATETIME'), + schema.SchemaField('t', 'TIME')]) + + bigquery_client.load_table_from_dataframe(df, table, job_config=config).result() + data = list(map(list, bigquery_client.list_rows(table))) + assert data == [ + [datetime.datetime(2020, 1, 8, 8, 0), + datetime.time(0, 0, 10, 100001), + ], + [datetime.datetime(2020, 1, 8, 15, 0), + None, + ], + ] From 842c806f69547ee32b9e611e8978007e8bdba321 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Mon, 23 Aug 2021 15:19:53 -0600 Subject: [PATCH 2/5] blacken --- tests/system/test_pandas.py | 46 ++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 59cdb8c5d..2fa6697c2 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -802,39 +802,43 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client): def test_upload_time_and_datetime(bigquery_client, dataset_id): df = pandas.DataFrame( - dict(dt=[datetime.datetime(2020, 1, 8, 8, 0, 0), - datetime.datetime( - 2020, 1, 8, 8, 0, 0, - tzinfo=datetime.timezone(datetime.timedelta(hours=-7))), - ], - t=[datetime.time(0, 0, 10, 100001), None], - )) + dict( + dt=[ + datetime.datetime(2020, 1, 8, 8, 0, 0), + datetime.datetime( + 2020, + 1, + 8, + 8, + 0, + 0, + tzinfo=datetime.timezone(datetime.timedelta(hours=-7)), + ), + ], + t=[datetime.time(0, 0, 10, 100001), None], + ) + ) table = f"{dataset_id}.test_upload_time_and_datetime" bigquery_client.load_table_from_dataframe(df, table).result() data = list(map(list, bigquery_client.list_rows(table))) assert data == [ - [datetime.datetime(2020, 1, 8, 8, 0, tzinfo=datetime.timezone.utc), - datetime.time(0, 0, 10, 100001), - ], - [datetime.datetime(2020, 1, 8, 15, 0, tzinfo=datetime.timezone.utc), - None, - ], + [ + datetime.datetime(2020, 1, 8, 8, 0, tzinfo=datetime.timezone.utc), + datetime.time(0, 0, 10, 100001), + ], + [datetime.datetime(2020, 1, 8, 15, 0, tzinfo=datetime.timezone.utc), None], ] from google.cloud.bigquery import job, schema table = f"{dataset_id}.test_upload_time_and_datetime_dt" config = job.LoadJobConfig( - schema=[schema.SchemaField('dt', 'DATETIME'), - schema.SchemaField('t', 'TIME')]) + schema=[schema.SchemaField("dt", "DATETIME"), schema.SchemaField("t", "TIME")] + ) bigquery_client.load_table_from_dataframe(df, table, job_config=config).result() data = list(map(list, bigquery_client.list_rows(table))) assert data == [ - [datetime.datetime(2020, 1, 8, 8, 0), - datetime.time(0, 0, 10, 100001), - ], - [datetime.datetime(2020, 1, 8, 15, 0), - None, - ], + [datetime.datetime(2020, 1, 8, 8, 0), datetime.time(0, 0, 10, 100001)], + [datetime.datetime(2020, 1, 8, 15, 0), None], ] From 129778cbe37ad67dd1142a813be10f9e87552050 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Tue, 24 Aug 2021 08:22:51 -0600 Subject: [PATCH 3/5] Uncommented the DATETIME column in test_load_table_from_dataframe_w_explicit_schema and added the issue numner (56) to test_upload_time_and_datetime --- tests/system/test_pandas.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 2fa6697c2..131f67b63 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -279,8 +279,6 @@ def test_load_table_from_dataframe_w_required(bigquery_client, dataset_id): def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id): # Schema with all scalar types. - # TODO: Uploading DATETIME columns currently fails, thus that field type - # is temporarily removed from the test. # See: # https://github.com/googleapis/python-bigquery/issues/61 # https://issuetracker.google.com/issues/151765076 @@ -288,7 +286,7 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id bigquery.SchemaField("bool_col", "BOOLEAN"), bigquery.SchemaField("bytes_col", "BYTES"), bigquery.SchemaField("date_col", "DATE"), - # bigquery.SchemaField("dt_col", "DATETIME"), + bigquery.SchemaField("dt_col", "DATETIME"), bigquery.SchemaField("float_col", "FLOAT"), bigquery.SchemaField("geo_col", "GEOGRAPHY"), bigquery.SchemaField("int_col", "INTEGER"), @@ -313,14 +311,14 @@ def test_load_table_from_dataframe_w_explicit_schema(bigquery_client, dataset_id ("bool_col", [True, None, False]), ("bytes_col", [b"abc", None, b"def"]), ("date_col", [datetime.date(1, 1, 1), None, datetime.date(9999, 12, 31)]), - # ( - # "dt_col", - # [ - # datetime.datetime(1, 1, 1, 0, 0, 0), - # None, - # datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), - # ], - # ), + ( + "dt_col", + [ + datetime.datetime(1, 1, 1, 0, 0, 0), + None, + datetime.datetime(9999, 12, 31, 23, 59, 59, 999999), + ], + ), ("float_col", [float("-inf"), float("nan"), float("inf")]), ( "geo_col", @@ -800,7 +798,7 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client): assert len(dataframe.index) == 100 -def test_upload_time_and_datetime(bigquery_client, dataset_id): +def test_upload_time_and_datetime_56(bigquery_client, dataset_id): df = pandas.DataFrame( dict( dt=[ From 8b92e60d2f96e4edb0fb1b3371348abae7a598aa Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 24 Aug 2021 16:40:42 +0200 Subject: [PATCH 4/5] Add missing closing bracket --- tests/system/test_pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index c31158963..7cea2643e 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -839,7 +839,7 @@ def test_upload_time_and_datetime_56(bigquery_client, dataset_id): assert data == [ [datetime.datetime(2020, 1, 8, 8, 0), datetime.time(0, 0, 10, 100001)], [datetime.datetime(2020, 1, 8, 15, 0), None], - + ] def test_to_dataframe_geography_as_objects(bigquery_client, dataset_id): wkt = pytest.importorskip("shapely.wkt") From d2f99bb31d1f1da78ca0a7da38515161b8da249e Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Tue, 24 Aug 2021 16:41:28 +0200 Subject: [PATCH 5/5] Add missing blank line --- tests/system/test_pandas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 7cea2643e..93ce23481 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -840,7 +840,8 @@ def test_upload_time_and_datetime_56(bigquery_client, dataset_id): [datetime.datetime(2020, 1, 8, 8, 0), datetime.time(0, 0, 10, 100001)], [datetime.datetime(2020, 1, 8, 15, 0), None], ] - + + def test_to_dataframe_geography_as_objects(bigquery_client, dataset_id): wkt = pytest.importorskip("shapely.wkt") bigquery_client.query(