From b122cccae9ffefaa06efba5898c0f1197e008493 Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Mon, 27 Sep 2021 21:23:36 +0530 Subject: [PATCH 1/8] fix: add support for json data type --- google/cloud/spanner_dbapi/parse_utils.py | 1 + google/cloud/spanner_v1/_helpers.py | 5 +++ google/cloud/spanner_v1/batch.py | 3 +- tests/system/test_dbapi.py | 46 +++++++++++++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/google/cloud/spanner_dbapi/parse_utils.py b/google/cloud/spanner_dbapi/parse_utils.py index d967330cea..57122879c8 100644 --- a/google/cloud/spanner_dbapi/parse_utils.py +++ b/google/cloud/spanner_dbapi/parse_utils.py @@ -38,6 +38,7 @@ DateStr: spanner.param_types.DATE, TimestampStr: spanner.param_types.TIMESTAMP, decimal.Decimal: spanner.param_types.NUMERIC, + dict: spanner.param_types.JSON, } SPANNER_RESERVED_KEYWORDS = { diff --git a/google/cloud/spanner_v1/_helpers.py b/google/cloud/spanner_v1/_helpers.py index c7cdf7aedc..49a0c76f31 100644 --- a/google/cloud/spanner_v1/_helpers.py +++ b/google/cloud/spanner_v1/_helpers.py @@ -17,6 +17,7 @@ import datetime import decimal import math +import json import six @@ -166,6 +167,10 @@ def _make_value_pb(value): if isinstance(value, decimal.Decimal): _assert_numeric_precision_and_scale(value) return Value(string_value=str(value)) + if isinstance(value, dict): + return Value( + string_value=json.dumps(value, sort_keys=True, separators=(",", ":"),) + ) raise ValueError("Unknown type: %s" % (value,)) diff --git a/google/cloud/spanner_v1/batch.py b/google/cloud/spanner_v1/batch.py index 2172d9d051..baf0a50a15 100644 --- a/google/cloud/spanner_v1/batch.py +++ b/google/cloud/spanner_v1/batch.py @@ -118,8 +118,7 @@ def delete(self, table, keyset): class Batch(_BatchBase): - """Accumulate mutations for transmission during :meth:`commit`. - """ + """Accumulate mutations for transmission during :meth:`commit`.""" committed = None commit_stats = None diff --git a/tests/system/test_dbapi.py b/tests/system/test_dbapi.py index f6af3c3763..bdd225678a 100644 --- a/tests/system/test_dbapi.py +++ b/tests/system/test_dbapi.py @@ -328,6 +328,52 @@ def test_DDL_autocommit(shared_instance, dbapi_database): conn.commit() +def test_autocommit_with_json_data(shared_instance, dbapi_database): + """Check that DDLs in autocommit mode are immediately executed for + json fields.""" + # Create table + conn = Connection(shared_instance, dbapi_database) + conn.autocommit = True + + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE JsonDetails ( + DataId INT64 NOT NULL, + Details JSON, + ) PRIMARY KEY (DataId) + """ + ) + conn.close() + + # Insert data to table + conn = Connection(shared_instance, dbapi_database) + conn.autocommit = True + cur = conn.cursor() + cur.execute( + sql="INSERT INTO JsonDetails (DataId, Details) VALUES (%s, %s)", + args=(123, {"name": "Jakob", "age": "26"}), + ) + + # Read back the data. + conn = Connection(shared_instance, dbapi_database) + cur = conn.cursor() + cur.execute("""select * from JsonDetails;""") + got_rows = cur.fetchall() + conn.close() + + # Assert the response + assert len(got_rows) == 1 + assert got_rows[0][0] == 123 + assert got_rows[0][1] == '{"age":"26","name":"Jakob"}' + + # Drop the table + conn = Connection(shared_instance, dbapi_database) + cur = conn.cursor() + cur.execute("DROP TABLE JsonDetails") + conn.commit() + + def test_DDL_commit(shared_instance, dbapi_database): """Check that DDLs in commit mode are executed on calling `commit()`.""" conn = Connection(shared_instance, dbapi_database) From 5f9b4fef00aee98989d6721088061f2281e42d11 Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Mon, 27 Sep 2021 22:45:47 +0530 Subject: [PATCH 2/8] fix: skip json test for emulator --- tests/system/test_dbapi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system/test_dbapi.py b/tests/system/test_dbapi.py index bdd225678a..02628bb77b 100644 --- a/tests/system/test_dbapi.py +++ b/tests/system/test_dbapi.py @@ -328,6 +328,7 @@ def test_DDL_autocommit(shared_instance, dbapi_database): conn.commit() +@pytest.mark.skipif(_helpers.USE_EMULATOR, reason="Emulator does not support json.") def test_autocommit_with_json_data(shared_instance, dbapi_database): """Check that DDLs in autocommit mode are immediately executed for json fields.""" From 325aaa7860274e37824754ba1514f63a43e1a3fe Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Wed, 29 Sep 2021 14:41:46 +0530 Subject: [PATCH 3/8] refactor: move JsonObject data type to spanner_v1/types/datatypes.py --- google/cloud/spanner_dbapi/parse_utils.py | 3 ++- google/cloud/spanner_v1/_helpers.py | 5 ++-- google/cloud/spanner_v1/types/__init__.py | 2 ++ google/cloud/spanner_v1/types/data_types.py | 25 ++++++++++++++++++++ tests/system/test_dbapi.py | 4 ++-- tests/unit/spanner_dbapi/test_parse_utils.py | 11 ++++++--- 6 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 google/cloud/spanner_v1/types/data_types.py diff --git a/google/cloud/spanner_dbapi/parse_utils.py b/google/cloud/spanner_dbapi/parse_utils.py index 57122879c8..bd8bdb8868 100644 --- a/google/cloud/spanner_dbapi/parse_utils.py +++ b/google/cloud/spanner_dbapi/parse_utils.py @@ -21,6 +21,7 @@ import sqlparse from google.cloud import spanner_v1 as spanner +from google.cloud.spanner_v1.types.data_types import JsonObject from .exceptions import Error, ProgrammingError from .parser import parse_values @@ -38,7 +39,7 @@ DateStr: spanner.param_types.DATE, TimestampStr: spanner.param_types.TIMESTAMP, decimal.Decimal: spanner.param_types.NUMERIC, - dict: spanner.param_types.JSON, + JsonObject: spanner.param_types.JSON, } SPANNER_RESERVED_KEYWORDS = { diff --git a/google/cloud/spanner_v1/_helpers.py b/google/cloud/spanner_v1/_helpers.py index 49a0c76f31..9ef656b11e 100644 --- a/google/cloud/spanner_v1/_helpers.py +++ b/google/cloud/spanner_v1/_helpers.py @@ -18,6 +18,7 @@ import decimal import math import json +from google.cloud.spanner_v1.types.data_types import JsonObject import six @@ -29,7 +30,7 @@ from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud.spanner_v1 import TypeCode from google.cloud.spanner_v1 import ExecuteSqlRequest - +from google.cloud.spanner_v1.types.data_types import JsonObject # Validation error messages NUMERIC_MAX_SCALE_ERR_MSG = ( @@ -167,7 +168,7 @@ def _make_value_pb(value): if isinstance(value, decimal.Decimal): _assert_numeric_precision_and_scale(value) return Value(string_value=str(value)) - if isinstance(value, dict): + if isinstance(value, JsonObject): return Value( string_value=json.dumps(value, sort_keys=True, separators=(",", ":"),) ) diff --git a/google/cloud/spanner_v1/types/__init__.py b/google/cloud/spanner_v1/types/__init__.py index 5f7bbfb8b1..0aec3be325 100644 --- a/google/cloud/spanner_v1/types/__init__.py +++ b/google/cloud/spanner_v1/types/__init__.py @@ -62,6 +62,7 @@ Type, TypeCode, ) +from .data_types import JsonObject __all__ = ( "CommitResponse", @@ -101,4 +102,5 @@ "StructType", "Type", "TypeCode", + "JsonObject", ) diff --git a/google/cloud/spanner_v1/types/data_types.py b/google/cloud/spanner_v1/types/data_types.py new file mode 100644 index 0000000000..830f654a64 --- /dev/null +++ b/google/cloud/spanner_v1/types/data_types.py @@ -0,0 +1,25 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Custom data types for spanner.""" + + +class JsonObject(dict): + """ + JsonObject type help format Django jsonfield to compatible Cloud Spanner's + JSON type. Before making queries, it'll help differentiate between + normal parameters and JSON parameters. + """ + + pass diff --git a/tests/system/test_dbapi.py b/tests/system/test_dbapi.py index 02628bb77b..9ae6546245 100644 --- a/tests/system/test_dbapi.py +++ b/tests/system/test_dbapi.py @@ -15,11 +15,11 @@ import hashlib import pickle import pkg_resources - import pytest from google.cloud import spanner_v1 from google.cloud.spanner_dbapi.connection import connect, Connection +from google.cloud.spanner_v1.types.data_types import JsonObject from . import _helpers DATABASE_NAME = "dbapi-txn" @@ -353,7 +353,7 @@ def test_autocommit_with_json_data(shared_instance, dbapi_database): cur = conn.cursor() cur.execute( sql="INSERT INTO JsonDetails (DataId, Details) VALUES (%s, %s)", - args=(123, {"name": "Jakob", "age": "26"}), + args=(123, JsonObject({"name": "Jakob", "age": "26"})), ) # Read back the data. diff --git a/tests/unit/spanner_dbapi/test_parse_utils.py b/tests/unit/spanner_dbapi/test_parse_utils.py index 4de429076e..37ca6b2b75 100644 --- a/tests/unit/spanner_dbapi/test_parse_utils.py +++ b/tests/unit/spanner_dbapi/test_parse_utils.py @@ -16,6 +16,7 @@ import unittest from google.cloud.spanner_v1 import param_types +from google.cloud.spanner_v1.types.data_types import JsonObject class TestParseUtils(unittest.TestCase): @@ -333,9 +334,11 @@ def test_get_param_types(self): import datetime import decimal - from google.cloud.spanner_dbapi.parse_utils import DateStr - from google.cloud.spanner_dbapi.parse_utils import TimestampStr - from google.cloud.spanner_dbapi.parse_utils import get_param_types + from google.cloud.spanner_dbapi.parse_utils import ( + DateStr, + TimestampStr, + get_param_types, + ) params = { "a1": 10, @@ -349,6 +352,7 @@ def test_get_param_types(self): "i1": b"bytes", "j1": None, "k1": decimal.Decimal("3.194387483193242e+19"), + "l1": JsonObject({"key": "value"}), } want_types = { "a1": param_types.INT64, @@ -361,6 +365,7 @@ def test_get_param_types(self): "h1": param_types.DATE, "i1": param_types.BYTES, "k1": param_types.NUMERIC, + "l1": param_types.JSON, } got_types = get_param_types(params) self.assertEqual(got_types, want_types) From ff7f125e0e0e53ce15b1326e3a3a93f9e4290ff9 Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Wed, 29 Sep 2021 14:45:37 +0530 Subject: [PATCH 4/8] refactor: remove duplicate import --- google/cloud/spanner_v1/_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/google/cloud/spanner_v1/_helpers.py b/google/cloud/spanner_v1/_helpers.py index 9ef656b11e..9f1e239187 100644 --- a/google/cloud/spanner_v1/_helpers.py +++ b/google/cloud/spanner_v1/_helpers.py @@ -18,7 +18,6 @@ import decimal import math import json -from google.cloud.spanner_v1.types.data_types import JsonObject import six From d1e3282fa927e95570c0f4fb0f38ecc62dbbca04 Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Thu, 30 Sep 2021 15:08:48 +0530 Subject: [PATCH 5/8] refactor: remove extra connection creation in test --- google/cloud/spanner_v1/types/data_types.py | 2 +- tests/system/test_dbapi.py | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/google/cloud/spanner_v1/types/data_types.py b/google/cloud/spanner_v1/types/data_types.py index 830f654a64..305c0cb2a9 100644 --- a/google/cloud/spanner_v1/types/data_types.py +++ b/google/cloud/spanner_v1/types/data_types.py @@ -17,7 +17,7 @@ class JsonObject(dict): """ - JsonObject type help format Django jsonfield to compatible Cloud Spanner's + JsonObject type help format Django JSONField to compatible Cloud Spanner's JSON type. Before making queries, it'll help differentiate between normal parameters and JSON parameters. """ diff --git a/tests/system/test_dbapi.py b/tests/system/test_dbapi.py index 9ae6546245..7cf0240982 100644 --- a/tests/system/test_dbapi.py +++ b/tests/system/test_dbapi.py @@ -345,23 +345,16 @@ def test_autocommit_with_json_data(shared_instance, dbapi_database): ) PRIMARY KEY (DataId) """ ) - conn.close() # Insert data to table - conn = Connection(shared_instance, dbapi_database) - conn.autocommit = True - cur = conn.cursor() cur.execute( sql="INSERT INTO JsonDetails (DataId, Details) VALUES (%s, %s)", args=(123, JsonObject({"name": "Jakob", "age": "26"})), ) # Read back the data. - conn = Connection(shared_instance, dbapi_database) - cur = conn.cursor() cur.execute("""select * from JsonDetails;""") got_rows = cur.fetchall() - conn.close() # Assert the response assert len(got_rows) == 1 @@ -369,10 +362,9 @@ def test_autocommit_with_json_data(shared_instance, dbapi_database): assert got_rows[0][1] == '{"age":"26","name":"Jakob"}' # Drop the table - conn = Connection(shared_instance, dbapi_database) - cur = conn.cursor() cur.execute("DROP TABLE JsonDetails") conn.commit() + conn.close() def test_DDL_commit(shared_instance, dbapi_database): From ca10cddce44a8d3c68155a1f69c4379d5b39cb32 Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Fri, 1 Oct 2021 12:21:19 +0530 Subject: [PATCH 6/8] refactor: move data_types.py file to google/cloud/spanner_v1/ --- google/cloud/spanner_dbapi/parse_utils.py | 2 +- google/cloud/spanner_v1/__init__.py | 3 +++ google/cloud/spanner_v1/_helpers.py | 2 +- google/cloud/spanner_v1/{types => }/data_types.py | 0 google/cloud/spanner_v1/types/__init__.py | 2 -- tests/system/test_dbapi.py | 2 +- tests/unit/spanner_dbapi/test_parse_utils.py | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) rename google/cloud/spanner_v1/{types => }/data_types.py (100%) diff --git a/google/cloud/spanner_dbapi/parse_utils.py b/google/cloud/spanner_dbapi/parse_utils.py index bd8bdb8868..4f55a7b2c4 100644 --- a/google/cloud/spanner_dbapi/parse_utils.py +++ b/google/cloud/spanner_dbapi/parse_utils.py @@ -21,7 +21,7 @@ import sqlparse from google.cloud import spanner_v1 as spanner -from google.cloud.spanner_v1.types.data_types import JsonObject +from google.cloud.spanner_v1 import JsonObject from .exceptions import Error, ProgrammingError from .parser import parse_values diff --git a/google/cloud/spanner_v1/__init__.py b/google/cloud/spanner_v1/__init__.py index 4ece165503..4aa08d2c29 100644 --- a/google/cloud/spanner_v1/__init__.py +++ b/google/cloud/spanner_v1/__init__.py @@ -58,6 +58,7 @@ from .types.type import StructType from .types.type import Type from .types.type import TypeCode +from .data_types import JsonObject from google.cloud.spanner_v1 import param_types from google.cloud.spanner_v1.client import Client @@ -132,6 +133,8 @@ "TransactionSelector", "Type", "TypeCode", + # Custom spanner related data types + "JsonObject", # google.cloud.spanner_v1.services "SpannerClient", ) diff --git a/google/cloud/spanner_v1/_helpers.py b/google/cloud/spanner_v1/_helpers.py index 9f1e239187..fc3512f0ec 100644 --- a/google/cloud/spanner_v1/_helpers.py +++ b/google/cloud/spanner_v1/_helpers.py @@ -29,7 +29,7 @@ from google.cloud._helpers import _datetime_to_rfc3339 from google.cloud.spanner_v1 import TypeCode from google.cloud.spanner_v1 import ExecuteSqlRequest -from google.cloud.spanner_v1.types.data_types import JsonObject +from google.cloud.spanner_v1 import JsonObject # Validation error messages NUMERIC_MAX_SCALE_ERR_MSG = ( diff --git a/google/cloud/spanner_v1/types/data_types.py b/google/cloud/spanner_v1/data_types.py similarity index 100% rename from google/cloud/spanner_v1/types/data_types.py rename to google/cloud/spanner_v1/data_types.py diff --git a/google/cloud/spanner_v1/types/__init__.py b/google/cloud/spanner_v1/types/__init__.py index 0aec3be325..5f7bbfb8b1 100644 --- a/google/cloud/spanner_v1/types/__init__.py +++ b/google/cloud/spanner_v1/types/__init__.py @@ -62,7 +62,6 @@ Type, TypeCode, ) -from .data_types import JsonObject __all__ = ( "CommitResponse", @@ -102,5 +101,4 @@ "StructType", "Type", "TypeCode", - "JsonObject", ) diff --git a/tests/system/test_dbapi.py b/tests/system/test_dbapi.py index 7cf0240982..2d1b4097dc 100644 --- a/tests/system/test_dbapi.py +++ b/tests/system/test_dbapi.py @@ -19,7 +19,7 @@ from google.cloud import spanner_v1 from google.cloud.spanner_dbapi.connection import connect, Connection -from google.cloud.spanner_v1.types.data_types import JsonObject +from google.cloud.spanner_v1 import JsonObject from . import _helpers DATABASE_NAME = "dbapi-txn" diff --git a/tests/unit/spanner_dbapi/test_parse_utils.py b/tests/unit/spanner_dbapi/test_parse_utils.py index 37ca6b2b75..994b02d615 100644 --- a/tests/unit/spanner_dbapi/test_parse_utils.py +++ b/tests/unit/spanner_dbapi/test_parse_utils.py @@ -16,7 +16,7 @@ import unittest from google.cloud.spanner_v1 import param_types -from google.cloud.spanner_v1.types.data_types import JsonObject +from google.cloud.spanner_v1 import JsonObject class TestParseUtils(unittest.TestCase): From 3f88139c41fb4d645e8bf5531ab13a8f6929831d Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Mon, 4 Oct 2021 15:11:01 +0530 Subject: [PATCH 7/8] fix: increased db version time to current time, to give db backup more time --- tests/system/test_backup_api.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py index 59237113e6..1523bfdfb6 100644 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -81,12 +81,10 @@ def diff_config_instance( @pytest.fixture(scope="session") def database_version_time(shared_database): - shared_database.reload() - diff = ( - datetime.datetime.now(datetime.timezone.utc) - - shared_database.earliest_version_time - ) - return shared_database.earliest_version_time + diff / 2 + # Backup tests are failing because of timeout. As a temporary fix + # we are increasing db version time to current time. + # Read more: https://github.com/googleapis/python-spanner/issues/496 + return datetime.datetime.now(datetime.timezone.utc) @pytest.fixture(scope="session") From e7b6524b1cedc3fb455cbf0b9a4800ccf893e4f2 Mon Sep 17 00:00:00 2001 From: Vikash Singh <3116482+vi3k6i5@users.noreply.github.com> Date: Mon, 4 Oct 2021 19:12:17 +0530 Subject: [PATCH 8/8] fix: undo database_version_time method definition. --- tests/system/test_backup_api.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/system/test_backup_api.py b/tests/system/test_backup_api.py index 1523bfdfb6..d9fded9c0b 100644 --- a/tests/system/test_backup_api.py +++ b/tests/system/test_backup_api.py @@ -81,10 +81,12 @@ def diff_config_instance( @pytest.fixture(scope="session") def database_version_time(shared_database): - # Backup tests are failing because of timeout. As a temporary fix - # we are increasing db version time to current time. - # Read more: https://github.com/googleapis/python-spanner/issues/496 - return datetime.datetime.now(datetime.timezone.utc) + shared_database.reload() + diff = ( + datetime.datetime.now(datetime.timezone.utc) + - shared_database.earliest_version_time + ) + return shared_database.earliest_version_time + diff / 2 @pytest.fixture(scope="session") @@ -398,6 +400,11 @@ def test_instance_list_backups( ) expire_time_1_stamp = expire_time_1.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + # Backup tests are failing because of timeout. As a temporary fix + # we are increasing db version time to current time. + # Read more: https://github.com/googleapis/python-spanner/issues/496 + database_version_time = datetime.datetime.now(datetime.timezone.utc) + backup1 = shared_instance.backup( backup_id_1, database=shared_database,