From d3304936ce737e424be4833a00557135b882213c Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:08:07 +0000 Subject: [PATCH 1/3] test(django-google-spanner): run live Spanner test apps in parallel with unique DB per app ## Problem - `run_testing_worker.py` invokes `django_test_suite_6.0.sh` against a live Cloud Spanner instance without setting `RUNNING_SPANNER_BACKEND_TESTS=1`, leaving `supports_foreign_keys = True` and `sql_create_fk` enabled during Django test database setup and causing every test app to execute synchronous `ALTER TABLE ... ADD CONSTRAINT ... FOREIGN KEY` deferred DDL operations. - `django_test_suite_6.0.sh` and `django_test_suite_5.2.sh` reuse a single database name and execute `python3 runtests.py $DJANGO_TEST_APP` sequentially across all apps in `django_test_apps.txt` when running against a live Cloud Spanner instance. ## Solution - Set `RUNNING_SPANNER_BACKEND_TESTS=1` in `run_testing_worker.py` and default `RUNNING_SPANNER_BACKEND_TESTS="${RUNNING_SPANNER_BACKEND_TESTS:-1}"` in `django_test_suite_6.0.sh` and `django_test_suite_5.2.sh` so foreign key DDL constraints are skipped, unsupported tests in `DatabaseFeatures.skip_tests` are marked skipped, and test column names are sanitized. - Generate a dedicated database name and settings file per test app with a unique random ID per run (`${APP_PREFIX}_${RUN_ID}`) in `django_test_suite_6.0.sh` and `django_test_suite_5.2.sh`, and run all test apps concurrently in parallel on the same instance when running against live Cloud Spanner (`SPANNER_EMULATOR_HOST` unset). --- .../django_test_suite_5.2.sh | 21 +++++++++++++++++-- .../django_test_suite_6.0.sh | 21 +++++++++++++++++-- .../run_testing_worker.py | 2 +- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/packages/django-google-spanner/django_test_suite_5.2.sh b/packages/django-google-spanner/django_test_suite_5.2.sh index 52e489561f31..3b1a1ce28140 100755 --- a/packages/django-google-spanner/django_test_suite_5.2.sh +++ b/packages/django-google-spanner/django_test_suite_5.2.sh @@ -75,15 +75,32 @@ INSTALLED_APPS = [ } cd $TESTS_DIR/django/tests -create_settings + +export RUNNING_SPANNER_BACKEND_TESTS="${RUNNING_SPANNER_BACKEND_TESTS:-1}" EXIT_STATUS=0 +PIDS="" for DJANGO_TEST_APP in $DJANGO_TEST_APPS do if [ "$DJANGO_TEST_APP" = "order_with_respect_to" ] || [ "$DJANGO_TEST_APP" = "contenttypes_tests" ] || [ "$DJANGO_TEST_APP" = "inspectdb" ]; then echo "Skipping $DJANGO_TEST_APP as it is incompatible with Spanner" continue fi - python3 runtests.py $DJANGO_TEST_APP --verbosity=3 --noinput --settings $SETTINGS_FILE || EXIT_STATUS=$? + APP_PREFIX=$(echo "$DJANGO_TEST_APP" | tr -cd 'a-z0-9' | cut -c1-6) + RUN_ID=$(python3 -c 'import os; print(os.urandom(6).hex())') + TEST_DBNAME="${SPANNER_TEST_DB:-${APP_PREFIX}_${RUN_ID}}" + TEST_DBNAME_OTHER="${TEST_DBNAME}-ot" + SETTINGS_FILE="${TEST_DBNAME}_settings" + create_settings + if [ -z "$SPANNER_EMULATOR_HOST" ]; then + python3 runtests.py "$DJANGO_TEST_APP" --verbosity=3 --noinput --settings "$SETTINGS_FILE" & + PIDS="$PIDS $!" + else + python3 runtests.py "$DJANGO_TEST_APP" --verbosity=3 --noinput --settings "$SETTINGS_FILE" || EXIT_STATUS=$? + fi +done +for PID in $PIDS +do + wait "$PID" || EXIT_STATUS=$? done exit $EXIT_STATUS diff --git a/packages/django-google-spanner/django_test_suite_6.0.sh b/packages/django-google-spanner/django_test_suite_6.0.sh index eeab3d35e62d..fb7704916d25 100755 --- a/packages/django-google-spanner/django_test_suite_6.0.sh +++ b/packages/django-google-spanner/django_test_suite_6.0.sh @@ -76,15 +76,32 @@ INSTALLED_APPS = [ } cd $TESTS_DIR/django/tests -create_settings + +export RUNNING_SPANNER_BACKEND_TESTS="${RUNNING_SPANNER_BACKEND_TESTS:-1}" EXIT_STATUS=0 +PIDS="" for DJANGO_TEST_APP in $DJANGO_TEST_APPS do if [ "$DJANGO_TEST_APP" = "order_with_respect_to" ] || [ "$DJANGO_TEST_APP" = "contenttypes_tests" ] || [ "$DJANGO_TEST_APP" = "inspectdb" ]; then echo "Skipping $DJANGO_TEST_APP as it is incompatible with Spanner" continue fi - python3 runtests.py $DJANGO_TEST_APP --verbosity=3 --noinput --settings $SETTINGS_FILE || EXIT_STATUS=$? + APP_PREFIX=$(echo "$DJANGO_TEST_APP" | tr -cd 'a-z0-9' | cut -c1-6) + RUN_ID=$(python3 -c 'import os; print(os.urandom(6).hex())') + TEST_DBNAME="${SPANNER_TEST_DB:-${APP_PREFIX}_${RUN_ID}}" + TEST_DBNAME_OTHER="${TEST_DBNAME}-ot" + SETTINGS_FILE="${TEST_DBNAME}_settings" + create_settings + if [ -z "$SPANNER_EMULATOR_HOST" ]; then + python3 runtests.py "$DJANGO_TEST_APP" --verbosity=3 --noinput --settings "$SETTINGS_FILE" & + PIDS="$PIDS $!" + else + python3 runtests.py "$DJANGO_TEST_APP" --verbosity=3 --noinput --settings "$SETTINGS_FILE" || EXIT_STATUS=$? + fi +done +for PID in $PIDS +do + wait "$PID" || EXIT_STATUS=$? done exit $EXIT_STATUS diff --git a/packages/django-google-spanner/run_testing_worker.py b/packages/django-google-spanner/run_testing_worker.py index 06c295b73650..b572912a6226 100644 --- a/packages/django-google-spanner/run_testing_worker.py +++ b/packages/django-google-spanner/run_testing_worker.py @@ -70,7 +70,7 @@ def __exit__(self, exc, exc_value, traceback): suite = os.environ.get("DJANGO_TEST_SUITE", "./django_test_suite_6.0.sh") with TestInstance() as instance_name: os.system( - """DJANGO_TEST_APPS="{apps}" SPANNER_TEST_INSTANCE={instance} bash {suite}""".format( + """RUNNING_SPANNER_BACKEND_TESTS=1 DJANGO_TEST_APPS="{apps}" SPANNER_TEST_INSTANCE={instance} bash {suite}""".format( apps=" ".join(test_apps), instance=instance_name, suite=suite ) ) From fe0c251c5922e8095d6a6e6d2bc48aca45d80df5 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 4 Nov 2026 16:50:46 +0000 Subject: [PATCH 2/3] Addressed comments --- .../django_spanner/__init__.py | 12 +++ .../django_spanner/operations.py | 10 +++ .../django_test_suite_5.2.sh | 16 ++-- .../django_test_suite_6.0.sh | 8 +- packages/django-google-spanner/noxfile.py | 5 -- .../django-google-spanner/tests/settings.py | 7 +- .../django_spanner/test_check_constraint.py | 2 + .../system/django_spanner/test_decimal.py | 2 + .../system/django_spanner/test_json_field.py | 2 + .../system/django_spanner/test_queries.py | 2 + .../tests/system/django_spanner/utils.py | 83 +++++++++++-------- .../unit/django_spanner/test_operations.py | 19 +++++ 12 files changed, 116 insertions(+), 52 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/__init__.py b/packages/django-google-spanner/django_spanner/__init__.py index fc8cb56b2104..a442ed76c3e1 100644 --- a/packages/django-google-spanner/django_spanner/__init__.py +++ b/packages/django-google-spanner/django_spanner/__init__.py @@ -93,7 +93,19 @@ def get_prep_value(self, value): return value +old_jsonfield_from_db_value = getattr(JSONField, "from_db_value", None) + + +def jsonfield_from_db_value(self, value, expression, connection): + if isinstance(value, (dict, list, JsonObject)): + return value + if old_jsonfield_from_db_value: + return old_jsonfield_from_db_value(self, value, expression, connection) + return value + + JSONField.get_prep_value = get_prep_value +JSONField.from_db_value = jsonfield_from_db_value old_datetimewithnanoseconds_eq = getattr(DatetimeWithNanoseconds, "__eq__", None) diff --git a/packages/django-google-spanner/django_spanner/operations.py b/packages/django-google-spanner/django_spanner/operations.py index 5a5b8e3e4612..55b340b7dee4 100644 --- a/packages/django-google-spanner/django_spanner/operations.py +++ b/packages/django-google-spanner/django_spanner/operations.py @@ -20,6 +20,7 @@ TimestampStr, escape_name, ) +from google.cloud.spanner_v1 import JsonObject class DatabaseOperations(BaseDatabaseOperations): @@ -770,3 +771,12 @@ def savepoint_rollback_sql(self, sid): Return the SQL for rolling back to a savepoint. """ return "SELECT 1" + + def adapt_json_value(self, value, encoder): + if value is None: + return None + if isinstance(value, JsonObject): + return value + if isinstance(value, (dict, list)): + return JsonObject(value) + return super().adapt_json_value(value, encoder) diff --git a/packages/django-google-spanner/django_test_suite_5.2.sh b/packages/django-google-spanner/django_test_suite_5.2.sh index 3b1a1ce28140..884609c9ce53 100755 --- a/packages/django-google-spanner/django_test_suite_5.2.sh +++ b/packages/django-google-spanner/django_test_suite_5.2.sh @@ -27,15 +27,8 @@ export PYTHONPATH=$PYTHONPATH:$(pwd):$(pwd)/$DJANGO_TESTS_DIR/django python3 create_test_instance.py -# If no SPANNER_TEST_DB is set, generate a unique one -# so that we can have multiple tests running without -# conflicting which changes and constraints. We'll always -# cleanup the created database. -TEST_DBNAME=${SPANNER_TEST_DB:-$(python3 -c 'import os, time; print(chr(ord("a") + time.time_ns() % 26)+os.urandom(10).hex())')} -TEST_DBNAME_OTHER="$TEST_DBNAME-ot" INSTANCE=${SPANNER_TEST_INSTANCE:-spanner-django-python-systest} PROJECT=${PROJECT_ID:-$GOOGLE_CLOUD_PROJECT} -SETTINGS_FILE="$TEST_DBNAME-settings" TESTS_DIR=${DJANGO_TESTS_DIR:-django_tests} create_settings() { @@ -76,8 +69,6 @@ INSTALLED_APPS = [ cd $TESTS_DIR/django/tests -export RUNNING_SPANNER_BACKEND_TESTS="${RUNNING_SPANNER_BACKEND_TESTS:-1}" - EXIT_STATUS=0 PIDS="" for DJANGO_TEST_APP in $DJANGO_TEST_APPS @@ -88,7 +79,12 @@ do fi APP_PREFIX=$(echo "$DJANGO_TEST_APP" | tr -cd 'a-z0-9' | cut -c1-6) RUN_ID=$(python3 -c 'import os; print(os.urandom(6).hex())') - TEST_DBNAME="${SPANNER_TEST_DB:-${APP_PREFIX}_${RUN_ID}}" + if [ -n "$SPANNER_TEST_DB" ]; then + DB_PREFIX=$(echo "$SPANNER_TEST_DB" | cut -c1-20) + TEST_DBNAME="${DB_PREFIX}_${APP_PREFIX}" + else + TEST_DBNAME="${APP_PREFIX}_${RUN_ID}" + fi TEST_DBNAME_OTHER="${TEST_DBNAME}-ot" SETTINGS_FILE="${TEST_DBNAME}_settings" create_settings diff --git a/packages/django-google-spanner/django_test_suite_6.0.sh b/packages/django-google-spanner/django_test_suite_6.0.sh index fb7704916d25..73462387e0b4 100755 --- a/packages/django-google-spanner/django_test_suite_6.0.sh +++ b/packages/django-google-spanner/django_test_suite_6.0.sh @@ -77,8 +77,6 @@ INSTALLED_APPS = [ cd $TESTS_DIR/django/tests -export RUNNING_SPANNER_BACKEND_TESTS="${RUNNING_SPANNER_BACKEND_TESTS:-1}" - EXIT_STATUS=0 PIDS="" for DJANGO_TEST_APP in $DJANGO_TEST_APPS @@ -89,7 +87,11 @@ do fi APP_PREFIX=$(echo "$DJANGO_TEST_APP" | tr -cd 'a-z0-9' | cut -c1-6) RUN_ID=$(python3 -c 'import os; print(os.urandom(6).hex())') - TEST_DBNAME="${SPANNER_TEST_DB:-${APP_PREFIX}_${RUN_ID}}" + if [ -n "$SPANNER_TEST_DB" ]; then + TEST_DBNAME="${SPANNER_TEST_DB}_${APP_PREFIX}" + else + TEST_DBNAME="${APP_PREFIX}_${RUN_ID}" + fi TEST_DBNAME_OTHER="${TEST_DBNAME}-ot" SETTINGS_FILE="${TEST_DBNAME}_settings" create_settings diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py index 1889aa16ef87..b951e3f2c475 100644 --- a/packages/django-google-spanner/noxfile.py +++ b/packages/django-google-spanner/noxfile.py @@ -232,11 +232,6 @@ def system_test(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): - # TODO(https://github.com/googleapis/google-cloud-python/issues/16099): - # System tests were disabled in the split repo to speed up the build process. - # See: https://github.com/googleapis/python-spanner-django/pull/929. - session.skip("System tests are temporarily disabled.") - print("System tests with django 5.2") system_test(session) diff --git a/packages/django-google-spanner/tests/settings.py b/packages/django-google-spanner/tests/settings.py index 77631946de8f..5eb4555bad94 100644 --- a/packages/django-google-spanner/tests/settings.py +++ b/packages/django-google-spanner/tests/settings.py @@ -5,6 +5,8 @@ # https://developers.google.com/open-source/licenses/bsd import os +import time +import uuid DEBUG = True USE_TZ = True @@ -29,7 +31,10 @@ PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") # _get_test_db_name method in creation.py addes prefix of 'test_' to db name. -DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db") +DATABASE_NAME = os.getenv( + "DJANGO_SPANNER_DB", + f"dj_sys_{int(time.time())}_{uuid.uuid4().hex[:6]}", +) DATABASES = { "default": { diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py index ae546f2cd736..46efaac0232f 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py @@ -25,6 +25,8 @@ @unittest.skipIf(USE_EMULATOR, "Check Constraint is not implemented in emulator.") class TestCheckConstraint(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py index 691a68882e29..1731c8b99b8a 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py @@ -20,6 +20,8 @@ class TestDecimal(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py index 396bd8c4cfb1..c47a8fadb9e6 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py @@ -22,6 +22,8 @@ @unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.") class TestJsonField(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py index 2db465917716..a5cb4ac409ee 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py @@ -19,6 +19,8 @@ class TestQueries(TransactionTestCase): + available_apps = [] + @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/utils.py b/packages/django-google-spanner/tests/system/django_spanner/utils.py index b4dd759570f8..5424403d56ec 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/utils.py +++ b/packages/django-google-spanner/tests/system/django_spanner/utils.py @@ -4,30 +4,37 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd +import atexit import os import time -from django.core.management import call_command -from django.db import connection +from django.conf import settings +from django.db import connections from google.api_core import exceptions from google.cloud.spanner_v1 import Client from google.cloud.spanner_v1.instance import Backup, Instance from test_utils.retry import RetryErrors from django_spanner import USE_EMULATOR -from django_spanner.creation import DatabaseCreation CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int( - os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60) + os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 240) ) EXISTING_INSTANCES = [] INSTANCE_ID = os.environ.get( - "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", "spanner-django-python-systest" + "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", + settings.DATABASES["default"]["INSTANCE"], +) +PROJECT_ID = os.getenv( + "GOOGLE_CLOUD_PROJECT", + settings.DATABASES["default"]["PROJECT"], +) +DATABASE_NAME = os.getenv( + "DJANGO_SPANNER_DB", + settings.DATABASES["default"]["NAME"], ) -PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") -DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db") class Config(object): @@ -48,12 +55,15 @@ def _list_instances(): def setup_instance(): + if Config.INSTANCE is not None: + return + if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials Config.CLIENT = Client(project=PROJECT_ID, credentials=AnonymousCredentials()) else: - Config.CLIENT = Client() + Config.CLIENT = Client(project=PROJECT_ID) retry = RetryErrors(exceptions.ServiceUnavailable) @@ -79,7 +89,8 @@ def setup_instance(): backup.delete() instance.delete() - if CREATE_INSTANCE: + Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) + if not Config.INSTANCE.exists(): if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. @@ -96,48 +107,54 @@ def setup_instance(): Config.INSTANCE = Config.CLIENT.instance( INSTANCE_ID, config_name, labels=labels ) - if not Config.INSTANCE.exists(): + try: created_op = Config.INSTANCE.create() created_op.result( SPANNER_OPERATION_TIMEOUT_IN_SECONDS ) # block until completion - else: + except exceptions.AlreadyExists: Config.INSTANCE.reload() - else: - Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload() def teardown_instance(): - if CREATE_INSTANCE: - Config.INSTANCE.delete() + pass def setup_database(): + if Config.DATABASE is not None: + return + Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) if not Config.DATABASE.exists(): - creation = DatabaseCreation(connection) - creation._create_test_db(verbosity=0, autoclobber=False, keepdb=True) - - # Running migrations on the db. - call_command("migrate", interactive=False) + try: + created_op = Config.DATABASE.create() + created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS) + except exceptions.AlreadyExists: + pass def teardown_database(): - from django.db import connections - connections.close_all() - Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) - if Config.DATABASE.exists(): - Config.DATABASE.drop() - # Clear the global client cache to avoid Stale Session errors - # when the next test class creates a new database. - try: - import django_spanner.base - - django_spanner.base._SPANNER_CLIENT_CACHE = None - except ImportError: - pass +def _cleanup_test_resources(): + connections.close_all() + if Config.DATABASE is not None: + try: + if Config.DATABASE.exists(): + Config.DATABASE.drop() + except Exception: + pass + Config.DATABASE = None + if CREATE_INSTANCE and Config.INSTANCE is not None: + try: + if Config.INSTANCE.exists(): + Config.INSTANCE.delete() + except Exception: + pass + Config.INSTANCE = None + + +atexit.register(_cleanup_test_resources) diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py b/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py index 520d268bce10..6b9b6c886526 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py @@ -347,3 +347,22 @@ def test_datetime_cast_sql(self): "TIMESTAMP(FORMAT_TIMESTAMP('%Y-%m-%d %R:%E9S %Z', field, 'UTC'))", ) self.assertIsNone(params) + + def test_adapt_json_value(self): + from django.db.models import JSONField + from google.cloud.spanner_v1 import JsonObject + + self.assertIsNone(self.db_operations.adapt_json_value(None, None)) + obj = JsonObject({"a": 1}) + self.assertIs(self.db_operations.adapt_json_value(obj, None), obj) + adapted_dict = self.db_operations.adapt_json_value({"a": 1}, None) + self.assertIsInstance(adapted_dict, JsonObject) + adapted_list = self.db_operations.adapt_json_value([1, 2], None) + self.assertIsInstance(adapted_list, JsonObject) + self.assertEqual(self.db_operations.adapt_json_value("str", None), '"str"') + + field = JSONField() + self.assertIs(field.from_db_value(obj, None, None), obj) + self.assertEqual(field.from_db_value({"a": 1}, None, None), {"a": 1}) + self.assertEqual(field.from_db_value('{"a": 1}', None, None), {"a": 1}) + self.assertIsNone(field.from_db_value(None, None, None)) From 206eb4f02e55c63aac122d9ea9ba4177132824f4 Mon Sep 17 00:00:00 2001 From: Sakthivel Subramanian <179120858+sakthivelmanii@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:09:07 +0000 Subject: [PATCH 3/3] Revert unrelated changes into separate PRs --- .../django_spanner/__init__.py | 12 --- .../django_spanner/operations.py | 10 --- packages/django-google-spanner/noxfile.py | 5 ++ .../django-google-spanner/tests/settings.py | 7 +- .../django_spanner/test_check_constraint.py | 2 - .../system/django_spanner/test_decimal.py | 2 - .../system/django_spanner/test_json_field.py | 2 - .../system/django_spanner/test_queries.py | 2 - .../tests/system/django_spanner/utils.py | 83 ++++++++----------- .../unit/django_spanner/test_operations.py | 19 ----- 10 files changed, 39 insertions(+), 105 deletions(-) diff --git a/packages/django-google-spanner/django_spanner/__init__.py b/packages/django-google-spanner/django_spanner/__init__.py index a442ed76c3e1..fc8cb56b2104 100644 --- a/packages/django-google-spanner/django_spanner/__init__.py +++ b/packages/django-google-spanner/django_spanner/__init__.py @@ -93,19 +93,7 @@ def get_prep_value(self, value): return value -old_jsonfield_from_db_value = getattr(JSONField, "from_db_value", None) - - -def jsonfield_from_db_value(self, value, expression, connection): - if isinstance(value, (dict, list, JsonObject)): - return value - if old_jsonfield_from_db_value: - return old_jsonfield_from_db_value(self, value, expression, connection) - return value - - JSONField.get_prep_value = get_prep_value -JSONField.from_db_value = jsonfield_from_db_value old_datetimewithnanoseconds_eq = getattr(DatetimeWithNanoseconds, "__eq__", None) diff --git a/packages/django-google-spanner/django_spanner/operations.py b/packages/django-google-spanner/django_spanner/operations.py index 55b340b7dee4..5a5b8e3e4612 100644 --- a/packages/django-google-spanner/django_spanner/operations.py +++ b/packages/django-google-spanner/django_spanner/operations.py @@ -20,7 +20,6 @@ TimestampStr, escape_name, ) -from google.cloud.spanner_v1 import JsonObject class DatabaseOperations(BaseDatabaseOperations): @@ -771,12 +770,3 @@ def savepoint_rollback_sql(self, sid): Return the SQL for rolling back to a savepoint. """ return "SELECT 1" - - def adapt_json_value(self, value, encoder): - if value is None: - return None - if isinstance(value, JsonObject): - return value - if isinstance(value, (dict, list)): - return JsonObject(value) - return super().adapt_json_value(value, encoder) diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py index b951e3f2c475..1889aa16ef87 100644 --- a/packages/django-google-spanner/noxfile.py +++ b/packages/django-google-spanner/noxfile.py @@ -232,6 +232,11 @@ def system_test(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): + # TODO(https://github.com/googleapis/google-cloud-python/issues/16099): + # System tests were disabled in the split repo to speed up the build process. + # See: https://github.com/googleapis/python-spanner-django/pull/929. + session.skip("System tests are temporarily disabled.") + print("System tests with django 5.2") system_test(session) diff --git a/packages/django-google-spanner/tests/settings.py b/packages/django-google-spanner/tests/settings.py index 5eb4555bad94..77631946de8f 100644 --- a/packages/django-google-spanner/tests/settings.py +++ b/packages/django-google-spanner/tests/settings.py @@ -5,8 +5,6 @@ # https://developers.google.com/open-source/licenses/bsd import os -import time -import uuid DEBUG = True USE_TZ = True @@ -31,10 +29,7 @@ PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") # _get_test_db_name method in creation.py addes prefix of 'test_' to db name. -DATABASE_NAME = os.getenv( - "DJANGO_SPANNER_DB", - f"dj_sys_{int(time.time())}_{uuid.uuid4().hex[:6]}", -) +DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db") DATABASES = { "default": { diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py index 46efaac0232f..ae546f2cd736 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_check_constraint.py @@ -25,8 +25,6 @@ @unittest.skipIf(USE_EMULATOR, "Check Constraint is not implemented in emulator.") class TestCheckConstraint(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py index 1731c8b99b8a..691a68882e29 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_decimal.py @@ -20,8 +20,6 @@ class TestDecimal(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py index c47a8fadb9e6..396bd8c4cfb1 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_json_field.py @@ -22,8 +22,6 @@ @unittest.skipIf(USE_EMULATOR, "Jsonfield is not implemented in emulator.") class TestJsonField(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py index a5cb4ac409ee..2db465917716 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/test_queries.py +++ b/packages/django-google-spanner/tests/system/django_spanner/test_queries.py @@ -19,8 +19,6 @@ class TestQueries(TransactionTestCase): - available_apps = [] - @classmethod def setUpClass(cls): setup_instance() diff --git a/packages/django-google-spanner/tests/system/django_spanner/utils.py b/packages/django-google-spanner/tests/system/django_spanner/utils.py index 5424403d56ec..b4dd759570f8 100644 --- a/packages/django-google-spanner/tests/system/django_spanner/utils.py +++ b/packages/django-google-spanner/tests/system/django_spanner/utils.py @@ -4,37 +4,30 @@ # license that can be found in the LICENSE file or at # https://developers.google.com/open-source/licenses/bsd -import atexit import os import time -from django.conf import settings -from django.db import connections +from django.core.management import call_command +from django.db import connection from google.api_core import exceptions from google.cloud.spanner_v1 import Client from google.cloud.spanner_v1.instance import Backup, Instance from test_utils.retry import RetryErrors from django_spanner import USE_EMULATOR +from django_spanner.creation import DatabaseCreation CREATE_INSTANCE = os.getenv("GOOGLE_CLOUD_TESTS_CREATE_SPANNER_INSTANCE") is not None SPANNER_OPERATION_TIMEOUT_IN_SECONDS = int( - os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 240) + os.getenv("SPANNER_OPERATION_TIMEOUT_IN_SECONDS", 60) ) EXISTING_INSTANCES = [] INSTANCE_ID = os.environ.get( - "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", - settings.DATABASES["default"]["INSTANCE"], -) -PROJECT_ID = os.getenv( - "GOOGLE_CLOUD_PROJECT", - settings.DATABASES["default"]["PROJECT"], -) -DATABASE_NAME = os.getenv( - "DJANGO_SPANNER_DB", - settings.DATABASES["default"]["NAME"], + "GOOGLE_CLOUD_TESTS_SPANNER_INSTANCE", "spanner-django-python-systest" ) +PROJECT_ID = os.getenv("GOOGLE_CLOUD_PROJECT", "emulator-test-project") +DATABASE_NAME = os.getenv("DJANGO_SPANNER_DB", "django_test_db") class Config(object): @@ -55,15 +48,12 @@ def _list_instances(): def setup_instance(): - if Config.INSTANCE is not None: - return - if USE_EMULATOR: from google.auth.credentials import AnonymousCredentials Config.CLIENT = Client(project=PROJECT_ID, credentials=AnonymousCredentials()) else: - Config.CLIENT = Client(project=PROJECT_ID) + Config.CLIENT = Client() retry = RetryErrors(exceptions.ServiceUnavailable) @@ -89,8 +79,7 @@ def setup_instance(): backup.delete() instance.delete() - Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) - if not Config.INSTANCE.exists(): + if CREATE_INSTANCE: if not USE_EMULATOR: # Defend against back-end returning configs for regions we aren't # actually allowed to use. @@ -107,54 +96,48 @@ def setup_instance(): Config.INSTANCE = Config.CLIENT.instance( INSTANCE_ID, config_name, labels=labels ) - try: + if not Config.INSTANCE.exists(): created_op = Config.INSTANCE.create() created_op.result( SPANNER_OPERATION_TIMEOUT_IN_SECONDS ) # block until completion - except exceptions.AlreadyExists: + else: Config.INSTANCE.reload() + else: + Config.INSTANCE = Config.CLIENT.instance(INSTANCE_ID) Config.INSTANCE.reload() def teardown_instance(): - pass + if CREATE_INSTANCE: + Config.INSTANCE.delete() def setup_database(): - if Config.DATABASE is not None: - return - Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) if not Config.DATABASE.exists(): - try: - created_op = Config.DATABASE.create() - created_op.result(SPANNER_OPERATION_TIMEOUT_IN_SECONDS) - except exceptions.AlreadyExists: - pass + creation = DatabaseCreation(connection) + creation._create_test_db(verbosity=0, autoclobber=False, keepdb=True) + + # Running migrations on the db. + call_command("migrate", interactive=False) def teardown_database(): + from django.db import connections + connections.close_all() + Config.DATABASE = Config.INSTANCE.database(DATABASE_NAME) + if Config.DATABASE.exists(): + Config.DATABASE.drop() -def _cleanup_test_resources(): - connections.close_all() - if Config.DATABASE is not None: - try: - if Config.DATABASE.exists(): - Config.DATABASE.drop() - except Exception: - pass - Config.DATABASE = None - if CREATE_INSTANCE and Config.INSTANCE is not None: - try: - if Config.INSTANCE.exists(): - Config.INSTANCE.delete() - except Exception: - pass - Config.INSTANCE = None - - -atexit.register(_cleanup_test_resources) + # Clear the global client cache to avoid Stale Session errors + # when the next test class creates a new database. + try: + import django_spanner.base + + django_spanner.base._SPANNER_CLIENT_CACHE = None + except ImportError: + pass diff --git a/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py b/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py index 6b9b6c886526..520d268bce10 100644 --- a/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py +++ b/packages/django-google-spanner/tests/unit/django_spanner/test_operations.py @@ -347,22 +347,3 @@ def test_datetime_cast_sql(self): "TIMESTAMP(FORMAT_TIMESTAMP('%Y-%m-%d %R:%E9S %Z', field, 'UTC'))", ) self.assertIsNone(params) - - def test_adapt_json_value(self): - from django.db.models import JSONField - from google.cloud.spanner_v1 import JsonObject - - self.assertIsNone(self.db_operations.adapt_json_value(None, None)) - obj = JsonObject({"a": 1}) - self.assertIs(self.db_operations.adapt_json_value(obj, None), obj) - adapted_dict = self.db_operations.adapt_json_value({"a": 1}, None) - self.assertIsInstance(adapted_dict, JsonObject) - adapted_list = self.db_operations.adapt_json_value([1, 2], None) - self.assertIsInstance(adapted_list, JsonObject) - self.assertEqual(self.db_operations.adapt_json_value("str", None), '"str"') - - field = JSONField() - self.assertIs(field.from_db_value(obj, None, None), obj) - self.assertEqual(field.from_db_value({"a": 1}, None, None), {"a": 1}) - self.assertEqual(field.from_db_value('{"a": 1}', None, None), {"a": 1}) - self.assertIsNone(field.from_db_value(None, None, None))