refactor: python imports (#1730)

refactor: unified python imports for consistency in local, production, and test environments

.circleci/config.yml

@@ -290,8 +290,8 @@ jobs:
       - setup-mysql
       - create-tokenserver-database
       # XXX: currently the time needed to setup-sccache negates its savings
-      #- setup-sccache
-      #- restore-sccache-cache
+      # - setup-sccache
+      # - restore-sccache-cache
       - write-version
       - install-test-deps
       - make-test-dir
@@ -310,7 +310,8 @@ jobs:
           source: workflow/test-results
           destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/coverage
           extension: json
-      #- save-sccache-cache
+      # - save-sccache-cache
+
   build-mysql-image:
     docker:
       - image: cimg/rust:1.86 # RUST_VER

Makefile

@@ -147,8 +147,9 @@ merge_coverage_results:
 
 .ONESHELL:
 run_token_server_integration_tests:
-	pip3 install -r tools/tokenserver/requirements.txt
-	pytest tools/tokenserver --junit-xml=${INTEGRATION_JUNIT_XML}
+	cd tools/tokenserver
+	poetry install --no-root --without dev
+	poetry run pytest tools/tokenserver --junit-xml=${INTEGRATION_JUNIT_XML}
 
 .PHONY: install
 install: $(INSTALL_STAMP)  ##  Install dependencies with poetry

docker-compose.e2e.mysql.yaml

@@ -41,8 +41,8 @@ services:
     entrypoint: >
       /bin/sh -c "
         exit_code=0;
-        pytest /app/tools/integration_tests/ --junit-xml=/mysql_integration_results.xml || exit_code=$$?;
+        PYTHONPATH=/app pytest /app/tools/integration_tests/ --junit-xml=/mysql_integration_results.xml || exit_code=$$?;
         export JWK_CACHE_DISABLED=true;
-        pytest /app/tools/integration_tests/ --junit-xml=/mysql_no_jwk_integration_results.xml || exit_code=$$?;
+        PYTHONPATH=/app pytest /app/tools/integration_tests/ --junit-xml=/mysql_no_jwk_integration_results.xml || exit_code=$$?;
         exit $$exit_code;
       "

docker-compose.e2e.spanner.yaml

@@ -41,8 +41,8 @@ services:
     entrypoint: >
       /bin/sh -c "
         exit_code=0;
-        pytest /app/tools/integration_tests/ --junit-xml=/spanner_integration_results.xml || exit_code=$$?;
+        PYTHONPATH=/app pytest /app/tools/integration_tests/ --junit-xml=/spanner_integration_results.xml || exit_code=$$?;
         export JWK_CACHE_DISABLED=true;
-        pytest /app/tools/integration_tests/ --junit-xml=/spanner_no_jwk_integration_results.xml || exit_code=$$?;
+        PYTHONPATH=/app pytest /app/tools/integration_tests/ --junit-xml=/spanner_no_jwk_integration_results.xml || exit_code=$$?;
         exit $$exit_code;
       "

tools/integration_tests/test_storage.py

@@ -30,7 +30,8 @@ import simplejson
 
 from pyramid.interfaces import IAuthenticationPolicy
 from webtest.app import AppError
-from test_support import StorageFunctionalTestCase
+
+from tools.integration_tests.test_support import StorageFunctionalTestCase
 
 import tokenlib
 

tools/integration_tests/test_support.py

@@ -1,7 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
-""" Base test class, with an instanciated app.
+""" Base test class, with an instantiated app.
 """
 
 import contextlib
@@ -537,10 +537,12 @@ class TokenServerAuthenticationPolicy(HawkAuthenticationPolicy):
         if "secrets_file" in settings:
             if "secret" in settings:
                 raise ValueError("can't use both 'secret' and 'secrets_file'")
-            secrets["backend"] = "test_support.Secrets"
+            secrets["backend"] = "tools.integration_tests.test_support.Secrets"
             secrets["filename"] = settings.pop("secrets_file")
         elif "secret" in settings:
-            secrets["backend"] = "test_support.FixedSecrets"
+            secrets["backend"] = (
+                "tools.integration_tests.test_support.FixedSecrets"
+                )
             secrets["secrets"] = settings.pop("secret")
         for name in settings.keys():
             if name.startswith(secrets_prefix):

tools/integration_tests/tokenserver/test_authorization.py

@@ -3,7 +3,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 import pytest
 import unittest
-from tokenserver.test_support import TestCase
+from integration_tests.tokenserver.test_support import TestCase
 
 
 @pytest.mark.usefixtures('setup_server_local_testing_with_oauth')

tools/integration_tests/tokenserver/test_e2e.py

@@ -21,7 +21,7 @@ from fxa.errors import ClientError, ServerError
 from fxa.tests.utils import TestEmailAccount
 from hashlib import sha256
 
-from tokenserver.test_support import TestCase
+from integration_tests.tokenserver.test_support import TestCase
 
 # This is the client ID used for Firefox Desktop. The FxA team confirmed that
 # this is the proper client ID to be using for these integration tests.

tools/integration_tests/tokenserver/test_misc.py

@@ -4,7 +4,7 @@
 import pytest
 import unittest
 
-from tokenserver.test_support import TestCase
+from integration_tests.tokenserver.test_support import TestCase
 
 MAX_GENERATION = 9223372036854775807
 

tools/integration_tests/tokenserver/test_node_assignment.py

@@ -4,7 +4,7 @@
 import pytest
 import unittest
 
-from tokenserver.test_support import TestCase
+from integration_tests.tokenserver.test_support import TestCase
 
 
 @pytest.mark.usefixtures('setup_server_local_testing_with_oauth')

tools/spanner/count_expired_rows.py

@@ -13,7 +13,7 @@ from statsd.defaults.env import statsd
 from urllib import parse
 
 from google.cloud import spanner
-from utils import ids_from_env
+from spanner.utils import ids_from_env
 
 # set up logger
 logging.basicConfig(

tools/spanner/purge_ttl.py

@@ -17,7 +17,7 @@ from google.cloud.spanner_v1.database import Database
 from google.cloud.spanner_v1 import param_types
 from statsd.defaults.env import statsd
 
-from utils import ids_from_env, Mode
+from spanner.utils import ids_from_env, Mode
 
 # set up logger
 logging.basicConfig(

tools/spanner/test_count_expired_rows.py

@@ -4,8 +4,8 @@ from unittest.mock import MagicMock
 import pytest
 import logging
 
-import count_expired_rows
-from utils import ids_from_env
+from spanner import count_expired_rows
+from spanner.utils import ids_from_env
 
 def test_spanner_read_data_counts_and_logs(monkeypatch, caplog):
     # Prepare mocks

tools/spanner/test_purge_ttl.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 import sys
 
 # Import the functions to test from purge_ttl.py
-import purge_ttl
+from spanner import purge_ttl
 
 def test_parse_args_list_single_item():
     assert purge_ttl.parse_args_list("foo") == ["foo"]
@@ -55,7 +55,7 @@ def test_add_conditions_with_prefix():
     assert params["prefix"] == "abc"
     assert types["prefix"] == purge_ttl.param_types.STRING
 
-@mock.patch("purge_ttl.statsd")
+@mock.patch("spanner.purge_ttl.statsd")
 def test_deleter_dryrun(statsd_mock):
     database = mock.Mock()
     statsd_mock.timer.return_value.__enter__.return_value = None
@@ -63,7 +63,7 @@ def test_deleter_dryrun(statsd_mock):
     purge_ttl.deleter(database, "batches", "DELETE FROM batches", dryrun=True)
     database.execute_partitioned_dml.assert_not_called()
 
-@mock.patch("purge_ttl.statsd")
+@mock.patch("spanner.purge_ttl.statsd")
 def test_deleter_executes(statsd_mock):
     database = mock.Mock()
     statsd_mock.timer.return_value.__enter__.return_value = None
@@ -73,10 +73,10 @@ def test_deleter_executes(statsd_mock):
     purge_ttl.deleter(database, "batches", "DELETE FROM batches", dryrun=False)
     database.execute_partitioned_dml.assert_called_once()
 
-@mock.patch("purge_ttl.deleter")
-@mock.patch("purge_ttl.add_conditions")
-@mock.patch("purge_ttl.get_expiry_condition")
-@mock.patch("purge_ttl.client")
+@mock.patch("spanner.purge_ttl.deleter")
+@mock.patch("spanner.purge_ttl.add_conditions")
+@mock.patch("spanner.purge_ttl.get_expiry_condition")
+@mock.patch("spanner.purge_ttl.client")
 def test_spanner_purge_both(client_mock, get_expiry_condition_mock, add_conditions_mock, deleter_mock):
     # Setup
     args = SimpleNamespace(

tools/spanner/test_utils.py

@@ -1,6 +1,6 @@
 import pytest
 
-import utils
+from spanner.utils import ids_from_env
 from unittest.mock import MagicMock
 
 @pytest.fixture(autouse=True)
@@ -18,7 +18,7 @@ def test_ids_from_env_parses_url(monkeypatch):
     """Test with passed in DSN"""
     monkeypatch.setenv("SYNC_SYNCSTORAGE__DATABASE_URL", "spanner://projects/proj/instances/inst/databases/db")
     dsn = "SYNC_SYNCSTORAGE__DATABASE_URL"
-    instance_id, database_id, project_id = utils.ids_from_env(dsn)
+    instance_id, database_id, project_id = ids_from_env(dsn)
     assert project_id == "proj"
     assert instance_id == "inst"
     assert database_id == "db"
@@ -28,7 +28,7 @@ def test_ids_from_env_with_missing_url(monkeypatch):
     monkeypatch.setenv("INSTANCE_ID", "foo")
     monkeypatch.setenv("DATABASE_ID", "bar")
     monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "baz")
-    instance_id, database_id, project_id = utils.ids_from_env()
+    instance_id, database_id, project_id = ids_from_env()
     assert instance_id == "foo"
     assert database_id == "bar"
     assert project_id == "baz"
@@ -40,7 +40,7 @@ def test_from_env_with_invalid_url(monkeypatch):
     monkeypatch.setenv("DATABASE_ID", "default-db")
     monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "default-proj")
 
-    instance_id, database_id, project_id = utils.ids_from_env()
+    instance_id, database_id, project_id = ids_from_env()
     assert instance_id == "default"
     assert database_id == "default-db"
     assert project_id == "default-proj"

tools/spanner/utils.py

@@ -4,11 +4,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-from enum import auto, Enum
 import os
+from enum import auto, Enum
 from urllib import parse
 from typing import Tuple
-from unittest.mock import MagicMock
 
 DSN_URL = "SYNC_SYNCSTORAGE__DATABASE_URL"
 """

tools/spanner/write_batch.py

@@ -21,7 +21,7 @@ from google.api_core.exceptions import AlreadyExists
 from google.cloud import spanner
 from google.cloud.spanner_v1 import param_types
 
-from utils import ids_from_env
+from spanner.utils import ids_from_env
 
 
 # max batch size for this write is 2000, otherwise we run into: