test(e2e): run integration and e2e tests with pytest (#1697)

- Update docker compose steps for mysql and spanner to use pytest
- Add infra and configuration for pytest to run tests
- Remove old "run.py" test setup

Closes STOR-235

.circleci/config.yml

@@ -101,7 +101,7 @@ commands:
     steps:
       - run: 
           name: Install test dependencies
-          command: cargo install cargo-nextest cargo-llvm-cov
+          command: cargo install --locked cargo-nextest cargo-llvm-cov
 
   make-test-dir:
     steps:
@@ -145,32 +145,16 @@ commands:
                make run_token_server_integration_tests
           environment:
               SYNCSTORAGE_RS_IMAGE: app:build
-  run-e2e-mysql-tests:
+  run-e2e-tests:
+    parameters:
+      db:
+        type: enum
+        enum: ["mysql", "spanner"]
     steps:
       - run:
-          name: e2e tests (syncstorage mysql)
+          name: e2e tests (syncstorage << parameters.db >>)
           command: >
-               /usr/local/bin/docker-compose
+            make docker_run_<< parameters.db >>_e2e_tests
-               -f docker-compose.mysql.yaml
-               -f docker-compose.e2e.mysql.yaml
-               up
-               --exit-code-from mysql-e2e-tests
-               --abort-on-container-exit
-          environment:
-              SYNCSTORAGE_RS_IMAGE: app:build
-
-
-  run-e2e-spanner-tests:
-    steps:
-      - run:
-          name: e2e tests (syncstorage spanner)
-          command: >
-               /usr/local/bin/docker-compose
-               -f docker-compose.spanner.yaml
-               -f docker-compose.e2e.spanner.yaml
-               up
-               --exit-code-from spanner-e2e-tests
-               --abort-on-container-exit
           environment:
             SYNCSTORAGE_RS_IMAGE: app:build
 
@@ -283,7 +267,6 @@ jobs:
       # if the above tests don't run tokenserver-db tests (i.e. using --workspace)
       # then run-tokenserver-scripts-tests will fail. These tests expect the db to be
       # configured already, and it appears unit-tests modify the db to the expected state
-      - run-tokenserver-integration-tests
       - store-test-results
       - upload-to-gcs:
           source: workflow/test-results
@@ -324,11 +307,13 @@ jobs:
       - run:
           name: Save docker-compose config
           command: cp docker-compose*mysql.yaml /home/circleci/cache
+      - run:
+          name: Save Makefile to cache
+          command: cp Makefile /home/circleci/cache
       - save_cache:
           key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
           paths:
             - /home/circleci/cache
-
   build-spanner-image:
     docker:
       - image: cimg/rust:1.86  # RUST_VER
@@ -361,6 +346,9 @@ jobs:
       - run:
           name: Save docker-compose config
           command: cp docker-compose*spanner.yaml /home/circleci/cache
+      - run:
+          name: Save Makefile to cache
+          command: cp Makefile /home/circleci/cache
       - save_cache:
           key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
           paths:
@@ -422,7 +410,7 @@ jobs:
 
   mysql-e2e-tests:
     docker:
-      - image: docker/compose:1.24.0
+      - image: cimg/base:2025.04
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -434,14 +422,24 @@ jobs:
       - run:
           name: Restore Docker image cache
           command: docker load -i /home/circleci/cache/docker.tar
+      - run:
+          name: Restore Makefile from save_cache
+          command: cp /home/circleci/cache/Makefile .
       - run:
           name: Restore docker-compose config
           command: cp /home/circleci/cache/docker-compose*.yaml .
-      - run-e2e-mysql-tests
+      - make-test-dir
+      - run-e2e-tests:
+          db: mysql
+      - store-test-results
+      - upload-to-gcs:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
+          extension: xml
 
   spanner-e2e-tests:
     docker:
-      - image: docker/compose:1.24.0
+      - image: cimg/base:2025.04
         auth:
           username: $DOCKER_USER
           password: $DOCKER_PASS
@@ -453,10 +451,20 @@ jobs:
       - run:
           name: Restore Docker image cache
           command: docker load -i /home/circleci/cache/docker.tar
+      - run:
+          name: Restore Makefile from save_cache
+          command: cp /home/circleci/cache/Makefile .
       - run:
           name: Restore docker-compose config
           command: cp /home/circleci/cache/docker-compose*.yaml .
-      - run-e2e-spanner-tests
+      - make-test-dir
+      - run-e2e-tests:
+          db: spanner
+      - store-test-results
+      - upload-to-gcs:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
+          extension: xml
 
   deploy:
     docker:

Makefile

@@ -20,7 +20,13 @@ TEST_PROFILE := $(if $(CIRCLECI),ci,default)
 TEST_FILE_PREFIX := $(if $(CIRCLECI),$(CIRCLE_BUILD_NUM)__$(EPOCH_TIME)__$(CIRCLE_PROJECT_REPONAME)__$(WORKFLOW)__)
 UNIT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__results.xml
 UNIT_COVERAGE_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__coverage.json
-INTEGRATION_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__results.xml
+
+SPANNER_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_integration__results.xml
+SPANNER_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_no_oauth_integration__results.xml
+MYSQL_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_integration__results.xml
+MYSQL_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_no_oauth_integration__results.xml
+
+LOCAL_INTEGRATION_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)local_integration__results.xml
 SYNC_SYNCSTORAGE__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/syncstorage_rs
 SYNC_TOKENSERVER__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/tokenserver_rs
 
@@ -40,22 +46,48 @@ clean:
 	rm -r venv
 
 docker_start_mysql:
-	docker-compose -f docker-compose.mysql.yaml up -d
+	docker compose -f docker-compose.mysql.yaml up -d
 
 docker_start_mysql_rebuild:
-	docker-compose -f docker-compose.mysql.yaml up --build -d
+	docker compose -f docker-compose.mysql.yaml up --build -d
 
 docker_stop_mysql:
-	docker-compose -f docker-compose.mysql.yaml down
+	docker compose -f docker-compose.mysql.yaml down
 
 docker_start_spanner:
-	docker-compose -f docker-compose.spanner.yaml up -d
+	docker compose -f docker-compose.spanner.yaml up -d
 
 docker_start_spanner_rebuild:
-	docker-compose -f docker-compose.spanner.yaml up --build -d
+	docker compose -f docker-compose.spanner.yaml up --build -d
 
 docker_stop_spanner:
-	docker-compose -f docker-compose.spanner.yaml down
+	docker compose -f docker-compose.spanner.yaml down
+
+.ONESHELL:
+docker_run_mysql_e2e_tests:
+	docker compose \
+		-f docker-compose.mysql.yaml \
+		-f docker-compose.e2e.mysql.yaml \
+	 	up \
+	 	--exit-code-from mysql-e2e-tests \
+	 	--abort-on-container-exit;
+	exit_code=$$?;
+	docker cp mysql-e2e-tests:/mysql_integration_results.xml ${MYSQL_INT_JUNIT_XML};
+	docker cp mysql-e2e-tests:/mysql_no_jwk_integration_results.xml ${MYSQL_NO_JWK_INT_JUNIT_XML};
+	exit $$exit_code;
+
+.ONESHELL:
+docker_run_spanner_e2e_tests:
+	docker compose \
+		-f docker-compose.spanner.yaml \
+		-f docker-compose.e2e.spanner.yaml \
+	 	up \
+	 	--exit-code-from spanner-e2e-tests \
+	 	--abort-on-container-exit; 
+	exit_code=$$?;
+	docker cp spanner-e2e-tests:/spanner_integration_results.xml ${SPANNER_INT_JUNIT_XML};
+	docker cp spanner-e2e-tests:/spanner_no_jwk_integration_results.xml ${SPANNER_NO_JWK_INT_JUNIT_XML};
+	exit $$exit_code;
 
 python:
 	python3 -m venv venv

docker-compose.e2e.mysql.yaml

@@ -1,27 +1,24 @@
 version: "3"
 services:
-  sync-db:
-  tokenserver-db:
-  syncserver:
-    depends_on:
-      - sync-db
-      - tokenserver-db
-    # TODO: either syncserver should retry the db connection
-    # itself a few times or should include a wait-for-it.sh script
-    # inside its docker that would do this for us.
-    entrypoint: >
-      /bin/sh -c "
-        sleep 15;
-        /app/bin/syncserver;
-      "
   mysql-e2e-tests:
+    container_name: mysql-e2e-tests
     depends_on:
-      - mock-fxa-server
+      sync-db:
-      - syncserver
+        condition: service_healthy
+      mock-fxa-server:
+        condition: service_started
+      tokenserver-db:
+        condition: service_healthy
+    # this depend is to avoid migration collisions.
+    # the syncserver isn't actually used for the tests,
+    # but collisions can happen particularly in CI.
+      syncserver:
+        condition: service_started
     image: app:build
     privileged: true
     user: root
     environment:
+      JWK_CACHE_DISABLED: false
       MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000
       SYNC_HOST: 0.0.0.0
       SYNC_MASTER_SECRET: secret0
@@ -43,5 +40,9 @@ services:
       TOKENSERVER_HOST: http://localhost:8000
     entrypoint: >
       /bin/sh -c "
-        sleep 28; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
+        exit_code=0;
+        pytest /app/tools/integration_tests/ --junit-xml=/mysql_integration_results.xml || exit_code=$$?;
+        export JWK_CACHE_DISABLED=true;
+        pytest /app/tools/integration_tests/ --junit-xml=/mysql_no_jwk_integration_results.xml || exit_code=$$?;
+        exit $$exit_code;
       "

docker-compose.e2e.spanner.yaml

@@ -1,27 +1,23 @@
 version: "3"
 services:
-  sync-db:
-  sync-db-setup:
-  tokenserver-db:
-  syncserver:
-    depends_on:
-      - sync-db-setup
-    # TODO: either syncserver should retry the db connection
-    # itself a few times or should include a wait-for-it.sh script
-    # inside its docker that would do this for us.
-    entrypoint: >
-      /bin/sh -c "
-        sleep 15;
-        /app/bin/syncserver;
-      "
   spanner-e2e-tests:
+    container_name: spanner-e2e-tests
     depends_on:
-      - mock-fxa-server
+      mock-fxa-server:
-      - syncserver
+        condition: service_started
+      syncserver:
+        condition: service_started
+      tokenserver-db:
+        condition: service_healthy
     image: app:build
     privileged: true
     user: root
     environment:
+      # Some tests can run without the `FXA_OAUTH...` vars.
+      # Setting this to false will delete any of those keys before starting
+      # the syncserver and startging the test. This can be set/passed
+      # in from CircleCI when calling `docker-compose -f docker-compose.e2e.spanner.yaml`
+      JWK_CACHE_DISABLED: false
       MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000
       SYNC_HOST: 0.0.0.0
       SYNC_MASTER_SECRET: secret0
@@ -44,5 +40,9 @@ services:
       TOKENSERVER_HOST: http://localhost:8000
     entrypoint: >
       /bin/sh -c "
-        sleep 28; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
+        exit_code=0;
+        pytest /app/tools/integration_tests/ --junit-xml=/spanner_integration_results.xml || exit_code=$$?;
+        export JWK_CACHE_DISABLED=true;
+        pytest /app/tools/integration_tests/ --junit-xml=/spanner_no_jwk_integration_results.xml || exit_code=$$?;
+        exit $$exit_code;
       "

docker-compose.mysql.yaml

@@ -24,6 +24,12 @@ services:
       MYSQL_DATABASE: syncstorage
       MYSQL_USER: test
       MYSQL_PASSWORD: test
+    healthcheck:
+      test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"]
+      interval: 2s
+      retries: 10
+      start_period: 20s
+      timeout: 2s
 
   tokenserver-db:
     image: docker.io/library/mysql:5.7
@@ -39,6 +45,12 @@ services:
       MYSQL_DATABASE: tokenserver
       MYSQL_USER: test
       MYSQL_PASSWORD: test
+    healthcheck:
+      test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"]
+      interval: 2s
+      retries: 10
+      start_period: 20s
+      timeout: 2s
 
   mock-fxa-server:
     image: app:build
@@ -58,8 +70,10 @@ services:
     ports:
       - "8000:8000"
     depends_on:
-      - sync-db
+      sync-db:
-      - tokenserver-db
+        condition: service_healthy
+      tokenserver-db:
+        condition: service_healthy
     environment:
       SYNC_HOST: 0.0.0.0
       SYNC_MASTER_SECRET: secret0

docker-compose.spanner.yaml

@@ -35,6 +35,12 @@ services:
       MYSQL_DATABASE: tokenserver
       MYSQL_USER: test
       MYSQL_PASSWORD: test
+    healthcheck:
+      test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"]
+      interval: 2s
+      retries: 10
+      start_period: 20s
+      timeout: 2s
   mock-fxa-server:
     image: app:build
     restart: "no"

tools/integration_tests/conftest.py

@@ -0,0 +1,162 @@
+import os
+import psutil
+import signal
+import subprocess
+import time
+import pytest
+import requests
+import logging
+
+DEBUG_BUILD = "target/debug/syncserver"
+RELEASE_BUILD = "/app/bin/syncserver"
+# max number of attempts to check server heartbeat
+SYNC_SERVER_STARTUP_MAX_ATTEMPTS = 30
+JWK_CACHE_DISABLED = os.environ.get("JWK_CACHE_DISABLED")
+
+logger = logging.getLogger("tokenserver.scripts.conftest")
+
+# Local setup for fixtures
+
+
+def _terminate_process(process):
+    """
+    Gracefully terminate the process and its children.
+    """
+    proc = psutil.Process(pid=process.pid)
+    child_proc = proc.children(recursive=True)
+    for p in [proc] + child_proc:
+        os.kill(p.pid, signal.SIGTERM)
+    process.wait()
+
+
+def _wait_for_server_startup(max_attempts=SYNC_SERVER_STARTUP_MAX_ATTEMPTS):
+    """
+    Waits for the __heartbeat__ endpoint to return a 200, pausing for 1 second
+    between attempts. Raises a RuntimeError if the server does not start after
+    the specific number of attempts.
+    """
+    itter = 0
+    while True:
+        if itter >= max_attempts:
+            raise RuntimeError(
+                "Server failed to start within the timeout period."
+            )
+        try:
+            req = requests.get("http://localhost:8000/__heartbeat__",
+                               timeout=2)
+            if req.status_code == 200:
+                break
+        except requests.exceptions.RequestException as e:
+            logger.warning("Connection failed: %s", e)
+        time.sleep(1)
+        itter += 1
+
+
+def _start_server():
+    """
+    Starts the syncserver process, waits for it to be running,
+    and return the process handle.
+    """
+    target_binary = None
+    if os.path.exists(DEBUG_BUILD):
+        target_binary = DEBUG_BUILD
+    elif os.path.exists(RELEASE_BUILD):
+        target_binary = RELEASE_BUILD
+    else:
+        raise RuntimeError(
+            "Neither {DEBUG_BUILD} nor {RELEASE_BUILD} were found."
+        )
+
+    server_proc = subprocess.Popen(
+        target_binary,
+        shell=True,
+        text=True,
+        env=os.environ,
+    )
+
+    _wait_for_server_startup()
+
+    return server_proc
+
+
+def _server_manager():
+    """
+    Context manager to gracefully start and stop the server.
+    """
+    server_process = _start_server()
+    try:
+        yield server_process
+    finally:
+        _terminate_process(server_process)
+
+
+def _set_local_test_env_vars():
+    """
+    Set environment variables for local testing.
+    This function sets the necessary environment variables for the syncserver.
+    """
+    os.environ.setdefault("SYNC_MASTER_SECRET", "secret0")
+    os.environ.setdefault("SYNC_CORS_MAX_AGE", "555")
+    os.environ.setdefault("SYNC_CORS_ALLOWED_ORIGIN", "*")
+    os.environ["MOZSVC_TEST_REMOTE"] = "localhost"
+    os.environ["SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL"] = \
+        os.environ["MOCK_FXA_SERVER_URL"]
+
+# Fixtures
+
+
+@pytest.fixture(scope="session")
+def setup_server_local_testing():
+    """
+    Fixture to set up the server for local testing.
+    This fixture sets the necessary environment variables and
+    starts the server.
+    """
+    _set_local_test_env_vars()
+    yield from _server_manager()
+
+
+@pytest.fixture(scope="session")
+def setup_server_local_testing_with_oauth():
+    """
+    Fixture to set up the server for local testing with OAuth.
+    This fixture sets the necessary environment variables and
+    starts the server.
+    """
+    _set_local_test_env_vars()
+
+    # Set OAuth-specific environment variables
+    os.environ["TOKENSERVER_AUTH_METHOD"] = "oauth"
+
+    # Start the server
+    yield from _server_manager()
+
+
+@pytest.fixture(scope="session")
+def setup_server_end_to_end_testing():
+    """
+    Fixture to set up the server for end-to-end testing.
+    This fixture sets the necessary environment variables and
+    starts the server.
+    """
+    _set_local_test_env_vars()
+    # debatable if this should ONLY be here since it was only
+    # done against the "run_end_to_end_tests" prior, of if we
+    # just do it in _set_local_test_env_vars...
+    if JWK_CACHE_DISABLED:
+        del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KTY"]
+        del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__ALG"]
+        del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KID"]
+        del os.environ[
+            "SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__FXA_CREATED_AT"
+        ]
+        del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__USE"]
+        del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__N"]
+        del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__E"]
+
+    # Set OAuth-specific environment variables
+    os.environ["SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL"] = \
+        "https://oauth.stage.mozaws.net"
+
+    # Start the server
+    yield from _server_manager()

tools/integration_tests/run.py

@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-
-import os.path
-import psutil
-import signal
-import subprocess
-import sys
-from test_storage import TestStorage
-from test_support import run_live_functional_tests
-import time
-from tokenserver.run import (run_end_to_end_tests, run_local_tests)
-
-DEBUG_BUILD = "target/debug/syncserver"
-RELEASE_BUILD = "/app/bin/syncserver"
-
-
-def terminate_process(process):
-    proc = psutil.Process(pid=process.pid)
-    child_proc = proc.children(recursive=True)
-    for p in [proc] + child_proc:
-        os.kill(p.pid, signal.SIGTERM)
-    process.wait()
-
-
-if __name__ == "__main__":
-    # When run as a script, this file will execute the
-    # functional tests against a live webserver.
-    target_binary = None
-    if os.path.exists(DEBUG_BUILD):
-        target_binary = DEBUG_BUILD
-    elif os.path.exists(RELEASE_BUILD):
-        target_binary = RELEASE_BUILD
-    else:
-        raise RuntimeError(
-            "Neither target/debug/syncserver \
-                nor /app/bin/syncserver were found."
-        )
-
-    def start_server():
-        the_server_subprocess = subprocess.Popen(
-            target_binary, shell=True, env=os.environ
-        )
-
-        # TODO we should change this to watch for a log message on startup
-        # to know when to continue instead of sleeping for a fixed amount
-        time.sleep(20)
-
-        return the_server_subprocess
-
-    os.environ.setdefault("SYNC_MASTER_SECRET", "secret0")
-    os.environ.setdefault("SYNC_CORS_MAX_AGE", "555")
-    os.environ.setdefault("SYNC_CORS_ALLOWED_ORIGIN", "*")
-    mock_fxa_server_url = os.environ["MOCK_FXA_SERVER_URL"]
-    url = "%s/v2" % mock_fxa_server_url
-    os.environ["SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL"] = mock_fxa_server_url
-    the_server_subprocess = start_server()
-    try:
-        res = 0
-        res |= run_live_functional_tests(TestStorage, sys.argv)
-        os.environ["TOKENSERVER_AUTH_METHOD"] = "oauth"
-        res |= run_local_tests()
-    finally:
-        terminate_process(the_server_subprocess)
-
-    os.environ["SYNC_TOKENSERVER__FXA_OAUTH_SERVER_URL"] = \
-        "https://oauth.stage.mozaws.net"
-    the_server_subprocess = start_server()
-    try:
-        res |= run_end_to_end_tests()
-    finally:
-        terminate_process(the_server_subprocess)
-
-    # Run the Tokenserver end-to-end tests without the JWK cached
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KTY"]
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__ALG"]
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__KID"]
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__FXA_CREATED_AT"]
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__USE"]
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__N"]
-    del os.environ["SYNC_TOKENSERVER__FXA_OAUTH_PRIMARY_JWK__E"]
-
-    the_server_subprocess = start_server()
-    try:
-        verbosity = int(os.environ.get("VERBOSITY", "1"))
-        res |= run_end_to_end_tests(verbosity=verbosity)
-    finally:
-        terminate_process(the_server_subprocess)
-
-    sys.exit(res)

tools/integration_tests/test_support.py

@@ -9,7 +9,6 @@ import functools
 from konfig import Config, SettingsDict
 import hawkauthlib
 import os
-import optparse
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.interfaces import IAuthenticationPolicy
@@ -32,7 +31,6 @@ from webtest import TestApp
 from zope.interface import implementer
 
 
-global_secret = None
 VALID_FXA_ID_REGEX = re.compile("^[A-Za-z0-9=\\-_]{1,64}$")
 
 
@@ -357,7 +355,6 @@ class FunctionalTestCase(TestCase):
         # This call implicitly commits the configurator. We probably still
         # want it for the side effects.
         self.config.make_wsgi_app()
-
         host_url = urlparse.urlparse(self.host_url)
         self.app = TestApp(
             self.host_url,
@@ -397,6 +394,7 @@ class StorageFunctionalTestCase(FunctionalTestCase, StorageTestCase):
 
     def _authenticate(self):
         policy = self.config.registry.getUtility(IAuthenticationPolicy)
+        global_secret = os.environ.get("SYNC_MASTER_SECRET")
         if global_secret is not None:
             policy.secrets._secrets = [global_secret]
         self.user_id = random.randint(1, 100000)
@@ -801,90 +799,3 @@ class SyncStorageAuthenticationPolicy(TokenServerAuthenticationPolicy):
                 raise ValueError("invalid device_id in token data")
         """
         return user
-
-
-def run_live_functional_tests(TestCaseClass, argv=None):
-    """Execute the given suite of testcases against a live server."""
-    if argv is None:
-        argv = sys.argv
-
-    # This will only work using a StorageFunctionalTestCase subclass,
-    # since we override the _authenticate() method.
-    assert issubclass(TestCaseClass, StorageFunctionalTestCase)
-
-    usage = "Usage: %prog [options] <server-url>"
-    parser = optparse.OptionParser(usage=usage)
-    parser.add_option(
-        "-x",
-        "--failfast",
-        action="store_true",
-        help="stop after the first failed test",
-    )
-    parser.add_option(
-        "",
-        "--config-file",
-        help="name of the config file in use by the server",
-    )
-    parser.add_option(
-        "",
-        "--use-token-server",
-        action="store_true",
-        help="the given URL is a tokenserver, not an endpoint",
-    )
-    parser.add_option(
-        "", "--email", help="email address to use for tokenserver tests"
-    )
-    parser.add_option(
-        "",
-        "--audience",
-        help="assertion audience to use for tokenserver tests",
-    )
-
-    try:
-        opts, args = parser.parse_args(argv)
-    except SystemExit as e:
-        return e.args[0]
-    if len(args) != 2:
-        parser.print_usage()
-        return 2
-
-    url = args[1]
-    if opts.config_file is not None:
-        os.environ["MOZSVC_TEST_INI_FILE"] = opts.config_file
-
-    # If we're not using the tokenserver, the default implementation of
-    # _authenticate will do just fine.  We optionally accept the token
-    # signing secret in the url hash fragement.
-    if opts.email is not None:
-        msg = "cant specify email address unless using live tokenserver"
-        raise ValueError(msg)
-    if opts.audience is not None:
-        msg = "cant specify audience unless using live tokenserver"
-        raise ValueError(msg)
-    host_url = urlparse.urlparse(url)
-    if host_url.fragment:
-        global global_secret
-        global_secret = host_url.fragment
-        host_url = host_url._replace(fragment="")
-    os.environ["MOZSVC_TEST_REMOTE"] = "localhost"
-
-    # Now use the unittest2 runner to execute them.
-    suite = unittest.TestSuite()
-    import test_storage
-
-    test_prefix = os.environ.get("SYNC_TEST_PREFIX", "test")
-    suite.addTest(unittest.findTestCases(test_storage, test_prefix))
-    # suite.addTest(unittest.makeSuite(LiveTestCases, prefix=test_prefix))
-    runner = unittest.TextTestRunner(
-        stream=sys.stderr,
-        failfast=opts.failfast,
-        verbosity=2,
-    )
-    res = runner.run(suite)
-    if not res.wasSuccessful():
-        return 1
-    return 0
-
-
-# Tell over-zealous test discovery frameworks that this isn't a real test.
-run_live_functional_tests.__test__ = False

tools/integration_tests/tokenserver/run.py

@@ -1,35 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-import unittest
-
-from tokenserver.test_authorization import TestAuthorization
-from tokenserver.test_e2e import TestE2e
-from tokenserver.test_misc import TestMisc
-from tokenserver.test_node_assignment import TestNodeAssignment
-
-
-def run_local_tests():
-    test_classes = [TestAuthorization, TestMisc, TestNodeAssignment]
-
-    return run_tests(test_classes)
-
-
-def run_end_to_end_tests(verbosity=1):
-    return run_tests([TestE2e], verbosity=verbosity)
-
-
-def run_tests(test_cases, verbosity=1):
-    loader = unittest.TestLoader()
-    success = True
-
-    for test_case in test_cases:
-        suite = loader.loadTestsFromTestCase(test_case)
-        runner = unittest.TextTestRunner(verbosity=verbosity)
-        res = runner.run(suite)
-        success = success and res.wasSuccessful()
-
-    if success:
-        return 0
-    else:
-        return 1

tools/integration_tests/tokenserver/test_authorization.py

@@ -1,10 +1,12 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
+import pytest
 import unittest
 from tokenserver.test_support import TestCase
 
 
+@pytest.mark.usefixtures('setup_server_local_testing_with_oauth')
 class TestAuthorization(TestCase, unittest.TestCase):
     def setUp(self):
         super(TestAuthorization, self).setUp()
@@ -370,15 +372,15 @@ class TestAuthorization(TestCase, unittest.TestCase):
                                            client_state='aaaa')
         # It's ok to request a shorter-duration token.
         res = self.app.get('/1.0/sync/1.5?duration=12', headers=headers)
-        self.assertEquals(res.json['duration'], 12)
+        self.assertEqual(res.json['duration'], 12)
         # But you can't exceed the server's default value.
         res = self.app.get('/1.0/sync/1.5?duration=4000', headers=headers)
-        self.assertEquals(res.json['duration'], 3600)
+        self.assertEqual(res.json['duration'], 3600)
         # And nonsense values are ignored.
         res = self.app.get('/1.0/sync/1.5?duration=lolwut', headers=headers)
-        self.assertEquals(res.json['duration'], 3600)
+        self.assertEqual(res.json['duration'], 3600)
         res = self.app.get('/1.0/sync/1.5?duration=-1', headers=headers)
-        self.assertEquals(res.json['duration'], 3600)
+        self.assertEqual(res.json['duration'], 3600)
 
     # Although all servers are now writing keys_changed_at, we still need this
     # case to be handled. See this PR for more information:

tools/integration_tests/tokenserver/test_e2e.py

@@ -5,6 +5,7 @@ from base64 import urlsafe_b64decode
 import hmac
 import json
 import jwt
+import pytest
 import random
 import string
 import time
@@ -33,6 +34,7 @@ PASSWORD_LENGTH = 32
 SCOPE = 'https://identity.mozilla.com/apps/oldsync'
 
 
+@pytest.mark.usefixtures('setup_server_end_to_end_testing')
 class TestE2e(TestCase, unittest.TestCase):
 
     def setUp(self):

tools/integration_tests/tokenserver/test_misc.py

@@ -1,6 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
+import pytest
 import unittest
 
 from tokenserver.test_support import TestCase
@@ -8,6 +9,7 @@ from tokenserver.test_support import TestCase
 MAX_GENERATION = 9223372036854775807
 
 
+@pytest.mark.usefixtures('setup_server_local_testing_with_oauth')
 class TestMisc(TestCase, unittest.TestCase):
     def setUp(self):
         super(TestMisc, self).setUp()
@@ -57,7 +59,7 @@ class TestMisc(TestCase, unittest.TestCase):
         res = self.app.get('/1.0/sync/1.5', headers=headers)
         self.assertIn('https://example.com/1.5', res.json['api_endpoint'])
         self.assertIn('duration', res.json)
-        self.assertEquals(res.json['duration'], 3600)
+        self.assertEqual(res.json['duration'], 3600)
 
     def test_current_user_is_the_most_up_to_date(self):
         # Add some users

tools/integration_tests/tokenserver/test_node_assignment.py

@@ -1,11 +1,13 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
+import pytest
 import unittest
 
 from tokenserver.test_support import TestCase
 
 
+@pytest.mark.usefixtures('setup_server_local_testing_with_oauth')
 class TestNodeAssignment(TestCase, unittest.TestCase):
     def setUp(self):
         super(TestNodeAssignment, self).setUp()