From aeedcf1e19e622b4f0d0e9c813ba4da3c712f125 Mon Sep 17 00:00:00 2001
From: Nick Shirley <nshirley@mozilla.com>
Date: Tue, 25 Mar 2025 16:33:49 -0600
Subject: [PATCH] chore: upload test artifacts to gcs

* chore: upload test results to gcs

* chore: use specific ete variables for gcp upload

* chore: override google_project_number to blank

* chore: use actual env var

* style: fixes flake8 E302 error

* docs: add documentation for new circleci vars

Closes SYNC-4584
---
 .circleci/config.yml                        | 47 +++++++++++++++++++++
 tools/tokenserver/pytest.ini                |  6 +++
 tools/tokenserver/test_database.py          |  2 +-
 tools/tokenserver/test_purge_old_records.py |  2 +
 4 files changed, 56 insertions(+), 1 deletion(-)
 create mode 100644 tools/tokenserver/pytest.ini

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5a0017a2..26356aee 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -4,7 +4,17 @@
 # DOCKER_USER    - login info for docker hub
 # DOCKER_PASS
 #
+# To avoid collision with other GCP connections we create specific vars for
+# the ETE Test Pipeline.
+# ETE_GOOGLE_PROJECT_ID     - GCP Project ID for ecosystem-test-eng
+# ETE_GCLOUD_SERVICE_KEY    - GCP syncstorage specific Service Account JSON Key 
+# ETE_GOOGLE_PROJECT_NUMBER - GCP Project Number for ecosystem-test-eng
+#
 version: 2.1
+
+orbs:
+  gcp-cli: circleci/gcp-cli@3.3.1
+
 commands:
   display-versions:
     steps:
@@ -163,6 +173,31 @@ commands:
           environment:
               SYNCSTORAGE_RS_IMAGE: app:build
 
+  upload-to-gcs:
+    parameters:
+      source:
+        type: string
+      destination:
+        type: string
+      extension:
+        type: enum
+        enum: ["xml", "json"]
+    steps:
+      - run:
+          name: Upload << parameters.source >> << parameters.extension >> Files to GCS
+          when: always  # Ensure the step runs even if previous steps, like test runs, fail
+          command: |
+            if [ "$CIRCLE_BRANCH" = "master" ]; then
+              FILES=$(ls -1 << parameters.source>>/*.<< parameters.extension>> )
+              if [ -z "$FILES" ]; then
+                echo "No << parameters.extension >> files found in << parameters.source >>/"
+                exit 1
+              fi
+              gsutil cp $FILES << parameters.destination >>
+            else
+              echo "Skipping artifact upload, not on 'master' branch."
+            fi
+
   setup-sccache:
     steps:
       - run:
@@ -227,6 +262,10 @@ jobs:
             MYSQL_DATABASE: syncstorage
     resource_class: large
     steps:
+      - gcp-cli/setup:
+          google_project_id: ETE_GOOGLE_PROJECT_ID
+          gcloud_service_key: ETE_GCLOUD_SERVICE_KEY
+          google_project_number: ETE_GOOGLE_PROJECT_NUMBER
       - checkout
       - display-versions
       - setup-python
@@ -245,6 +284,14 @@ jobs:
       # configured already, and it appears unit-tests modify the db to the expected state
       - run-tokenserver-integration-tests
       - store-test-results
+      - upload-to-gcs:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
+          extension: xml
+      - upload-to-gcs:
+          source: workflow/test-results
+          destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/coverage
+          extension: json
       #- save-sccache-cache
   build-mysql-image:
     docker:
diff --git a/tools/tokenserver/pytest.ini b/tools/tokenserver/pytest.ini
new file mode 100644
index 00000000..cd237d5c
--- /dev/null
+++ b/tools/tokenserver/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+markers =
+    migration_records: mark a test as a migration records test
+
+addopts = 
+    -m 'not migration_records'
\ No newline at end of file
diff --git a/tools/tokenserver/test_database.py b/tools/tokenserver/test_database.py
index b5714247..ef1e7560 100644
--- a/tools/tokenserver/test_database.py
+++ b/tools/tokenserver/test_database.py
@@ -43,7 +43,7 @@ class TestDatabase(unittest.TestCase):
 
     def test_node_allocation(self):
         user = self.database.get_user('test1@example.com')
-        self.assertEquals(user, None)
+        self.assertEqual(user, None)
 
         user = self.database.allocate_user('test1@example.com')
         wanted = 'https://phx12'
diff --git a/tools/tokenserver/test_purge_old_records.py b/tools/tokenserver/test_purge_old_records.py
index c804cbb9..bbd39b41 100644
--- a/tools/tokenserver/test_purge_old_records.py
+++ b/tools/tokenserver/test_purge_old_records.py
@@ -1,6 +1,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
+import pytest
 
 import hawkauthlib
 import re
@@ -189,6 +190,7 @@ class TestPurgeOldRecords(PurgeOldRecordsTestCase):
         self.assertEqual(len(self.service_requests), 0)
 
 
+@pytest.mark.migration_records
 class TestMigrationRecords(PurgeOldRecordsTestCase):
     """Test user records that were migrated from the old MySQL cluster of
     syncstorage nodes to a single Spanner node