Chore/update 2312 (#1510)

* chore: Update for 12-2023

Closes: #1503, #1502

switch to fetching mysql_pubkey.asc

Closes #1509 SYNC-4055

Cargo.toml

@@ -43,6 +43,7 @@ lazy_static = "1.4"
 protobuf = "=2.25.2" # pin to 2.25.2 to prevent side updating
 rand = "0.8"
 regex = "1.4"
+reqwest = "^0.10" # 0.11+ requires tokio v1.34
 sentry = { version = "0.31", features = ["curl"] }
 sentry-backtrace = "0.31"
 serde = "1.0"
@@ -60,6 +61,7 @@ slog-mozlog-json = "0.1"
 slog-scope = "4.3"
 slog-stdlog = "4.1"
 slog-term = "2.6"
+tokio = "^0.2.4"
 
 [profile.release]
 # Enables line numbers in Sentry reporting

Dockerfile

@@ -8,16 +8,13 @@ RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS cacher
 ARG DATABASE_BACKEND=spanner
-COPY --from=planner /app/mysql_pubkey.asc mysql_pubkey.asc
 
 # cmake is required to build grpcio-sys for Spanner builds
 RUN \
+    # Fetch and load the MySQL public key. We need to install libmysqlclient-dev to build syncstorage-rs
+    # which wants the mariadb
+    wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
     echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
-    # mysql_pubkey.asc from:
-    # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html
-    # related:
-    # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup
-    apt-key adv --import mysql_pubkey.asc && \
     apt-get -q update && \
     apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake
 
@@ -32,12 +29,13 @@ COPY --from=cacher /app/target /app/target
 COPY --from=cacher $CARGO_HOME /app/$CARGO_HOME
 
 RUN \
+    # Fetch and load the MySQL public key
+    wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
     echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
     # mysql_pubkey.asc from:
     # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html
     # related:
     # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup
-    apt-key adv --import mysql_pubkey.asc && \
     apt-get -q update && \
     apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel && \
     pip3 install -r requirements.txt && \
@@ -54,11 +52,14 @@ RUN \
 FROM docker.io/library/debian:bullseye-slim
 WORKDIR /app
 COPY --from=builder /app/requirements.txt /app
-COPY --from=builder /app/mysql_pubkey.asc /app
 # Due to a build error that occurs with the Python cryptography package, we
 # have to set this env var to prevent the cryptography package from building
 # with Rust. See this link for more information:
 # https://pythonshowcase.com/question/problem-installing-cryptography-on-raspberry-pi
+
+RUN \
+    apt-get -q update && apt-get -qy install wget
+
 ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
 RUN \
     groupadd --gid 10001 app && \
@@ -66,11 +67,12 @@ RUN \
     # first, an apt-get update is required for gnupg, which is required for apt-key adv
     apt-get -q update && \
     # and ca-certificates needed for https://repo.mysql.com
-    apt-get install -y gnupg ca-certificates && \
+    apt-get install -y gnupg ca-certificates wget && \
     echo "deb https://repo.mysql.com/apt/debian/ bullseye mysql-8.0" >> /etc/apt/sources.list && \
-    apt-key adv --import mysql_pubkey.asc && \
+    wget -qO- https://repo.mysql.com/RPM-GPG-KEY-mysql-2023 > /etc/apt/trusted.gpg.d/mysql.asc && \
     # update again now that we trust repo.mysql.com
     apt-get -q update && \
+    # Fetch and load the MySQL public key
     apt-get -q install -y build-essential libmysqlclient-dev libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel cargo curl jq && \
     # The python3-cryptography debian package installs version 2.6.1, but we
     # we want to use the version specified in requirements.txt. To do this,

docker-compose.e2e.mysql.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 services:
   sync-db:
   tokenserver-db:

docker-compose.e2e.spanner.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 services:
   sync-db:
   sync-db-setup:

mysql_pubkey.asc

@@ -1,49 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: SKS 1.1.6
-Comment: Hostname: pgp.mit.edu
-
-mQINBGG4urcBEACrbsRa7tSSyxSfFkB+KXSbNM9rxYqoB78u107skReefq4/+Y72TpDvlDZL
-mdv/lK0IpLa3bnvsM9IE1trNLrfi+JES62kaQ6hePPgn2RqxyIirt2seSi3Z3n3jlEg+mSdh
-AvW+b+hFnqxo+TY0U+RBwDi4oO0YzHefkYPSmNPdlxRPQBMv4GPTNfxERx6XvVSPcL1+jQ4R
-2cQFBryNhidBFIkoCOszjWhm+WnbURsLheBp757lqEyrpCufz77zlq2gEi+wtPHItfqsx3rz
-xSRqatztMGYZpNUHNBJkr13npZtGW+kdN/xu980QLZxN+bZ88pNoOuzD6dKcpMJ0LkdUmTx5
-z9ewiFiFbUDzZ7PECOm2g3veJrwr79CXDLE1+39Hr8rDM2kDhSr9tAlPTnHVDcaYIGgSNIBc
-YfLmt91133klHQHBIdWCNVtWJjq5YcLQJ9TxG9GQzgABPrm6NDd1t9j7w1L7uwBvMB1wgpir
-RTPVfnUSCd+025PEF+wTcBhfnzLtFj5xD7mNsmDmeHkF/sDfNOfAzTE1v2wq0ndYU60xbL6/
-yl/Nipyr7WiQjCG0m3WfkjjVDTfs7/DXUqHFDOu4WMF9v+oqwpJXmAeGhQTWZC/QhWtrjrNJ
-AgwKpp263gDSdW70ekhRzsok1HJwX1SfxHJYCMFs2aH6ppzNsQARAQABtDZNeVNRTCBSZWxl
-YXNlIEVuZ2luZWVyaW5nIDxteXNxbC1idWlsZEBvc3Mub3JhY2xlLmNvbT6JAlQEEwEIAD4W
-IQSFm+jXxYb1OEMLGcJGe5QtOnm9KQUCYbi6twIbAwUJA8JnAAULCQgHAgYVCgkICwIEFgID
-AQIeAQIXgAAKCRBGe5QtOnm9KUewD/992sS31WLGoUQ6NoL7qOB4CErkqXtMzpJAKKg2jtBG
-G3rKE1/0VAg1D8AwEK4LcCO407wohnH0hNiUbeDck5x20pgS5SplQpuXX1K9vPzHeL/WNTb9
-8S3H2Mzj4o9obED6Ey52tTupttMF8pC9TJ93LxbJlCHIKKwCA1cXud3GycRN72eqSqZfJGds
-aeWLmFmHf6oee27d8XLoNjbyAxna/4jdWoTqmp8oT3bgv/TBco23NzqUSVPi+7ljS1hHvcJu
-oJYqaztGrAEf/lWIGdfl/kLEh8IYx8OBNUojh9mzCDlwbs83CBqoUdlzLNDdwmzu34Aw7xK1
-4RAVinGFCpo/7EWoX6weyB/zqevUIIE89UABTeFoGih/hx2jdQV/NQNthWTW0jH0hmPnajBV
-AJPYwAuO82rx2pnZCxDATMn0elOkTue3PCmzHBF/GT6c65aQC4aojj0+Veh787QllQ9FrWbw
-nTz+4fNzU/MBZtyLZ4JnsiWUs9eJ2V1g/A+RiIKu357Qgy1ytLqlgYiWfzHFlYjdtbPYKjDa
-ScnvtY8VO2Rktm7XiV4zKFKiaWp+vuVYpR0/7Adgnlj5Jt9lQQGOr+Z2VYx8SvBcC+by3XAt
-YkRHtX5u4MLlVS3gcoWfDiWwCpvqdK21EsXjQJxRr3dbSn0HaVj4FJZX0QQ7WZm6WLkCDQRh
-uLq3ARAA6RYjqfC0YcLGKvHhoBnsX29vy9Wn1y2JYpEnPUIB8X0VOyz5/ALv4Hqtl4THkH+m
-mMuhtndoq2BkCCk508jWBvKS1S+Bd2esB45BDDmIhuX3ozu9Xza4i1FsPnLkQ0uMZJv30ls2
-pXFmskhYyzmo6aOmH2536LdtPSlXtywfNV1HEr69V/AHbrEzfoQkJ/qvPzELBOjfjwtDPDeP
-iVgW9LhktzVzn/BjO7XlJxw4PGcxJG6VApsXmM3t2fPN9eIHDUq8ocbHdJ4en8/bJDXZd9eb
-QoILUuCg46hE3p6nTXfnPwSRnIRnsgCzeAz4rxDR4/Gv1Xpzv5wqpL21XQi3nvZKlcv7J1IR
-VdphK66De9GpVQVTqC102gqJUErdjGmxmyCA1OOORqEPfKTrXz5YUGsWwpH+4xCuNQP0qmre
-Rw3ghrH8potIr0iOVXFic5vJfBTgtcuEB6E6ulAN+3jqBGTaBML0jxgj3Z5VC5HKVbpg2DbB
-/wMrLwFHNAbzV5hj2Os5Zmva0ySP1YHB26pAW8dwB38GBaQvfZq3ezM4cRAo/iJ/GsVE98dZ
-EBO+Ml+0KYj+ZG+vyxzo20sweun7ZKT+9qZM90f6cQ3zqX6IfXZHHmQJBNv73mcZWNhDQOHs
-4wBoq+FGQWNqLU9xaZxdXw80r1viDAwOy13EUtcVbTkAEQEAAYkCPAQYAQgAJhYhBIWb6NfF
-hvU4QwsZwkZ7lC06eb0pBQJhuLq3AhsMBQkDwmcAAAoJEEZ7lC06eb0pSi8P/iy+dNnxrtiE
-Nn9vkkA7AmZ8RsvPXYVeDCDSsL7UfhbS77r2L1qTa2aB3gAZUDIOXln51lSxMeeLtOequLME
-V2Xi5km70rdtnja5SmWfc9fyExunXnsOhg6UG872At5CGEZU0c2Nt/hlGtOR3xbt3O/Uwl+d
-ErQPA4BUbW5K1T7OC6oPvtlKfF4bGZFloHgt2yE9YSNWZsTPe6XJSapemHZLPOxJLnhs3VBi
-rWE31QS0bRl5AzlO/fg7ia65vQGMOCOTLpgChTbcZHtozeFqva4IeEgE4xN+6r8WtgSYeGGD
-RmeMEVjPM9dzQObf+SvGd58u2z9f2agPK1H32c69RLoA0mHRe7Wkv4izeJUc5tumUY0e8Ojd
-enZZjT3hjLh6tM+mrp2oWnQIoed4LxUw1dhMOj0rYXv6laLGJ1FsW5eSke7ohBLcfBBTKnMC
-BohROHy2E63Wggfsdn3UYzfqZ8cfbXetkXuLS/OM3MXbiNjg+ElYzjgWrkayu7yLakZx+mx6
-sHPIJYm2hzkniMG29d5mGl7ZT9emP9b+CfqGUxoXJkjs0gnDl44bwGJ0dmIBu3ajVAaHODXy
-Y/zdDMGjskfEYbNXCAY2FRZSE58tgTvPKD++Kd2KGplMU2EIFT7JYfKhHAB5DGMkx92HUMid
-sTSKHe+QnnnoFmu4gnmDU31i
-=Xqbo
------END PGP PUBLIC KEY BLOCK-----

requirements.txt

@@ -3,6 +3,6 @@
 # the cryptography package with Rust results in a segfault, so we need to
 # set the CRYPTOGRAPHY_DONT_BUILD_RUST env var to force the package to build
 # with C instead. This env var is only present in cryptography<3.5.
-cryptography==3.4.8
+cryptography==41.0
 pyfxa==0.7.7
 tokenlib==2.0.0

syncserver-settings/Cargo.toml

@@ -1,15 +1,15 @@
 [package]
 name = "syncserver-settings"
-version.workspace=true
-license.workspace=true
-authors.workspace=true
-edition.workspace=true
+version.workspace = true
+license.workspace = true
+authors.workspace = true
+edition.workspace = true
 
 [dependencies]
-serde.workspace=true
-slog-scope.workspace=true
+serde.workspace = true
+slog-scope.workspace = true
 
-config = "0.11"
+config = "0.11"                                             # pin to 11, 12+ introduces a breaking change for env vars.
 num_cpus = "1"
 syncserver-common = { path = "../syncserver-common" }
 syncstorage-settings = { path = "../syncstorage-settings" }

syncserver-settings/src/lib.rs

@@ -112,7 +112,6 @@ impl Settings {
                             .to_string(),
                     );
                 }
-
                 Ok(s)
             }
             // Configuration errors are not very sysop friendly, Try to make them

syncserver/Cargo.toml

@@ -34,15 +34,18 @@ slog-stdlog.workspace = true
 slog-term.workspace = true
 
 actix-http = "2"
-actix-rt = "1"                                                       # Pin to 1.0, due to dependencies on Tokio
-actix-cors = "0.5"
+actix-rt = "1" # Pin to 1.0, due to dependencies on Tokio
+actix-cors = "0.5" # pin to 0.5. 0.6 introduces breaking change
 async-trait = "0.1.40"
 dyn-clone = "1.0.4"
 hostname = "0.3.1"
-hawk = "3.2"
+hawk = "4.0"
 hmac = "0.12"
 mime = "0.3"
-reqwest = { version = "0.10.10", features = ["json", "rustls-tls"] }
+reqwest = { workspace = true, features = [
+    "json",
+    "rustls-tls",
+] } # 0.11 requires tokio 1.34
 # pin to 0.19: https://github.com/getsentry/sentry-rust/issues/277
 syncserver-common = { path = "../syncserver-common" }
 syncserver-db-common = { path = "../syncserver-db-common" }
@@ -56,13 +59,14 @@ tokenserver-common = { path = "../tokenserver-common" }
 tokenserver-db = { path = "../tokenserver-db" }
 tokenserver-settings = { path = "../tokenserver-settings" }
 # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.)
-tokio = { version = "0.2.4", features = ["macros", "sync"] }
+tokio = { workspace = true, features = ["macros", "sync"] }
 urlencoding = "2.1"
 validator = "0.16"
 validator_derive = "0.16"
 woothee = "0.13"
 
 [features]
-default = ["syncstorage-db/mysql"]
+default = ["mysql"]
 no_auth = []
+mysql = ["syncstorage-db/mysql"]
 spanner = ["syncstorage-db/spanner"]

syncserver/src/server/test.rs

@@ -58,6 +58,18 @@ fn get_test_settings() -> Settings {
             .as_str(),
     )
     .expect("Could not get pool_size in get_test_settings");
+    if cfg!(feature = "mysql") && settings.syncstorage.uses_spanner() {
+        panic!(
+            "Spanner database_url specified for MySQL feature, please correct.\n\t{}",
+            &settings.syncstorage.database_url
+        )
+    }
+    if cfg!(feature = "spanner") && !&settings.syncstorage.uses_spanner() {
+        panic!(
+            "MySQL database_url specified for Spanner feature, please correct.\n\t{}",
+            &settings.syncstorage.database_url
+        )
+    }
     settings.port = port;
     settings.host = host;
     settings.syncstorage.database_pool_max_size = pool_size + 1;

syncstorage-db/Cargo.toml

@@ -1,17 +1,17 @@
 [package]
 name = "syncstorage-db"
-version.workspace=true
-license.workspace=true
-authors.workspace=true
-edition.workspace=true
+version.workspace = true
+license.workspace = true
+authors.workspace = true
+edition.workspace = true
 
 [dependencies]
-cadence.workspace=true
-env_logger.workspace=true
-futures.workspace=true
-lazy_static.workspace=true
-rand.workspace=true
-slog-scope.workspace=true
+cadence.workspace = true
+env_logger.workspace = true
+futures.workspace = true
+lazy_static.workspace = true
+rand.workspace = true
+slog-scope.workspace = true
 
 async-trait = "0.1.40"
 hostname = "0.3.1"
@@ -27,7 +27,7 @@ syncstorage-mysql = { path = "../syncstorage-mysql", optional = true }
 syncstorage-settings = { path = "../syncstorage-settings" }
 syncstorage-spanner = { path = "../syncstorage-spanner", optional = true }
 # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.)
-tokio = { version = "0.2.4", features = ["macros", "sync"] }
+tokio = { workspace = true, features = ["macros", "sync"] }
 
 [features]
 mysql = ['syncstorage-mysql']

syncstorage-mysql/migrations/2018-08-28-010336_init/down.sql

@@ -1,4 +1,4 @@
 -- DROP TABLE IF EXISTS `bso`;
 -- DROP TABLE IF EXISTS `collections`;
 -- DROP TABLE IF EXISTS `user_collections`;
--- DROP TABLE IF EXISTS `batches`;
+-- DROP TABLE IF EXISTS `batches`;

syncstorage-mysql/migrations/2018-08-28-010336_init/up.sql

@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS `bso`(
 
 
 DROP TABLE IF EXISTS `collections`;
-CREATE TABLE `collections`(
+CREATE TABLE IF NOT EXISTS `collections`(
     `id` INT PRIMARY KEY      NOT NULL AUTO_INCREMENT,
     `name` VARCHAR(32) UNIQUE NOT NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;

syncstorage-spanner/Cargo.toml

@@ -34,12 +34,12 @@ syncserver-db-common = { path = "../syncserver-db-common" }
 syncstorage-db-common = { path = "../syncstorage-db-common" }
 syncstorage-settings = { path = "../syncstorage-settings" }
 thiserror = "1.0.26"
-tokio = { version = "0.2.4", features = [
+tokio = { workspace = true, features = [
   "macros",
   "sync",
 ] } # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.)
 url = "2.1"
-uuid = { version = "0.8.2", features = ["serde", "v4"] }
+uuid = { version = "1.6", features = ["serde", "v4"] }
 
 [[bin]]
 name = "purge_ttl"

syncstorage-spanner/src/batch.rs

@@ -23,7 +23,7 @@ pub async fn create_async(
     db: &SpannerDb,
     params: params::CreateBatch,
 ) -> DbResult<results::CreateBatch> {
-    let batch_id = Uuid::new_v4().to_simple().to_string();
+    let batch_id = Uuid::new_v4().simple().to_string();
     let collection_id = db.get_collection_id_async(&params.collection).await?;
     let timestamp = db.checked_timestamp()?.as_i64();
 

tokenserver-auth/Cargo.toml

@@ -1,26 +1,26 @@
 [package]
 name = "tokenserver-auth"
-version.workspace=true
-license.workspace=true
-authors.workspace=true
-edition.workspace=true
+version.workspace = true
+license.workspace = true
+authors.workspace = true
+edition.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-futures.workspace=true
-serde.workspace=true
-serde_json.workspace=true
+futures.workspace = true
+serde.workspace = true
+serde_json.workspace = true
 
 async-trait = "0.1.40"
 dyn-clone = "1.0.4"
-pyo3 = { version = "0.14", features = ["auto-initialize"] }
-reqwest = { version = "0.10.10", features = ["json", "rustls-tls"] }
+pyo3 = { version = "0.20", features = ["auto-initialize"] }
+reqwest = { workspace = true, features = ["json", "rustls-tls"] }
 syncserver-common = { path = "../syncserver-common" }
 tokenserver-common = { path = "../tokenserver-common" }
 tokenserver-settings = { path = "../tokenserver-settings" }
 # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.)
-tokio = { version = "0.2.4", features = ["blocking"] }
+tokio = { workspace = true, features = ["blocking"] }
 
 [dev-dependencies]
 mockito = "0.30.0"

tokenserver-db/Cargo.toml

@@ -1,18 +1,18 @@
 [package]
 name = "tokenserver-db"
-version.workspace=true
-license.workspace=true
-authors.workspace=true
-edition.workspace=true
+version.workspace = true
+license.workspace = true
+authors.workspace = true
+edition.workspace = true
 
 [dependencies]
-backtrace.workspace=true
-futures.workspace=true
-http.workspace=true
-serde.workspace=true
-serde_derive.workspace=true
-serde_json.workspace=true
-slog-scope.workspace=true
+backtrace.workspace = true
+futures.workspace = true
+http.workspace = true
+serde.workspace = true
+serde_derive.workspace = true
+serde_json.workspace = true
+slog-scope.workspace = true
 
 async-trait = "0.1.40"
 diesel = { version = "1.4", features = ["mysql", "r2d2"] }
@@ -24,9 +24,9 @@ thiserror = "1.0.26"
 tokenserver-common = { path = "../tokenserver-common" }
 tokenserver-settings = { path = "../tokenserver-settings" }
 # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.)
-tokio = { version = "0.2.4", features = ["macros", "sync"] }
+tokio = { workspace = true, features = ["macros", "sync"] }
 
 [dev-dependencies]
-env_logger.workspace=true
+env_logger.workspace = true
 
 syncserver-settings = { path = "../syncserver-settings" }