From e6ec1acd8742fab3c456e0dc948ea4c8cc21864d Mon Sep 17 00:00:00 2001 From: Ethan Donowitz <8703826+ethowitz@users.noreply.github.com> Date: Wed, 19 May 2021 18:20:25 -0400 Subject: [PATCH] feat: Integrate Spanner emulator with CI (#1079) Closes #566 --- .circleci/config.yml | 23 ++++++-- Dockerfile | 8 ++- Makefile | 21 +++++-- README.md | 40 +++++++++++++- ....e2e.yaml => docker-compose.e2e.mysql.yaml | 0 docker-compose.e2e.spanner.yaml | 35 ++++++++++++ ...-compose.yaml => docker-compose.mysql.yaml | 0 docker-compose.spanner.yaml | 40 ++++++++++++++ scripts/prepare-spanner.sh | 31 +++++++++++ .../spanner/insert_standard_collections.sql | 17 ++++++ src/db/spanner/manager/session.rs | 11 +++- src/db/spanner/models.rs | 50 +++++++++++------ .../db/spanner/schema.ddl | 29 +++------- src/web/extractors.rs | 55 +++++++++---------- tools/integration_tests/test_storage.py | 9 ++- 15 files changed, 284 insertions(+), 85 deletions(-) rename docker-compose.e2e.yaml => docker-compose.e2e.mysql.yaml (100%) create mode 100644 docker-compose.e2e.spanner.yaml rename docker-compose.yaml => docker-compose.mysql.yaml (100%) create mode 100644 docker-compose.spanner.yaml create mode 100755 scripts/prepare-spanner.sh create mode 100644 src/db/spanner/insert_standard_collections.sql rename spanner-2019-10-01.ddl => src/db/spanner/schema.ddl (84%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 14627dc2..1999f150 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,14 +79,28 @@ commands: environment: SYNC_ENFORCE_QUOTA: 1 - run-e2e-tests: + run-e2e-mysql-tests: steps: - run: name: e2e tests command: > /usr/local/bin/docker-compose - -f docker-compose.yaml - -f docker-compose.e2e.yaml + -f docker-compose.mysql.yaml + -f docker-compose.e2e.mysql.yaml + up + --exit-code-from e2e-tests + --abort-on-container-exit + environment: + SYNCSTORAGE_RS_IMAGE: app:build + + run-e2e-spanner-tests: + steps: + - run: + name: e2e tests + command: > + /usr/local/bin/docker-compose + -f docker-compose.spanner.yaml + -f docker-compose.e2e.spanner.yaml up --exit-code-from e2e-tests --abort-on-container-exit @@ -207,7 +221,8 @@ jobs: - run: name: Restore docker-compose config command: cp /home/circleci/cache/docker-compose*.yaml . - - run-e2e-tests + - run-e2e-mysql-tests + - run-e2e-spanner-tests deploy: docker: diff --git a/Dockerfile b/Dockerfile index bea15f7f..d6d3a4c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.51-buster as builder +FROM rust:1.52.1-buster as builder WORKDIR /app ADD . /app ENV PATH=$PATH:/root/.cargo/bin @@ -20,7 +20,7 @@ RUN \ groupadd --gid 10001 app && \ useradd --uid 10001 --gid 10001 --home /app --create-home app && \ apt-get -q update && \ - apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-dev python3-pip && \ + apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-dev python3-pip curl jq && \ pip3 install tokenlib && \ rm -rf /var/lib/apt/lists/* @@ -29,6 +29,10 @@ COPY --from=builder /app/version.json /app COPY --from=builder /app/spanner_config.ini /app COPY --from=builder /app/tools/spanner /app/tools/spanner COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests +COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh +COPY --from=builder /app/src/db/spanner/schema.ddl /app/schema.ddl + +RUN chmod +x /app/scripts/prepare-spanner.sh USER app:app diff --git a/Makefile b/Makefile index 5990714d..274402f0 100644 --- a/Makefile +++ b/Makefile @@ -16,14 +16,23 @@ clippy: # Matches what's run in circleci cargo clippy --all --all-targets -- -D warnings -docker_start: - docker-compose up -d +docker_start_mysql: + docker-compose -f docker-compose.mysql.yaml up -d -docker_start_rebuild: - docker-compose up --build -d +docker_start_mysql_rebuild: + docker-compose -f docker-compose.mysql.yaml up --build -d -docker_stop: - docker-compose down +docker_stop_mysql: + docker-compose -f docker-compose.mysql.yaml down + +docker_start_spanner: + docker-compose -f docker-compose.spanner.yaml up -d + +docker_start_spanner_rebuild: + docker-compose -f docker-compose.spanner.yaml up --build -d + +docker_stop_spanner: + docker-compose -f docker-compose.spanner.yaml down run: RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml diff --git a/README.md b/README.md index f3d6b5f6..01c3141b 100644 --- a/README.md +++ b/README.md @@ -106,19 +106,53 @@ To point to a GCP hosted Spanner instance from your local machine, follow these 4. `make run_spanner`. 5. Visit `http://localhost:8000/__heartbeat__` to make sure the server is running. +#### Emulator +Google supports an in-memory Spanner emulator, which can run on your local machine for development purposes. You can install the emulator via the gcloud CLI or Docker by following the instructions [here](https://cloud.google.com/spanner/docs/emulator#installing_and_running_the_emulator). Once the emulator is running, you'll need to create a new instance and a new database. To create an instance using the REST API (exposed via port 9020 on the emulator), we can use `curl`: +```sh +curl --request POST \ + "localhost:9020/v1/projects/$PROJECT_ID/instances" \ + --header 'Accept: application/json' \ + --header 'Content-Type: application/json' \ + --data "{\"instance\":{\"config\":\"emulator-test-config\",\"nodeCount\":1,\"displayName\":\"Test Instance\"},\"instanceId\":\"$INSTANCE_ID\"}" +``` +Note that you may set `PROJECT_ID` and `INSTANCE_ID` to your liking. To create a new database on this instance, we'll use a similar HTTP request, but we'll need to include information about the database schema. Since we don't have migrations for Spanner, we keep an up-to-date schema in `src/db/spanner/schema.ddl`. The `jq` utility allows us to parse this file for use in the JSON body of an HTTP POST request: +```sh +DDL_STATEMENTS=$( + grep -v ^-- schema.ddl \ + | sed -n 's/ \+/ /gp' \ + | tr -d '\n' \ + | sed 's/\(.*\);/\1/' \ + | jq -R -s -c 'split(";")' +) +``` +Finally, to create the database: +```sh +curl -sS --request POST \ + "localhost:9020/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \ + --header 'Accept: application/json' \ + --header 'Content-Type: application/json' \ + --data "{\"createStatement\":\"CREATE DATABASE \`$DATABASE_ID\`\",\"extraStatements\":$DDL_STATEMENTS}" +``` +Note that, again, you may set `DATABASE_ID` to your liking. Make sure that the `database_url` config variable reflects your choice of project name, instance name, and database name (i.e. it should be of the format `spanner://projects//instances//databases/`). + +To run an application server that points to the local Spanner emulator: +```sh +SYNC_SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner +``` + ### Running via Docker This requires access to the mozilla-rust-sdk which is now available at `/vendor/mozilla-rust-adk`. 1. Make sure you have [Docker installed](https://docs.docker.com/install/) locally. 2. Copy the contents of mozilla-rust-sdk into top level root dir here. 3. Change cargo.toml mozilla-rust-sdk entry to point to `"path = "mozilla-rust-sdk/googleapis-raw"` instead of the parent dir. -4. Comment out the `image` value under `syncstorage-rs` in docker-compose.yml, and add this instead: +4. Comment out the `image` value under `syncstorage-rs` in either docker-compose.mysql.yml or docker-compose.spanner.yml (depending on which database backend you want to run), and add this instead: ```yml build: context: . ``` -5. Adjust the MySQL db credentials in docker-compose.yml to match your local setup. -6. `make docker_start` - You can verify it's working by visiting [localhost:8000/\_\_heartbeat\_\_](http://localhost:8000/__heartbeat__) +5. If you are using MySQL, adjust the MySQL db credentials in docker-compose.mysql.yml to match your local setup. +6. `make docker_start_mysql` or `make docker_start_spanner` - You can verify it's working by visiting [localhost:8000/\_\_heartbeat\_\_](http://localhost:8000/__heartbeat__) ### Connecting to Firefox diff --git a/docker-compose.e2e.yaml b/docker-compose.e2e.mysql.yaml similarity index 100% rename from docker-compose.e2e.yaml rename to docker-compose.e2e.mysql.yaml diff --git a/docker-compose.e2e.spanner.yaml b/docker-compose.e2e.spanner.yaml new file mode 100644 index 00000000..15f28b59 --- /dev/null +++ b/docker-compose.e2e.spanner.yaml @@ -0,0 +1,35 @@ +version: '3' +services: + db: + db-setup: + syncstorage-rs: + depends_on: + - db-setup + # TODO: either syncstorage-rs should retry the db connection + # itself a few times or should include a wait-for-it.sh script + # inside its docker that would do this for us. Same (probably + # the latter solution) for server-syncstorage below + entrypoint: > + /bin/sh -c " + sleep 15; + /app/bin/syncstorage; + " + e2e-tests: + depends_on: + - syncstorage-rs + image: app:build + privileged: true + user: root + environment: + SYNC_HOST: 0.0.0.0 + SYNC_MASTER_SECRET: secret0 + SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + SYNC_SPANNER_EMULATOR_HOST: db:9010 + SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver + SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ + SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB + SYNC_FXA_METRICS_HASH_SECRET: insecure + entrypoint: > + /bin/sh -c " + sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0' + " diff --git a/docker-compose.yaml b/docker-compose.mysql.yaml similarity index 100% rename from docker-compose.yaml rename to docker-compose.mysql.yaml diff --git a/docker-compose.spanner.yaml b/docker-compose.spanner.yaml new file mode 100644 index 00000000..7df76993 --- /dev/null +++ b/docker-compose.spanner.yaml @@ -0,0 +1,40 @@ +version: '3' +services: + db: + image: gcr.io/cloud-spanner-emulator/emulator + ports: + - "9010:9010" + - "9020:9020" + environment: + PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + db-setup: + image: app:build + depends_on: + - db + restart: "no" + entrypoint: "/app/scripts/prepare-spanner.sh" + environment: + SYNC_SPANNER_EMULATOR_HOST: db:9020 + syncstorage-rs: + image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest} + restart: always + ports: + - "8000:8000" + depends_on: + - db-setup + environment: + SYNC_HOST: 0.0.0.0 + SYNC_MASTER_SECRET: secret0 + SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database + SYNC_SPANNER_EMULATOR_HOST: db:9010 + SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver + SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ + SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB + SYNC_FXA_METRICS_HASH_SECRET: insecure + +volumes: + db_data: + +# Application runs off of port 8000. +# you can test if it's available with +# curl "http://localhost:8000/__heartbeat__" diff --git a/scripts/prepare-spanner.sh b/scripts/prepare-spanner.sh new file mode 100755 index 00000000..1fc6ab92 --- /dev/null +++ b/scripts/prepare-spanner.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +sleep 5 + +set -e + +PROJECT_ID=test-project +INSTANCE_ID=test-instance +DATABASE_ID=test-database + +DDL_STATEMENTS=$( + grep -v ^-- schema.ddl \ + | sed -n 's/ \+/ /gp' \ + | tr -d '\n' \ + | sed 's/\(.*\);/\1/' \ + | jq -R -s -c 'split(";")' +) + +curl -sS --request POST \ + "$SYNC_SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances" \ + --header 'Accept: application/json' \ + --header 'Content-Type: application/json' \ + --data "{\"instance\":{\"config\":\"emulator-test-config\",\"nodeCount\":1,\"displayName\":\"Test Instance\"},\"instanceId\":\"$INSTANCE_ID\"}" + +curl -sS --request POST \ + "$SYNC_SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \ + --header 'Accept: application/json' \ + --header 'Content-Type: application/json' \ + --data "{\"createStatement\":\"CREATE DATABASE \`$DATABASE_ID\`\",\"extraStatements\":$DDL_STATEMENTS}" + +sleep infinity diff --git a/src/db/spanner/insert_standard_collections.sql b/src/db/spanner/insert_standard_collections.sql new file mode 100644 index 00000000..7f6d1b63 --- /dev/null +++ b/src/db/spanner/insert_standard_collections.sql @@ -0,0 +1,17 @@ +-- These are the 13 standard collections that are expected to exist by clients. +-- The IDs are fixed. The below statement can be used to add these collections +-- to a Spanner instance. +INSERT INTO collections (collection_id, name) VALUES + ( 1, "clients"), + ( 2, "crypto"), + ( 3, "forms"), + ( 4, "history"), + ( 5, "keys"), + ( 6, "meta"), + ( 7, "bookmarks"), + ( 8, "prefs"), + ( 9, "tabs"), + (10, "passwords"), + (11, "addons"), + (12, "addresses"), + (13, "creditcards"); diff --git a/src/db/spanner/manager/session.rs b/src/db/spanner/manager/session.rs index 70ff6f4a..88258968 100644 --- a/src/db/spanner/manager/session.rs +++ b/src/db/spanner/manager/session.rs @@ -29,6 +29,8 @@ pub struct SpannerSession { /// Session has a similar `create_time` value that is managed by protobuf, /// but some clock skew issues are possible. pub(in crate::db::spanner) create_time: i64, + /// Whether we are using the Spanner emulator + pub using_spanner_emulator: bool, } /// Create a Session (and the underlying gRPC Channel) @@ -39,10 +41,8 @@ pub async fn create_spanner_session( use_test_transactions: bool, emulator_host: Option, ) -> Result { - // XXX: issue732: Could google_default_credentials (or - // ChannelBuilder::secure_connect) block?! + let using_spanner_emulator = emulator_host.is_some(); let chan = block(move || -> Result { - metrics.start_timer("storage.pool.grpc_auth", None); if let Some(spanner_emulator_address) = emulator_host { Ok(ChannelBuilder::new(env) .max_send_message_len(100 << 20) @@ -51,6 +51,10 @@ pub async fn create_spanner_session( } else { // Requires // GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json + metrics.start_timer("storage.pool.grpc_auth", None); + + // XXX: issue732: Could google_default_credentials (or + // ChannelBuilder::secure_connect) block?! let creds = ChannelCredentials::google_default_credentials()?; Ok(ChannelBuilder::new(env) .max_send_message_len(100 << 20) @@ -75,6 +79,7 @@ pub async fn create_spanner_session( client, use_test_transactions, create_time: now(), + using_spanner_emulator, }) } diff --git a/src/db/spanner/models.rs b/src/db/spanner/models.rs index c4dbb190..7cabc6db 100644 --- a/src/db/spanner/models.rs +++ b/src/db/spanner/models.rs @@ -1247,13 +1247,19 @@ impl SpannerDb { if let Some(timestamp) = offset.clone().unwrap_or_default().timestamp { query = match sort { Sorting::Newest => { - sqlparams.insert("older_eq".to_string(), as_value(timestamp.as_rfc3339()?)); - sqltypes.insert("older_eq".to_string(), as_type(TypeCode::TIMESTAMP)); + sqlparams.insert( + "older_eq".to_string(), + timestamp.as_rfc3339()?.to_spanner_value(), + ); + sqlparam_types.insert("older_eq".to_string(), as_type(TypeCode::TIMESTAMP)); format!("{} AND modified <= @older_eq", query) } Sorting::Oldest => { - sqlparams.insert("newer_eq".to_string(), as_value(timestamp.as_rfc3339()?)); - sqltypes.insert("newer_eq".to_string(), as_type(TypeCode::TIMESTAMP)); + sqlparams.insert( + "newer_eq".to_string(), + timestamp.as_rfc3339()?.to_spanner_value(), + ); + sqlparam_types.insert("newer_eq".to_string(), as_type(TypeCode::TIMESTAMP)); format!("{} AND modified >= @newer_eq", query) } _ => query, @@ -1270,20 +1276,23 @@ impl SpannerDb { sqlparams.insert("newer".to_string(), newer.as_rfc3339()?.to_spanner_value()); sqlparam_types.insert("newer".to_string(), as_type(TypeCode::TIMESTAMP)); } - query = match sort { - // issue559: Revert to previous sorting - /* - Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query), - Sorting::Newest | Sorting::None => { - format!("{} ORDER BY modified DESC, bso_id DESC", query) - } - Sorting::Oldest => format!("{} ORDER BY modified ASC, bso_id ASC", query), - */ - Sorting::Index => format!("{} ORDER BY sortindex DESC", query), - Sorting::Newest => format!("{} ORDER BY modified DESC", query), - Sorting::Oldest => format!("{} ORDER BY modified ASC", query), - _ => query, - }; + + if self.stabilize_bsos_sort_order() { + query = match sort { + Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query), + Sorting::Newest | Sorting::None => { + format!("{} ORDER BY modified DESC, bso_id DESC", query) + } + Sorting::Oldest => format!("{} ORDER BY modified ASC, bso_id ASC", query), + }; + } else { + query = match sort { + Sorting::Index => format!("{} ORDER BY sortindex DESC", query), + Sorting::Newest => format!("{} ORDER BY modified DESC", query), + Sorting::Oldest => format!("{} ORDER BY modified ASC", query), + _ => query, + }; + } if let Some(limit) = limit { // fetch an extra row to detect if there are more rows that match @@ -1311,6 +1320,11 @@ impl SpannerDb { .execute_async(&self.conn) } + /// Whether to stabilize the sort order for get_bsos_async + fn stabilize_bsos_sort_order(&self) -> bool { + self.inner.conn.using_spanner_emulator + } + pub fn encode_next_offset( &self, _sort: Sorting, diff --git a/spanner-2019-10-01.ddl b/src/db/spanner/schema.ddl similarity index 84% rename from spanner-2019-10-01.ddl rename to src/db/spanner/schema.ddl index ef714016..7215c749 100644 --- a/spanner-2019-10-01.ddl +++ b/src/db/spanner/schema.ddl @@ -15,6 +15,9 @@ CREATE TABLE user_collections ( fxa_kid STRING(MAX) NOT NULL, collection_id INT64 NOT NULL, modified TIMESTAMP NOT NULL, + + count INT64, + total_bytes INT64, ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id); CREATE TABLE bsos ( @@ -57,8 +60,9 @@ CREATE TABLE batches ( ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id, batch_id), INTERLEAVE IN PARENT user_collections ON DELETE CASCADE; - CREATE INDEX BatchExpiry - ON batches(expiry); + CREATE INDEX BatchExpireId + ON batches(fxa_uid, fxa_kid, collection_id, expiry), +INTERLEAVE IN user_collections; CREATE TABLE batch_bsos ( fxa_uid STRING(MAX) NOT NULL, @@ -78,21 +82,6 @@ CREATE TABLE batch_bsos ( -- no "modified" column because the modification timestamp gets set on -- batch commit. --- 8< Cut Here >8 -- --- Inserting values into table(s) should happen only --- after table creation. - -INSERT INTO collections (collection_id, name) VALUES - ( 1, "clients"), - ( 2, "crypto"), - ( 3, "forms"), - ( 4, "history"), - ( 5, "keys"), - ( 6, "meta"), - ( 7, "bookmarks"), - ( 8, "prefs"), - ( 9, "tabs"), - (10, "passwords"), - (11, "addons"), - (12, "addresses"), - (13, "creditcards"); +-- *NOTE*: +-- Newly created Spanner instances should pre-populate the `collections` table by +-- running the content of `insert_standard_collections.sql ` diff --git a/src/web/extractors.rs b/src/web/extractors.rs index f0b83334..5801e4b8 100644 --- a/src/web/extractors.rs +++ b/src/web/extractors.rs @@ -1245,33 +1245,33 @@ impl FromRequest for BsoQueryParams { })?; // issue559: Dead code (timestamp always None) /* - if params.sort != Sorting::Index { - if let Some(timestamp) = params.offset.as_ref().and_then(|offset| offset.timestamp) - { - let bound = timestamp.as_i64(); - if let Some(newer) = params.newer { - if bound < newer.as_i64() { - return Err(ValidationErrorKind::FromDetails( - format!("Invalid Offset {} {}", bound, newer.as_i64()), - RequestErrorLocation::QueryString, - Some("newer".to_owned()), - None, - ) - .into()); - } - } else if let Some(older) = params.older { - if bound > older.as_i64() { - return Err(ValidationErrorKind::FromDetails( - "Invalid Offset".to_owned(), - RequestErrorLocation::QueryString, - Some("older".to_owned()), - None, - ) - .into()); - } - } - } - } + if params.sort != Sorting::Index { + if let Some(timestamp) = params.offset.as_ref().and_then(|offset| offset.timestamp) + { + let bound = timestamp.as_i64(); + if let Some(newer) = params.newer { + if bound < newer.as_i64() { + return Err(ValidationErrorKind::FromDetails( + format!("Invalid Offset {} {}", bound, newer.as_i64()), + RequestErrorLocation::QueryString, + Some("newer".to_owned()), + None, + ) + .into()); + } + } else if let Some(older) = params.older { + if bound > older.as_i64() { + return Err(ValidationErrorKind::FromDetails( + "Invalid Offset".to_owned(), + RequestErrorLocation::QueryString, + Some("older".to_owned()), + None, + ) + .into()); + } + } + } + } */ Ok(params) }) @@ -2313,7 +2313,6 @@ mod tests { offset: 1234, }; - //Issue559: only use offset, don't use timestamp, even if set. let test_offset = Offset { timestamp: None, offset: sample_offset.offset, diff --git a/tools/integration_tests/test_storage.py b/tools/integration_tests/test_storage.py index 37b53d35..b0e28b22 100644 --- a/tools/integration_tests/test_storage.py +++ b/tools/integration_tests/test_storage.py @@ -1397,7 +1397,14 @@ class TestStorage(StorageFunctionalTestCase): secret = auth_policy._get_token_secrets(self.host_url)[-1] tm = tokenlib.TokenManager(secret=secret) exp = time.time() - 60 - data = {"uid": self.user_id, "node": self.host_url, "expires": exp} + data = { + "uid": self.user_id, + "node": self.host_url, + "expires": exp, + "hashed_fxa_uid": self.hashed_fxa_uid, + "fxa_uid": self.fxa_uid, + "fxa_kid": self.fxa_kid + } self.auth_token = tm.make_token(data) self.auth_secret = tm.get_derived_secret(self.auth_token)