From c95f2ff21a5e3b428b2715018e7e782b22a2dfa8 Mon Sep 17 00:00:00 2001 From: Donovan Preston Date: Tue, 10 Mar 2020 13:38:35 -0400 Subject: [PATCH 1/8] fix: GETs with a limit and no sort never advance X-Weave-Next-Offset Fix #468 --- src/db/mysql/models.rs | 7 +++---- src/db/spanner/models.rs | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/db/mysql/models.rs b/src/db/mysql/models.rs index aeedb3d4..8221892d 100644 --- a/src/db/mysql/models.rs +++ b/src/db/mysql/models.rs @@ -484,10 +484,9 @@ impl MysqlDb { } query = match sort { - Sorting::Index => query.order(bso::sortindex.desc()), - Sorting::Newest => query.order(bso::modified.desc()), - Sorting::Oldest => query.order(bso::modified.asc()), - _ => query, + Sorting::Index => query.order(bso::id.desc()).order(bso::sortindex.desc()), + Sorting::Newest => query.order(bso::id.desc()).order(bso::modified.desc()), + Sorting::Oldest | Sorting::None => query.order(bso::id.asc()).order(bso::modified.asc()), }; let limit = limit.map(i64::from).unwrap_or(-1); diff --git a/src/db/spanner/models.rs b/src/db/spanner/models.rs index 61778809..8520a6ce 100644 --- a/src/db/spanner/models.rs +++ b/src/db/spanner/models.rs @@ -975,10 +975,9 @@ impl SpannerDb { sqltypes.insert("newer".to_string(), as_type(TypeCode::TIMESTAMP)); } query = match sort { - Sorting::Index => format!("{} ORDER BY sortindex DESC", query), - Sorting::Newest => format!("{} ORDER BY modified DESC", query), - Sorting::Oldest => format!("{} ORDER BY modified ASC", query), - _ => query, + Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query), + Sorting::Newest => format!("{} ORDER BY modified DESC, bso_id DESC", query), + Sorting::Oldest | Sorting::None => format!("{} ORDER BY modified ASC, bso_id ASC", query), }; if let Some(limit) = limit { From ba4810896b65adbb94820d2f398ceef6cb36aac9 Mon Sep 17 00:00:00 2001 From: Greg Guthe Date: Tue, 18 Feb 2020 16:24:49 -0500 Subject: [PATCH 2/8] ci: rm rust dependency diff reverts #204 --- .circleci/config.yml | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b0332dd0..c5906142 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,31 +237,6 @@ jobs: echo "Not pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}" fi - rust-changelog: - docker: - - image: python:3.7-buster - steps: - - setup_remote_docker: - version: 18.02.0-ce - - run: - name: Install Docker client - command: | - set -x - VER="18.06.3-ce" - curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz - tar -xz -C /tmp -f /tmp/docker-$VER.tgz - mv /tmp/docker/* /usr/bin - - run: - name: pull find-package-rugaru image - command: | - docker pull mozilla/dependencyscan:latest - - run: - name: Check Rust Dependencies - command: | - printf "{\"org\": \"mozilla-services\", \"repo\": \"syncstorage-rs\", \"ref\": {\"value\": \"master\", \"kind\": \"branch\"}, \"repo_url\": \"https://github.com/mozilla-services/syncstorage-rs.git\"}\n{\"org\": \"mozilla-services\", \"repo\": \"syncstorage-rs\", \"ref\": {\"value\": \"$CIRCLE_SHA1\", \"kind\": \"commit\"}, \"repo_url\": \"https://github.com/mozilla-services/syncstorage-rs.git\"}\n" | \ - docker run --rm -i -v /var/run/docker.sock:/var/run/docker.sock --name fpr-cargo_metadata mozilla/dependencyscan:latest python fpr/run_pipeline.py cargo_metadata | \ - docker run --rm -i -v /var/run/docker.sock:/var/run/docker.sock --name fpr-rust_changelog mozilla/dependencyscan:latest python fpr/run_pipeline.py rust_changelog -m Cargo.toml - workflows: version: 2 build-deploy: @@ -280,11 +255,6 @@ workflows: filters: tags: only: /.*/ - - rust-changelog: - # email dependency-observatory@mozilla.com with feedback - filters: - tags: - only: /.*/ - deploy: requires: - e2e-tests From 2b0ec2ed2a646be7ade2eded4b63a3e4ab1cbe63 Mon Sep 17 00:00:00 2001 From: Donovan Preston Date: Tue, 10 Mar 2020 15:09:06 -0400 Subject: [PATCH 3/8] Default should be newest, not oldest --- src/db/mysql/models.rs | 4 ++-- src/db/spanner/models.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/db/mysql/models.rs b/src/db/mysql/models.rs index 8221892d..c22e8d08 100644 --- a/src/db/mysql/models.rs +++ b/src/db/mysql/models.rs @@ -485,8 +485,8 @@ impl MysqlDb { query = match sort { Sorting::Index => query.order(bso::id.desc()).order(bso::sortindex.desc()), - Sorting::Newest => query.order(bso::id.desc()).order(bso::modified.desc()), - Sorting::Oldest | Sorting::None => query.order(bso::id.asc()).order(bso::modified.asc()), + Sorting::Newest | Sorting::None => query.order(bso::id.desc()).order(bso::modified.desc()), + Sorting::Oldest => query.order(bso::id.asc()).order(bso::modified.asc()), }; let limit = limit.map(i64::from).unwrap_or(-1); diff --git a/src/db/spanner/models.rs b/src/db/spanner/models.rs index 8520a6ce..510059a2 100644 --- a/src/db/spanner/models.rs +++ b/src/db/spanner/models.rs @@ -976,8 +976,8 @@ impl SpannerDb { } query = match sort { Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query), - Sorting::Newest => format!("{} ORDER BY modified DESC, bso_id DESC", query), - Sorting::Oldest | Sorting::None => format!("{} ORDER BY modified ASC, bso_id ASC", query), + Sorting::Newest | Sorting::None => format!("{} ORDER BY modified DESC, bso_id DESC", query), + Sorting::Oldest => format!("{} ORDER BY modified ASC, bso_id ASC", query), }; if let Some(limit) = limit { From 14bcac8773c7fbbf159b5e4c0f1ec65eca9b0034 Mon Sep 17 00:00:00 2001 From: Donovan Preston Date: Tue, 10 Mar 2020 15:10:43 -0400 Subject: [PATCH 4/8] `cargo fmt` --- src/db/mysql/models.rs | 4 +++- src/db/spanner/models.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/db/mysql/models.rs b/src/db/mysql/models.rs index c22e8d08..db11ec20 100644 --- a/src/db/mysql/models.rs +++ b/src/db/mysql/models.rs @@ -485,7 +485,9 @@ impl MysqlDb { query = match sort { Sorting::Index => query.order(bso::id.desc()).order(bso::sortindex.desc()), - Sorting::Newest | Sorting::None => query.order(bso::id.desc()).order(bso::modified.desc()), + Sorting::Newest | Sorting::None => { + query.order(bso::id.desc()).order(bso::modified.desc()) + } Sorting::Oldest => query.order(bso::id.asc()).order(bso::modified.asc()), }; diff --git a/src/db/spanner/models.rs b/src/db/spanner/models.rs index 510059a2..84377153 100644 --- a/src/db/spanner/models.rs +++ b/src/db/spanner/models.rs @@ -976,7 +976,9 @@ impl SpannerDb { } query = match sort { Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query), - Sorting::Newest | Sorting::None => format!("{} ORDER BY modified DESC, bso_id DESC", query), + Sorting::Newest | Sorting::None => { + format!("{} ORDER BY modified DESC, bso_id DESC", query) + } Sorting::Oldest => format!("{} ORDER BY modified ASC, bso_id ASC", query), }; From 5f53c9554c34ec1c0d720192093a355dbf475a61 Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Tue, 10 Mar 2020 16:29:28 -0700 Subject: [PATCH 5/8] chore: tag 0.2.4 --- CHANGELOG.md | 10 ++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be55e669..ebc340ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ + +## 0.2.4 (2020-03-10) + + +#### Bug Fixes + +* GETs with a limit and no sort never advance X-Weave-Next-Offset ([c95f2ff2](https://github.com/mozilla-services/syncstorage-rs/commit/c95f2ff21a5e3b428b2715018e7e782b22a2dfa8)) + + + ## 0.2.2 (2020-02-12) diff --git a/Cargo.lock b/Cargo.lock index 6b872d13..e8a50eb7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2442,7 +2442,7 @@ dependencies = [ [[package]] name = "syncstorage" -version = "0.2.2" +version = "0.2.4" dependencies = [ "actix-cors 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-http 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index d2ba1132..4c29e040 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "syncstorage" -version = "0.2.2" +version = "0.2.4" license = "MPL-2.0" authors = [ "Ben Bangert ", From 9b5bda5092ffa8852a812ba4f406358b0e6b780a Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Tue, 10 Mar 2020 21:28:34 -0700 Subject: [PATCH 6/8] fix: relax MAX_TTL to 9 digits Closes #480 --- src/web/extractors.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/web/extractors.rs b/src/web/extractors.rs index fb8727f0..04e2520d 100644 --- a/src/web/extractors.rs +++ b/src/web/extractors.rs @@ -39,7 +39,7 @@ use crate::web::{ const BATCH_MAX_IDS: usize = 100; // BSO const restrictions -const BSO_MAX_TTL: u32 = 31_536_000; +const BSO_MAX_TTL: u32 = 999_999_999; const BSO_MAX_SORTINDEX_VALUE: i32 = 999_999_999; const BSO_MIN_SORTINDEX_VALUE: i32 = -999_999_999; @@ -2279,4 +2279,19 @@ mod tests { assert_eq!(err["errors"][0]["name"], "uid"); */ } + + #[test] + fn test_max_ttl() { + let bso_body = json!([ + {"id": "123", "payload": "xxx", "sortindex": 23, "ttl": 94_608_000}, + {"id": "456", "payload": "xxxasdf", "sortindex": 23, "ttl": 999_999_999}, + {"id": "789", "payload": "xxxfoo", "sortindex": 23, "ttl": 1_000_000_000} + ]); + let result = post_collection("", &bso_body).unwrap(); + assert_eq!(result.user_id.legacy_id, *USER_ID); + assert_eq!(&result.collection, "tabs"); + assert_eq!(result.bsos.valid.len(), 2); + assert_eq!(result.bsos.invalid.len(), 1); + assert!(result.bsos.invalid.contains_key("789")); + } } From 55c6a120b978a6d4e87e8a0a27456c3ec792d32b Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Wed, 11 Mar 2020 11:05:44 -0700 Subject: [PATCH 7/8] chore: tag 0.2.5 --- CHANGELOG.md | 10 ++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebc340ce..6639b1a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ + +## 0.2.5 (2020-03-11) + + +#### Bug Fixes + +* relax MAX_TTL to 9 digits ([9b5bda50](https://github.com/mozilla-services/syncstorage-rs/commit/9b5bda5092ffa8852a812ba4f406358b0e6b780a), closes [#480](https://github.com/mozilla-services/syncstorage-rs/issues/480)) + + + ## 0.2.4 (2020-03-10) diff --git a/Cargo.lock b/Cargo.lock index e8a50eb7..a3ee2c12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2442,7 +2442,7 @@ dependencies = [ [[package]] name = "syncstorage" -version = "0.2.4" +version = "0.2.5" dependencies = [ "actix-cors 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-http 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 4c29e040..477f96e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "syncstorage" -version = "0.2.4" +version = "0.2.5" license = "MPL-2.0" authors = [ "Ben Bangert ", From fceea69e324b3d4d33b8d06eb614f1e944996a9b Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Wed, 11 Mar 2020 11:21:25 -0700 Subject: [PATCH 8/8] fix: adapt to async --- src/web/extractors.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/web/extractors.rs b/src/web/extractors.rs index d11d5bdc..8b93d823 100644 --- a/src/web/extractors.rs +++ b/src/web/extractors.rs @@ -2343,14 +2343,16 @@ mod tests { */ } - #[test] - fn test_max_ttl() { + #[actix_rt::test] + async fn test_max_ttl() { let bso_body = json!([ {"id": "123", "payload": "xxx", "sortindex": 23, "ttl": 94_608_000}, {"id": "456", "payload": "xxxasdf", "sortindex": 23, "ttl": 999_999_999}, {"id": "789", "payload": "xxxfoo", "sortindex": 23, "ttl": 1_000_000_000} ]); - let result = post_collection("", &bso_body).unwrap(); + let result = post_collection("", &bso_body) + .await + .expect("Could not get result in test_valid_collection_post_request"); assert_eq!(result.user_id.legacy_id, *USER_ID); assert_eq!(&result.collection, "tabs"); assert_eq!(result.bsos.valid.len(), 2);