diff --git a/.circleci/config.yml b/.circleci/config.yml index f13e3629..a27e4b5a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ commands: steps: - run: name: Display Rust Version - command: + command: rustc --version setup-rust-check: steps: @@ -43,22 +43,23 @@ commands: flake8 syncserver/src/tokenserver flake8 tools/integration_tests flake8 tools/tokenserver - rust-clippy: + rust-clippy-mysql: steps: - run: - name: Rust Clippy + name: Rust Clippy MySQL command: | - cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql -- -D warnings + rust-clippy-spanner: + steps: + - run: + name: Rust Clippy Spanner + command: | + cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner -- -D warnings cargo-build: steps: - run: name: cargo build command: cargo build - setup-gcp-grpc: - steps: - - run: - name: Install grpcio dependencies - command: sudo apt-get update && sudo apt-get install -y cmake golang-go setup-mysql: steps: - run: @@ -69,7 +70,7 @@ commands: - run: name: Create Tokenserver database command: | - mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' + mysql -u root -ppassword -h 127.0.0.1 -e 'CREATE DATABASE tokenserver;' mysql -u root -ppassword -h 127.0.0.1 -e "GRANT ALL ON tokenserver.* to 'test'@'%';" write-version: @@ -105,7 +106,7 @@ commands: -f docker-compose.mysql.yaml -f docker-compose.e2e.mysql.yaml up - --exit-code-from e2e-tests + --exit-code-from mysql-e2e-tests --abort-on-container-exit environment: SYNCSTORAGE_RS_IMAGE: app:build @@ -129,7 +130,7 @@ commands: -f docker-compose.spanner.yaml -f docker-compose.e2e.spanner.yaml up - --exit-code-from e2e-tests + --exit-code-from spanner-e2e-tests --abort-on-container-exit environment: SYNCSTORAGE_RS_IMAGE: app:build @@ -164,13 +165,14 @@ jobs: auth: username: $DOCKER_USER password: $DOCKER_PASS + resource_class: large steps: - checkout - display-rust - setup-rust-check - - setup-gcp-grpc - rust-check - - rust-clippy + - rust-clippy-spanner + - rust-clippy-mysql - setup-python - python-check @@ -197,20 +199,9 @@ jobs: MYSQL_DATABASE: syncstorage resource_class: large steps: - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Login to Dockerhub - command: | - if [ "${DOCKER_USER}" == "" ] || [ "${DOCKER_PASS}" == "" ]; then - echo "Skipping Login to DockerHub, credentials unavailable" - else - echo "${DOCKER_PASS}" | docker login -u="${DOCKER_USER}" --password-stdin - fi - checkout - display-rust - setup-python - - setup-gcp-grpc - setup-mysql - create-tokenserver-database # XXX: currently the time needed to setup-sccache negates its savings @@ -221,9 +212,21 @@ jobs: - run-tests - run-tokenserver-scripts-tests #- save-sccache-cache + build-mysql-image: + docker: + - image: cimg/rust:1.60.0 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + resource_class: large + steps: + - setup_remote_docker: + docker_layer_caching: true + - checkout + - write-version - run: - name: Build Docker image - command: docker build -t app:build . + name: Build MySQL Docker image + command: docker build -t app:build --build-arg DATABASE_BACKEND=mysql . no_output_timeout: 30m # save the built docker container into CircleCI's cache. This is # required since Workflows do not have the same remote docker instance. @@ -234,13 +237,44 @@ jobs: docker save -o /home/circleci/cache/docker.tar "app:build" - run: name: Save docker-compose config - command: cp docker-compose*.yaml /home/circleci/cache + command: cp docker-compose*mysql.yaml /home/circleci/cache - save_cache: - key: v1-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }} + key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }} paths: - /home/circleci/cache - e2e-tests: + build-spanner-image: + docker: + - image: cimg/rust:1.60.0 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + resource_class: large + steps: + - setup_remote_docker: + docker_layer_caching: true + - checkout + - write-version + - run: + name: Build Spanner Docker image + command: docker build -t app:build --build-arg DATABASE_BACKEND=spanner . + no_output_timeout: 30m + # save the built docker container into CircleCI's cache. This is + # required since Workflows do not have the same remote docker instance. + - run: + name: docker save app:build + command: | + mkdir -p /home/circleci/cache + docker save -o /home/circleci/cache/docker.tar "app:build" + - run: + name: Save docker-compose config + command: cp docker-compose*spanner.yaml /home/circleci/cache + - save_cache: + key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }} + paths: + - /home/circleci/cache + + mysql-e2e-tests: docker: - image: docker/compose:1.24.0 auth: @@ -249,7 +283,7 @@ jobs: steps: - setup_remote_docker - restore_cache: - key: v1-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} + key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} - run: name: Restore Docker image cache command: docker load -i /home/circleci/cache/docker.tar @@ -257,6 +291,23 @@ jobs: name: Restore docker-compose config command: cp /home/circleci/cache/docker-compose*.yaml . - run-e2e-mysql-tests + + spanner-e2e-tests: + docker: + - image: docker/compose:1.24.0 + auth: + username: $DOCKER_USER + password: $DOCKER_PASS + steps: + - setup_remote_docker + - restore_cache: + key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} + - run: + name: Restore Docker image cache + command: docker load -i /home/circleci/cache/docker.tar + - run: + name: Restore docker-compose config + command: cp /home/circleci/cache/docker-compose*.yaml . - run-e2e-spanner-tests deploy: @@ -268,7 +319,7 @@ jobs: steps: - setup_remote_docker - restore_cache: - key: v1-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} + key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }} - run: name: Restore Docker image cache command: docker load -i /home/circleci/cache/docker.tar @@ -346,21 +397,41 @@ workflows: filters: tags: only: /.*/ - - e2e-tests: + - build-mysql-image: requires: - build-and-test filters: tags: only: /.*/ + - build-spanner-image: + requires: + - build-and-test + filters: + tags: + only: /.*/ + - mysql-e2e-tests: + requires: + - build-mysql-image + filters: + tags: + only: /.*/ + - spanner-e2e-tests: + requires: + - build-spanner-image + filters: + tags: + only: /.*/ - deploy: requires: - - e2e-tests + - mysql-e2e-tests + - spanner-e2e-tests filters: tags: only: /.*/ - deploy-python-utils: requires: - - e2e-tests + - mysql-e2e-tests + - spanner-e2e-tests filters: tags: only: /.*/ diff --git a/Cargo.lock b/Cargo.lock index 00406a87..48c34c76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ "futures-core", "futures-sink", "log", - "pin-project 0.4.29", + "pin-project 0.4.30", "tokio", "tokio-util", ] @@ -63,7 +63,7 @@ dependencies = [ "actix-service", "actix-threadpool", "actix-utils", - "base64 0.13.0", + "base64 0.13.1", "bitflags", "brotli", "bytes 0.5.6", @@ -86,11 +86,11 @@ dependencies = [ "lazy_static", "log", "mime", - "percent-encoding 2.1.0", - "pin-project 1.0.10", + "percent-encoding 2.2.0", + "pin-project 1.0.12", "rand 0.7.3", "regex", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "serde_urlencoded", "sha-1", @@ -118,7 +118,7 @@ dependencies = [ "http", "log", "regex", - "serde 1.0.135", + "serde 1.0.153", ] [[package]] @@ -163,7 +163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0052435d581b5be835d11f4eb3bce417c8af18d87ddf8ace99f8e67e595882bb" dependencies = [ "futures-util", - "pin-project 0.4.29", + "pin-project 0.4.30", ] [[package]] @@ -191,7 +191,7 @@ dependencies = [ "lazy_static", "log", "num_cpus", - "parking_lot", + "parking_lot 0.11.2", "threadpool", ] @@ -223,7 +223,7 @@ dependencies = [ "futures-sink", "futures-util", "log", - "pin-project 0.4.29", + "pin-project 0.4.30", "slab", ] @@ -255,15 +255,15 @@ dependencies = [ "fxhash", "log", "mime", - "pin-project 1.0.10", + "pin-project 1.0.12", "regex", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "serde_urlencoded", "socket2 0.3.19", "time 0.2.27", "tinyvec", - "url 2.2.2", + "url 2.3.1", ] [[package]] @@ -279,9 +279,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.17.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" dependencies = [ "gimli", ] @@ -294,39 +294,48 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "aho-corasick" -version = "0.7.18" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] [[package]] name = "alloc-no-stdlib" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] name = "alloc-stdlib" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" dependencies = [ "alloc-no-stdlib", ] [[package]] -name = "anyhow" -version = "1.0.53" +name = "android_system_properties" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a45b455c14666b85fc40a019e8ab9eb75e3a124e05494f5397122bc9eb06e0" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "arc-swap" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d78ce20460b82d3fa150275ed9d55e21064fc7951177baacf86a145c4a4b1f" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" [[package]] name = "arrayvec" @@ -336,19 +345,19 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "assert-json-diff" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f1c3703dd33532d7f0ca049168930e9099ecac238e23cf932f3a69c42f06da" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" dependencies = [ - "serde 1.0.135", + "serde 1.0.153", "serde_json", ] [[package]] name = "async-trait" -version = "0.1.53" +version = "0.1.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" dependencies = [ "proc-macro2", "quote", @@ -361,16 +370,16 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi 0.3.9", ] [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "awc" @@ -382,40 +391,40 @@ dependencies = [ "actix-http", "actix-rt", "actix-service", - "base64 0.13.0", + "base64 0.13.1", "bytes 0.5.6", "cfg-if 1.0.0", "derive_more", "futures-core", "log", "mime", - "percent-encoding 2.1.0", + "percent-encoding 2.2.0", "rand 0.7.3", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "serde_urlencoded", ] [[package]] name = "backtrace" -version = "0.3.65" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" dependencies = [ "addr2line", "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide 0.5.1", + "miniz_oxide", "object", "rustc-demangle", ] [[package]] name = "base-x" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" +checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" [[package]] name = "base64" @@ -425,20 +434,9 @@ checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" - -[[package]] -name = "bb8" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374bba43fc924d90393ee7768e6f75d223a98307a488fe5bc34b66c3e96932a6" -dependencies = [ - "async-trait", - "futures 0.3.19", - "tokio", -] +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "bindgen" @@ -494,9 +492,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.3.3" +version = "3.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f838e47a451d5a8fa552371f80024dd6ace9b7acdf25c4c3d0f9bc6816fb1c39" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -505,9 +503,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.2" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -515,9 +513,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.9.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "byteorder" @@ -533,17 +531,17 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.1.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bytestring" -version = "1.0.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", ] [[package]] @@ -557,9 +555,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.72" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cexpr" @@ -584,23 +582,25 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" dependencies = [ - "libc", + "iana-time-zone", + "js-sys", "num-integer", - "num-traits 0.2.14", - "serde 1.0.135", - "time 0.1.43", + "num-traits 0.2.15", + "serde 1.0.153", + "time 0.1.45", + "wasm-bindgen", "winapi 0.3.9", ] [[package]] name = "clang-sys" -version = "1.3.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa66045b9cb23c2e9c1520732030608b02ee07e5cfaa5a521ec15ded7fa24c90" +checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" dependencies = [ "glob", "libc", @@ -609,13 +609,23 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.45" +version = "0.1.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb6210b637171dfba4cda12e579ac6dc73f5165ad56133e5d72ef3131f320855" +checksum = "db34956e100b30725f2eb215f90d4871051239535632f84fea3bc92722c66b7c" dependencies = [ "cc", ] +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "colored" version = "2.0.0" @@ -635,7 +645,7 @@ checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", "nom", - "serde 1.0.135", + "serde 1.0.153", ] [[package]] @@ -647,7 +657,7 @@ dependencies = [ "lazy_static", "nom", "rust-ini", - "serde 1.0.135", + "serde 1.0.153", "serde-hjson", "serde_json", "toml", @@ -672,7 +682,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ - "percent-encoding 2.1.0", + "percent-encoding 2.2.0", "time 0.2.27", "version_check", ] @@ -685,9 +695,9 @@ checksum = "a2df960f5d869b2dd8532793fde43eb5427cceb126c929747a26823ab0eeb536" [[package]] name = "core-foundation" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" dependencies = [ "core-foundation-sys", "libc", @@ -701,30 +711,30 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.1" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2209c310e29876f7f0b2721e7e26b84aff178aa3da5d091f9bfbf47669e60e3" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.6", + "crossbeam-utils 0.8.15", ] [[package]] @@ -751,12 +761,11 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.6" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if 1.0.0", - "lazy_static", ] [[package]] @@ -771,24 +780,24 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.42" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de97b894edd5b5bcceef8b78d7da9b75b1d2f2f9a910569d0bde3dd31d84939" +checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" dependencies = [ "curl-sys", "libc", "openssl-probe", "openssl-sys", "schannel", - "socket2 0.4.3", + "socket2 0.4.9", "winapi 0.3.9", ] [[package]] name = "curl-sys" -version = "0.4.52+curl-7.81.0" +version = "0.4.60+curl-7.88.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b8c2d1023ea5fded5b7b892e4b8e95f70038a421126a056761a84246a28971" +checksum = "717abe2cb465a5da6ce06617388a3980c9a2844196734bec8ccb8e575250f13f" dependencies = [ "cc", "libc", @@ -799,6 +808,50 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "cxx" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "deadpool" version = "0.5.2" @@ -808,17 +861,17 @@ dependencies = [ "config 0.10.1", "crossbeam-queue", "num_cpus", - "serde 1.0.135", + "serde 1.0.153", "tokio", ] [[package]] name = "debugid" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba" +checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" dependencies = [ - "serde 1.0.135", + "serde 1.0.153", "uuid", ] @@ -929,36 +982,36 @@ checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" dependencies = [ "lazy_static", "regex", - "serde 1.0.135", + "serde 1.0.153", "strsim", ] [[package]] name = "dyn-clone" -version = "1.0.5" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e50f3adc76d6a43f5ed73b698a87d0760ca74617f60f7c3b879003536fdd28" +checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" [[package]] name = "either" -version = "1.6.1" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "encoding_rs" -version = "0.8.30" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "enum-as-inner" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +checksum = "570d109b813e904becc80d8d5da38376818a143348413f7149f1340fe04754d4" dependencies = [ "heck", "proc-macro2", @@ -968,9 +1021,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.0" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" dependencies = [ "atty", "humantime", @@ -981,11 +1034,11 @@ dependencies = [ [[package]] name = "erased-serde" -version = "0.3.18" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56047058e1ab118075ca22f9ecd737bcc961aa3566a3019cb71388afa280bd8a" +checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" dependencies = [ - "serde 1.0.135", + "serde 1.0.153", ] [[package]] @@ -1033,23 +1086,21 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] [[package]] name = "flate2" -version = "1.0.22" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" dependencies = [ - "cfg-if 1.0.0", "crc32fast", - "libc", - "miniz_oxide 0.4.4", + "miniz_oxide", ] [[package]] @@ -1075,12 +1126,11 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" dependencies = [ - "matches", - "percent-encoding 2.1.0", + "percent-encoding 2.2.0", ] [[package]] @@ -1107,9 +1157,9 @@ checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" [[package]] name = "futures" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -1122,9 +1172,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -1132,15 +1182,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -1149,15 +1199,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-macro" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -1166,21 +1216,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-util" -version = "0.3.19" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures 0.1.31", "futures-channel", @@ -1190,7 +1240,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "pin-utils", "slab", ] @@ -1206,9 +1256,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" dependencies = [ "typenum", "version_check", @@ -1227,26 +1277,26 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.4" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.2+wasi-snapshot-preview1", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] name = "gimli" -version = "0.26.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "google-cloud-rust-raw" @@ -1254,7 +1304,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0f0936883f3207fa424f69fc218956a5778de6fb847ea3c491f1dc47a39fb26" dependencies = [ - "futures 0.3.19", + "futures 0.3.26", "grpcio", "protobuf", ] @@ -1265,11 +1315,11 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d99e00eed7e0a04ee2705112e7cfdbe1a3cc771147f22f016a8cd2d002187b" dependencies = [ - "futures 0.3.19", + "futures 0.3.26", "grpcio-sys", "libc", "log", - "parking_lot", + "parking_lot 0.11.2", "protobuf", ] @@ -1311,9 +1361,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hawk" @@ -1327,17 +1377,14 @@ dependencies = [ "once_cell", "ring", "thiserror", - "url 2.2.2", + "url 2.3.1", ] [[package]] name = "heck" -version = "0.3.3" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -1348,6 +1395,15 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -1387,13 +1443,13 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.1.0", + "bytes 1.4.0", "fnv", - "itoa 1.0.1", + "itoa 1.0.6", ] [[package]] @@ -1408,9 +1464,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.5.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" @@ -1440,7 +1496,7 @@ dependencies = [ "httparse", "httpdate", "itoa 0.4.8", - "pin-project 1.0.10", + "pin-project 1.0.12", "socket2 0.3.19", "tokio", "tower-service", @@ -1477,6 +1533,30 @@ dependencies = [ "tokio-tls", ] +[[package]] +name = "iana-time-zone" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "winapi 0.3.9", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +dependencies = [ + "cxx", + "cxx-build", +] + [[package]] name = "idna" version = "0.1.5" @@ -1499,6 +1579,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "if_chain" version = "1.0.2" @@ -1521,9 +1611,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" dependencies = [ "autocfg", "hashbrown", @@ -1594,9 +1684,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.3.1" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "itoa" @@ -1606,15 +1696,15 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.1" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.56" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -1668,9 +1758,9 @@ checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libloading" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ "cfg-if 1.0.0", "winapi 0.3.9", @@ -1678,9 +1768,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.3" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" dependencies = [ "cc", "libc", @@ -1689,10 +1779,19 @@ dependencies = [ ] [[package]] -name = "linked-hash-map" -version = "0.5.4" +name = "link-cplusplus" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +dependencies = [ + "cc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" @@ -1702,18 +1801,19 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "lock_api" -version = "0.4.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" dependencies = [ + "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", ] @@ -1735,9 +1835,9 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matches" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "maybe-uninit" @@ -1747,9 +1847,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "migrations_internals" @@ -1780,9 +1880,9 @@ checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" [[package]] name = "mime_guess" -version = "2.0.3" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" dependencies = [ "mime", "unicase", @@ -1790,19 +1890,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.4.4" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" -dependencies = [ - "adler", - "autocfg", -] - -[[package]] -name = "miniz_oxide" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" dependencies = [ "adler", ] @@ -1869,9 +1959,9 @@ dependencies = [ [[package]] name = "mysqlclient-sys" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9637d93448044078aaafea7419aed69d301b4a12bcc4aa0ae856eb169bef85" +checksum = "f61b381528ba293005c42a409dd73d034508e273bf90481f17ec2e964a6e969b" dependencies = [ "pkg-config", "vcpkg", @@ -1879,9 +1969,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" dependencies = [ "lazy_static", "libc", @@ -1897,9 +1987,9 @@ dependencies = [ [[package]] name = "net2" -version = "0.2.37" +version = "0.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +checksum = "74d0df99cfcd2530b2e694f6e17e7f37b8e26bb23983ac530c0c97408837c631" dependencies = [ "cfg-if 0.1.10", "libc", @@ -1919,12 +2009,12 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" dependencies = [ "autocfg", - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] @@ -1933,51 +2023,51 @@ version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.14", + "num-traits 0.2.15", ] [[package]] name = "num-traits" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] [[package]] name = "num_threads" -version = "0.1.2" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71a1eb3a36534514077c1e079ada2fb170ef30c47d203aa6916138cf882ecd52" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" dependencies = [ "libc", ] [[package]] name = "object" -version = "0.28.3" +version = "0.30.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40bec70ba014595f99f7aa110b84331ffe1ee9aece7fe6f387cc7e3ecda4d456" +checksum = "ea86265d3d3dcb6a27fc51bd29a4bf387fae9d2986b823079d4986af253eb439" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.9.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "opaque-debug" @@ -1987,18 +2077,30 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.38" +version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95" +checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ "bitflags", "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", + "openssl-macros", "openssl-sys", ] +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -2007,9 +2109,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.72" +version = "0.9.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb" +checksum = "23bbbf7854cd45b83958ebe919f0e8e516793727652e27fda10a8384cfc790b7" dependencies = [ "autocfg", "cc", @@ -2026,14 +2128,24 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.7", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" dependencies = [ "cfg-if 1.0.0", "instant", @@ -2043,6 +2155,19 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "parking_lot_core" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "smallvec", + "windows-sys 0.45.0", +] + [[package]] name = "paste" version = "0.1.18" @@ -2076,33 +2201,33 @@ checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" [[package]] name = "percent-encoding" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pin-project" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9615c18d31137579e9ff063499264ddc1278e7b1982757ebc111028c4d1dc909" +checksum = "3ef0f924a5ee7ea9cbcea77529dba45f8a9ba9f622419fe3386ca581a3ae9d5a" dependencies = [ - "pin-project-internal 0.4.29", + "pin-project-internal 0.4.30", ] [[package]] name = "pin-project" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" dependencies = [ - "pin-project-internal 1.0.10", + "pin-project-internal 1.0.12", ] [[package]] name = "pin-project-internal" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "044964427019eed9d49d9d5bbce6047ef18f37100ea400912a9fa4a3523ab12a" +checksum = "851c8d0ce9bebe43790dedfc86614c23494ac9f423dd618d3a61fc693eafe61e" dependencies = [ "proc-macro2", "quote", @@ -2111,9 +2236,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", @@ -2128,9 +2253,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e280fbe77cc62c91527259e9442153f4688736748d24660126286329742b4c6c" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -2140,15 +2265,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.24" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-error" @@ -2176,24 +2301,24 @@ dependencies = [ [[package]] name = "proc-macro-hack" -version = "0.5.19" +version = "0.5.20+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "protobuf" -version = "2.25.2" +version = "2.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c327e191621a2158159df97cdbc2e7074bb4e940275e35abf38eb3d2595754" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" [[package]] name = "pyo3" @@ -2204,7 +2329,7 @@ dependencies = [ "cfg-if 1.0.0", "indoc", "libc", - "parking_lot", + "parking_lot 0.11.2", "paste", "pyo3-build-config", "pyo3-macros", @@ -2251,21 +2376,21 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.15" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] [[package]] name = "r2d2" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" dependencies = [ "log", - "parking_lot", + "parking_lot 0.12.1", "scheduled-thread-pool", ] @@ -2290,7 +2415,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha 0.3.1", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2310,7 +2435,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.3", + "rand_core 0.6.4", ] [[package]] @@ -2324,11 +2449,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.4", + "getrandom 0.2.8", ] [[package]] @@ -2351,28 +2476,29 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.10" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.0" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom 0.2.4", + "getrandom 0.2.8", "redox_syscall", + "thiserror", ] [[package]] name = "regex" -version = "1.5.5" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -2381,9 +2507,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" [[package]] name = "reqwest" @@ -2391,7 +2517,7 @@ version = "0.10.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0718f81a8e14c4dbb3b34cf23dc6aaf9ab8a0dfec160c534b3dbca1aaa21f47c" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "bytes 0.5.6", "encoding_rs", "futures-core", @@ -2408,16 +2534,16 @@ dependencies = [ "mime", "mime_guess", "native-tls", - "percent-encoding 2.1.0", - "pin-project-lite 0.2.8", + "percent-encoding 2.2.0", + "pin-project-lite 0.2.9", "rustls", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "serde_urlencoded", "tokio", "tokio-rustls", "tokio-tls", - "url 2.2.2", + "url 2.3.1", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -2483,7 +2609,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.4", + "semver 1.0.16", ] [[package]] @@ -2515,15 +2641,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.6" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "ryu" -version = "1.0.9" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "same-file" @@ -2536,21 +2662,20 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "lazy_static", - "winapi 0.3.9", + "windows-sys 0.42.0", ] [[package]] name = "scheduled-thread-pool" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" dependencies = [ - "parking_lot", + "parking_lot 0.12.1", ] [[package]] @@ -2559,6 +2684,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" + [[package]] name = "sct" version = "0.6.1" @@ -2571,9 +2702,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.5.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09d3c15d814eda1d6a836f2f2b56a6abc1446c8a34351cb3180d3db92ffe4ce" +checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ "bitflags", "core-foundation", @@ -2584,9 +2715,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.5.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e90dd10c41c6bfc633da6e0c659bd25d31e0791e5974ac42970267d59eba87f7" +checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" dependencies = [ "core-foundation-sys", "libc", @@ -2603,9 +2734,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.4" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012" +checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" [[package]] name = "semver-parser" @@ -2698,10 +2829,10 @@ checksum = "87b41bac48a3586249431fa9efb88cd1414c3455117eb57c02f5bda9634e158d" dependencies = [ "chrono", "debugid", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "thiserror", - "url 2.2.2", + "url 2.3.1", "uuid", ] @@ -2713,9 +2844,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.135" +version = "1.0.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf9235533494ea2ddcdb794665461814781c53f19d87b76e571a1c35acbad2b" +checksum = "3a382c72b4ba118526e187430bb4963cd6d55051ebf13d9b25574d379cc98d20" dependencies = [ "serde_derive", ] @@ -2734,9 +2865,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.135" +version = "1.0.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8dcde03d87d4c973c04be249e7d8f0b35db1c848c487bd43032808e59dd8328d" +checksum = "1ef476a5790f0f6decbc66726b6e5d63680ed518283e64c7df415989d880954f" dependencies = [ "proc-macro2", "quote", @@ -2745,13 +2876,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.78" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23c1ba4cf0efd44be32017709280b32d1cea5c3f1275c3b6d9e8bc54f758085" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.6", "ryu", - "serde 1.0.135", + "serde 1.0.153", ] [[package]] @@ -2761,9 +2892,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.1", + "itoa 1.0.6", "ryu", - "serde 1.0.135", + "serde 1.0.153", ] [[package]] @@ -2815,9 +2946,9 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -2834,9 +2965,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +dependencies = [ + "autocfg", +] [[package]] name = "slog" @@ -2881,7 +3015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f400f1c5db96f1f52065e8931ca0c524cceb029f7537c9e6d5424488ca137ca0" dependencies = [ "chrono", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "slog", ] @@ -2918,14 +3052,14 @@ dependencies = [ "slog", "term", "thread_local", - "time 0.3.9", + "time 0.3.20", ] [[package]] name = "smallvec" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" @@ -2940,9 +3074,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.3" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f82496b90c36d70af5fcd482edaa2e0bd16fade569de1330405fecbbdac736b" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi 0.3.9", @@ -2991,7 +3125,7 @@ checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" dependencies = [ "proc-macro2", "quote", - "serde 1.0.135", + "serde 1.0.153", "serde_derive", "syn", ] @@ -3005,7 +3139,7 @@ dependencies = [ "base-x", "proc-macro2", "quote", - "serde 1.0.135", + "serde 1.0.153", "serde_derive", "serde_json", "sha1", @@ -3032,13 +3166,13 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -3048,44 +3182,28 @@ dependencies = [ "actix-cors", "actix-http", "actix-rt", - "actix-service", "actix-web", "async-trait", "backtrace", - "base64 0.13.0", - "bb8", - "bytes 1.1.0", + "base64 0.13.1", "cadence", "chrono", - "deadpool", - "diesel", - "diesel_logger", - "diesel_migrations", "docopt", "dyn-clone", "env_logger", - "futures 0.3.19", - "google-cloud-rust-raw", - "grpcio", + "futures 0.3.26", "hawk", "hex", "hmac", "hostname", - "http", "lazy_static", - "log", "mime", - "mockito", - "num_cpus", - "protobuf", - "pyo3", "rand 0.8.5", "regex", "reqwest", - "scheduled-thread-pool", "sentry", "sentry-backtrace", - "serde 1.0.135", + "serde 1.0.153", "serde_derive", "serde_json", "sha2", @@ -3099,15 +3217,16 @@ dependencies = [ "syncserver-common", "syncserver-db-common", "syncserver-settings", + "syncstorage-db", "syncstorage-settings", "thiserror", - "time 0.3.9", + "time 0.3.20", + "tokenserver-auth", "tokenserver-common", + "tokenserver-db", "tokenserver-settings", "tokio", - "url 2.2.2", "urlencoding", - "uuid", "validator", "validator_derive", "woothee", @@ -3117,30 +3236,29 @@ dependencies = [ name = "syncserver-common" version = "0.13.6" dependencies = [ + "actix-web", + "cadence", + "futures 0.3.26", "hkdf", + "serde 1.0.153", + "serde_json", "sha2", + "slog", + "slog-scope", ] [[package]] name = "syncserver-db-common" version = "0.13.6" dependencies = [ - "async-trait", "backtrace", - "chrono", "deadpool", "diesel", "diesel_migrations", - "futures 0.3.19", - "grpcio", - "hostname", + "futures 0.3.26", "http", - "lazy_static", - "serde 1.0.135", - "serde_json", "syncserver-common", "thiserror", - "url 2.2.2", ] [[package]] @@ -3149,12 +3267,77 @@ version = "0.13.6" dependencies = [ "config 0.11.0", "num_cpus", - "serde 1.0.135", + "serde 1.0.153", "slog-scope", "syncserver-common", "syncstorage-settings", "tokenserver-settings", - "url 2.2.2", + "url 2.3.1", +] + +[[package]] +name = "syncstorage-db" +version = "0.13.6" +dependencies = [ + "async-trait", + "cadence", + "env_logger", + "futures 0.3.26", + "hostname", + "lazy_static", + "log", + "rand 0.8.5", + "slog-scope", + "syncserver-common", + "syncserver-db-common", + "syncserver-settings", + "syncstorage-db-common", + "syncstorage-mysql", + "syncstorage-settings", + "syncstorage-spanner", + "tokio", +] + +[[package]] +name = "syncstorage-db-common" +version = "0.13.6" +dependencies = [ + "async-trait", + "backtrace", + "chrono", + "diesel", + "diesel_migrations", + "futures 0.3.26", + "http", + "lazy_static", + "serde 1.0.153", + "serde_json", + "syncserver-common", + "syncserver-db-common", + "thiserror", +] + +[[package]] +name = "syncstorage-mysql" +version = "0.13.6" +dependencies = [ + "async-trait", + "backtrace", + "base64 0.13.1", + "diesel", + "diesel_logger", + "diesel_migrations", + "env_logger", + "futures 0.3.26", + "http", + "slog-scope", + "syncserver-common", + "syncserver-db-common", + "syncserver-settings", + "syncstorage-db-common", + "syncstorage-settings", + "thiserror", + "url 2.3.1", ] [[package]] @@ -3162,9 +3345,35 @@ name = "syncstorage-settings" version = "0.13.6" dependencies = [ "rand 0.8.5", - "serde 1.0.135", + "serde 1.0.153", "syncserver-common", - "time 0.3.9", + "time 0.3.20", +] + +[[package]] +name = "syncstorage-spanner" +version = "0.13.6" +dependencies = [ + "async-trait", + "backtrace", + "cadence", + "deadpool", + "env_logger", + "futures 0.3.26", + "google-cloud-rust-raw", + "grpcio", + "http", + "log", + "protobuf", + "slog-scope", + "syncserver-common", + "syncserver-db-common", + "syncstorage-db-common", + "syncstorage-settings", + "thiserror", + "tokio", + "url 2.3.1", + "uuid", ] [[package]] @@ -3211,27 +3420,27 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ "proc-macro2", "quote", @@ -3240,10 +3449,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if 1.0.0", "once_cell", ] @@ -3258,11 +3468,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", + "wasi 0.10.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -3283,16 +3494,24 @@ dependencies = [ [[package]] name = "time" -version = "0.3.9" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "itoa 1.0.1", + "itoa 1.0.6", "libc", "num_threads", - "time-macros 0.2.4", + "serde 1.0.153", + "time-core", + "time-macros 0.2.8", ] +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + [[package]] name = "time-macros" version = "0.1.1" @@ -3305,9 +3524,12 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.4" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] [[package]] name = "time-macros-impl" @@ -3324,18 +3546,36 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" dependencies = [ "tinyvec_macros", ] [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokenserver-auth" +version = "0.13.6" +dependencies = [ + "async-trait", + "dyn-clone", + "futures 0.3.26", + "mockito", + "pyo3", + "reqwest", + "serde 1.0.153", + "serde_json", + "syncserver-common", + "tokenserver-common", + "tokenserver-settings", + "tokio", +] [[package]] name = "tokenserver-common" @@ -3343,18 +3583,42 @@ version = "0.13.6" dependencies = [ "actix-web", "backtrace", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "syncserver-common", - "syncserver-db-common", "thiserror", ] +[[package]] +name = "tokenserver-db" +version = "0.13.6" +dependencies = [ + "async-trait", + "backtrace", + "diesel", + "diesel_logger", + "diesel_migrations", + "env_logger", + "futures 0.3.26", + "http", + "serde 1.0.153", + "serde_derive", + "serde_json", + "slog-scope", + "syncserver-common", + "syncserver-db-common", + "syncserver-settings", + "thiserror", + "tokenserver-common", + "tokenserver-settings", + "tokio", +] + [[package]] name = "tokenserver-settings" version = "0.13.6" dependencies = [ - "serde 1.0.135", + "serde 1.0.153", "tokenserver-common", ] @@ -3430,38 +3694,38 @@ dependencies = [ [[package]] name = "toml" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ - "serde 1.0.135", + "serde 1.0.153", ] [[package]] name = "tower-service" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.29" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.8", + "pin-project-lite 0.2.9", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.21" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ - "lazy_static", + "once_cell", ] [[package]] @@ -3470,7 +3734,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 1.0.10", + "pin-project 1.0.12", "tracing", ] @@ -3483,7 +3747,7 @@ dependencies = [ "async-trait", "cfg-if 1.0.0", "enum-as-inner", - "futures 0.3.19", + "futures 0.3.26", "idna 0.2.3", "lazy_static", "log", @@ -3491,7 +3755,7 @@ dependencies = [ "smallvec", "thiserror", "tokio", - "url 2.2.2", + "url 2.3.1", ] [[package]] @@ -3501,7 +3765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "710f593b371175db53a26d0b38ed2978fafb9e9e8d3868b1acd753ea18df0ceb" dependencies = [ "cfg-if 0.1.10", - "futures 0.3.19", + "futures 0.3.26", "ipconfig", "lazy_static", "log", @@ -3515,15 +3779,15 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "typenum" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "uname" @@ -3545,36 +3809,42 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.7" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" + +[[package]] +name = "unicode-ident" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" -version = "0.1.19" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" dependencies = [ "tinyvec", ] [[package]] -name = "unicode-segmentation" -version = "1.8.0" +name = "unicode-width" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] name = "unicode-xid" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unindent" -version = "0.1.7" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" +checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" [[package]] name = "untrusted" @@ -3595,22 +3865,21 @@ dependencies = [ [[package]] name = "url" -version = "2.2.2" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" dependencies = [ "form_urlencoded", - "idna 0.2.3", - "matches", - "percent-encoding 2.1.0", - "serde 1.0.135", + "idna 0.3.0", + "percent-encoding 2.2.0", + "serde 1.0.153", ] [[package]] name = "urlencoding" -version = "2.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" [[package]] name = "uuid" @@ -3618,8 +3887,8 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom 0.2.4", - "serde 1.0.135", + "getrandom 0.2.8", + "serde 1.0.153", ] [[package]] @@ -3631,10 +3900,10 @@ dependencies = [ "idna 0.2.3", "lazy_static", "regex", - "serde 1.0.135", + "serde 1.0.153", "serde_derive", "serde_json", - "url 2.2.2", + "url 2.3.1", "validator_types", ] @@ -3705,31 +3974,37 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if 1.0.0", - "serde 1.0.135", + "serde 1.0.153", "serde_json", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", - "lazy_static", "log", + "once_cell", "proc-macro2", "quote", "syn", @@ -3738,9 +4013,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.29" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb6ec270a31b1d3c7e266b999739109abce8b6c87e4b31fcfcd788b65267395" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -3750,9 +4025,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3760,9 +4035,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -3773,15 +4048,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.79" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "web-sys" -version = "0.3.56" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 56eced59..02ec3080 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,17 @@ [workspace] resolver = "2" members = [ - "syncserver-settings", "syncserver-common", "syncserver-db-common", + "syncserver-settings", + "syncstorage-db", + "syncstorage-db-common", + "syncstorage-mysql", "syncstorage-settings", + "syncstorage-spanner", + "tokenserver-auth", "tokenserver-common", + "tokenserver-db", "tokenserver-settings", "syncserver", ] diff --git a/Dockerfile b/Dockerfile index 7a613456..f91592cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,35 @@ -FROM rust:1.65-buster as builder +FROM lukemathwalker/cargo-chef:0.1.50-rust-1.66-buster as chef WORKDIR /app -ADD . /app -ENV PATH=$PATH:/root/.cargo/bin -# temp removed --no-install-recommends due to CI docker build issue + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS cacher +ARG DATABASE_BACKEND=spanner +COPY --from=planner /app/mysql_pubkey.asc mysql_pubkey.asc + +# cmake is required to build grpcio-sys for Spanner builds +RUN \ + echo "deb https://repo.mysql.com/apt/debian/ buster mysql-8.0" >> /etc/apt/sources.list && \ + # mysql_pubkey.asc from: + # https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html + # related: + # https://dev.mysql.com/doc/mysql-apt-repo-quick-guide/en/#repo-qg-apt-repo-manual-setup + apt-key adv --import mysql_pubkey.asc && \ + apt-get -q update && \ + apt-get -q install -y --no-install-recommends libmysqlclient-dev cmake + +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --release --no-default-features --features=syncstorage-db/$DATABASE_BACKEND --recipe-path recipe.json + +FROM chef as builder +ARG DATABASE_BACKEND=spanner + +COPY . /app +COPY --from=cacher /app/target /app/target +COPY --from=cacher $CARGO_HOME /app/$CARGO_HOME + RUN \ echo "deb https://repo.mysql.com/apt/debian/ buster mysql-8.0" >> /etc/apt/sources.list && \ # mysql_pubkey.asc from: @@ -15,11 +42,13 @@ RUN \ pip3 install -r requirements.txt && \ rm -rf /var/lib/apt/lists/* +ENV PATH=$PATH:/root/.cargo/bin + RUN \ cargo --version && \ rustc --version && \ - cargo install --path ./syncserver --locked --root /app && \ - cargo install --path ./syncserver --locked --root /app --bin purge_ttl + cargo install --path ./syncserver --no-default-features --features=syncstorage-db/$DATABASE_BACKEND --locked --root /app && \ + if [ "$DATABASE_BACKEND" = "spanner" ] ; then cargo install --path ./syncstorage-spanner --locked --root /app --bin purge_ttl ; fi FROM debian:buster-slim WORKDIR /app @@ -55,7 +84,7 @@ COPY --from=builder /app/tools/spanner /app/tools/spanner COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests COPY --from=builder /app/tools/tokenserver /app/tools/tokenserver COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh -COPY --from=builder /app/syncserver/src/db/spanner/schema.ddl /app/schema.ddl +COPY --from=builder /app/syncstorage-spanner/src/schema.ddl /app/schema.ddl RUN chmod +x /app/scripts/prepare-spanner.sh RUN pip3 install -r /app/tools/integration_tests/requirements.txt diff --git a/Makefile b/Makefile index 39629617..dada9059 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,15 @@ PATH_TO_SYNC_SPANNER_KEYS = `pwd`/service-account.json # https://github.com/mozilla-services/server-syncstorage PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc/_cython/_credentials/roots.pem -clippy: +SRC_ROOT = $(shell pwd) + +clippy_mysql: # Matches what's run in circleci - cargo clippy --workspace --all-targets --all-features -- -D warnings + cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql -- -D warnings + +clippy_spanner: + # Matches what's run in circleci + cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/spanner -- -D warnings clean: cargo clean @@ -40,14 +46,28 @@ python: python3 -m venv venv venv/bin/python -m pip install -r requirements.txt -run: python - PATH="./venv/bin:$(PATH)" RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml +run_mysql: python + PATH="./venv/bin:$(PATH)" \ + # See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the + # below env var + PYTHONPATH=$(SRC_ROOT)/venv/lib/python3.9/site-packages \ + RUST_LOG=debug \ + RUST_BACKTRACE=full \ + cargo run --no-default-features --features=syncstorage-db/mysql -- --config config/local.toml -run_spanner: - GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) make run +run_spanner: python + GOOGLE_APPLICATION_CREDENTIALS=$(PATH_TO_SYNC_SPANNER_KEYS) \ + GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=$(PATH_TO_GRPC_CERT) \ + # See https://github.com/PyO3/pyo3/issues/1741 for discussion re: why we need to set the + # below env var + PYTHONPATH=$(SRC_ROOT)/venv/lib/python3.9/site-packages \ + PATH="./venv/bin:$(PATH)" \ + RUST_LOG=debug \ + RUST_BACKTRACE=full \ + cargo run --no-default-features --features=syncstorage-db/spanner -- --config config/local.toml test: SYNC_SYNCSTORAGE__DATABASE_URL=mysql://sample_user:sample_password@localhost/syncstorage_rs \ SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \ RUST_TEST_THREADS=1 \ - cargo test + cargo test --workspace diff --git a/config/local.example.toml b/config/local.example.toml index 7e2a687c..f845b5c9 100644 --- a/config/local.example.toml +++ b/config/local.example.toml @@ -1,22 +1,21 @@ -# Example MySQL DSN: -database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs" - -# Example Spanner DSN: -# database_url="spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB" - -"limits.max_total_records"=1666 # See issues #298/#333 master_secret = "INSERT_SECRET_KEY_HERE" # removing this line will default to moz_json formatted logs (which is preferred for production envs) human_logs = 1 +# Example Syncstorage settings: +# Example MySQL DSN: +syncstorage.database_url = "mysql://sample_user:sample_password@localhost/syncstorage_rs" +# Example Spanner DSN: +# database_url="spanner://projects/SAMPLE_GCP_PROJECT/instances/SAMPLE_SPANNER_INSTANCE/databases/SAMPLE_SPANNER_DB" # enable quota limits -enable_quota = 0 +syncstorage.enable_quota = 0 # set the quota limit to 2GB. # max_quota_limit = 200000000 +syncstorage.enabled = true +syncstorage.limits.max_total_records = 1666 # See issues #298/#333 # Example Tokenserver settings: -disable_syncstorage = false tokenserver.database_url = "mysql://sample_user:sample_password@localhost/tokenserver_rs" tokenserver.enabled = true tokenserver.fxa_email_domain = "api-accounts.stage.mozaws.net" diff --git a/docker-compose.e2e.mysql.yaml b/docker-compose.e2e.mysql.yaml index bb5d6e99..d2d42195 100644 --- a/docker-compose.e2e.mysql.yaml +++ b/docker-compose.e2e.mysql.yaml @@ -14,7 +14,7 @@ services: sleep 15; /app/bin/syncserver; " - e2e-tests: + mysql-e2e-tests: depends_on: - mock-fxa-server - syncserver diff --git a/docker-compose.e2e.spanner.yaml b/docker-compose.e2e.spanner.yaml index b431276b..e0a25a8b 100644 --- a/docker-compose.e2e.spanner.yaml +++ b/docker-compose.e2e.spanner.yaml @@ -14,7 +14,7 @@ services: sleep 15; /app/bin/syncserver; " - e2e-tests: + spanner-e2e-tests: depends_on: - mock-fxa-server - syncserver diff --git a/syncserver-common/Cargo.toml b/syncserver-common/Cargo.toml index 2ceb8e86..d939a016 100644 --- a/syncserver-common/Cargo.toml +++ b/syncserver-common/Cargo.toml @@ -4,5 +4,16 @@ version = "0.13.6" edition = "2021" [dependencies] +actix-web = "3" +cadence = "0.26" +futures = "0.3" hkdf = "0.11" sha2 = "0.9" +serde = "1.0" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +slog = { version = "2.5", features = [ + "max_level_info", + "release_max_level_info", + "dynamic-keys", +] } +slog-scope = "4.3" diff --git a/syncserver-common/src/lib.rs b/syncserver-common/src/lib.rs index 6d836ec9..1bbc9e9b 100644 --- a/syncserver-common/src/lib.rs +++ b/syncserver-common/src/lib.rs @@ -1,6 +1,19 @@ +#[macro_use] +extern crate slog_scope; + +mod metrics; + +use std::{ + fmt, + sync::atomic::{AtomicU64, Ordering}, +}; + +use actix_web::{error::BlockingError, web}; use hkdf::Hkdf; use sha2::Sha256; +pub use metrics::{metrics_from_opts, MetricError, Metrics}; + // header statics must be lower case, numbers and symbols per the RFC spec. This reduces chance of error. pub static X_LAST_MODIFIED: &str = "x-last-modified"; pub static X_WEAVE_TIMESTAMP: &str = "x-weave-timestamp"; @@ -56,3 +69,43 @@ pub trait InternalError { /// Constructs an internal error with the given error message. fn internal_error(message: String) -> Self; } + +/// A threadpool on which callers can spawn non-CPU-bound tasks that block their thread (this is +/// mostly useful for running I/O tasks). `BlockingThreadpool` intentionally does not implement +/// `Clone`: `Arc`s are not used internally, so a `BlockingThreadpool` should be instantiated once +/// and shared by passing around `Arc`s. +#[derive(Debug, Default)] +pub struct BlockingThreadpool { + spawned_tasks: AtomicU64, +} + +impl BlockingThreadpool { + /// Runs a function as a task on the blocking threadpool. + /// + /// WARNING: Spawning a blocking task through means other than calling this method will + /// result in inaccurate threadpool metrics being reported. If you want to spawn a task on + /// the blocking threadpool, you **must** use this function. + pub async fn spawn(&self, f: F) -> Result + where + F: FnOnce() -> Result + Send + 'static, + T: Send + 'static, + E: fmt::Debug + Send + InternalError + 'static, + { + self.spawned_tasks.fetch_add(1, Ordering::Relaxed); + + let result = web::block(f).await.map_err(|e| match e { + BlockingError::Error(e) => e, + BlockingError::Canceled => { + E::internal_error("Blocking threadpool operation canceled".to_owned()) + } + }); + + self.spawned_tasks.fetch_sub(1, Ordering::Relaxed); + + result + } + + pub fn active_threads(&self) -> u64 { + self.spawned_tasks.load(Ordering::Relaxed) + } +} diff --git a/syncserver/src/server/metrics.rs b/syncserver-common/src/metrics.rs similarity index 80% rename from syncserver/src/server/metrics.rs rename to syncserver-common/src/metrics.rs index 4b29f555..bee5bb78 100644 --- a/syncserver/src/server/metrics.rs +++ b/syncserver-common/src/metrics.rs @@ -2,18 +2,12 @@ use std::collections::HashMap; use std::net::UdpSocket; use std::time::Instant; -use actix_web::{dev::Payload, web::Data, FromRequest, HttpRequest}; use cadence::{ BufferedUdpMetricSink, Counted, Metric, NopMetricSink, QueuingMetricSink, StatsdClient, Timed, }; -use futures::future; -use futures::future::Ready; use slog::{Key, Record, KV}; -use crate::error::ApiError; -use crate::server::ServerState; -use crate::tokenserver; -use crate::web::tags::Taggable; +pub use cadence::MetricError; #[derive(Debug, Clone)] pub struct MetricTimer { @@ -58,55 +52,6 @@ impl Drop for Metrics { } } -impl FromRequest for Metrics { - type Config = (); - type Error = (); - type Future = Ready>; - - fn from_request(req: &HttpRequest, _: &mut Payload) -> Self::Future { - let client = { - let syncstorage_metrics = req - .app_data::>() - .map(|state| state.metrics.clone()); - let tokenserver_metrics = req - .app_data::>() - .map(|state| state.metrics.clone()); - - syncstorage_metrics.or(tokenserver_metrics) - }; - - if client.is_none() { - warn!("⚠️ metric error: No App State"); - } - - future::ok(Metrics { - client: client.as_deref().cloned(), - tags: req.get_tags(), - timer: None, - }) - } -} - -impl From<&StatsdClient> for Metrics { - fn from(client: &StatsdClient) -> Self { - Metrics { - client: Some(client.clone()), - tags: HashMap::default(), - timer: None, - } - } -} - -impl From<&ServerState> for Metrics { - fn from(state: &ServerState) -> Self { - Metrics { - client: Some(*state.metrics.clone()), - tags: HashMap::default(), - timer: None, - } - } -} - impl Metrics { pub fn sink() -> StatsdClient { StatsdClient::builder("", NopMetricSink).build() @@ -191,7 +136,7 @@ pub fn metrics_from_opts( label: &str, host: Option<&str>, port: u16, -) -> Result { +) -> Result { let builder = if let Some(statsd_host) = host { let socket = UdpSocket::bind("0.0.0.0:0")?; socket.set_nonblocking(true)?; @@ -210,6 +155,16 @@ pub fn metrics_from_opts( .build()) } +impl From<&StatsdClient> for Metrics { + fn from(client: &StatsdClient) -> Self { + Metrics { + client: Some(client.clone()), + tags: HashMap::default(), + timer: None, + } + } +} + /// A newtype used solely to allow us to implement KV on HashMap. struct MetricTags(HashMap); diff --git a/syncserver-db-common/Cargo.toml b/syncserver-db-common/Cargo.toml index 23775087..c5522508 100644 --- a/syncserver-db-common/Cargo.toml +++ b/syncserver-db-common/Cargo.toml @@ -4,25 +4,14 @@ version = "0.13.6" edition = "2021" [dependencies] -async-trait = "0.1.40" backtrace = "0.3.61" -chrono = "0.4" # Pin to 0.5 for now, to keep it under tokio 0.2 (issue977). # Fix for #803 (deadpool#92) points to our fork for now #deadpool = "0.5" # pin to 0.5 deadpool = { git = "https://github.com/mozilla-services/deadpool", branch = "deadpool-v0.5.2-issue92" } diesel = { version = "1.4", features = ["mysql", "r2d2"] } diesel_migrations = { version = "1.4.0", features = ["mysql"] } -# Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause -# syncstorage to either fail to either compile, or start. In those cases, try -# `cargo build --features grpcio/openssl ...` -grpcio = { version = "0.9" } -hostname = "0.3.1" -http = "0.2.6" futures = { version = "0.3", features = ["compat"] } -lazy_static = "1.4.0" -serde = "1.0" -serde_json = { version = "1.0", features = ["arbitrary_precision"] } +http = "0.2.6" syncserver-common = { path = "../syncserver-common" } thiserror = "1.0.26" -url = "2.1" diff --git a/syncserver-db-common/src/error.rs b/syncserver-db-common/src/error.rs index 1a2d7a66..e49e89fd 100644 --- a/syncserver-db-common/src/error.rs +++ b/syncserver-db-common/src/error.rs @@ -2,153 +2,58 @@ use std::fmt; use backtrace::Backtrace; use http::StatusCode; -use syncserver_common::{from_error, impl_fmt_display, InternalError}; +use syncserver_common::{from_error, impl_fmt_display}; use thiserror::Error; +/// Error specific to any MySQL database backend. These errors are not related to the syncstorage +/// or tokenserver application logic; rather, they are lower-level errors arising from diesel. #[derive(Debug)] -pub struct DbError { - kind: DbErrorKind, +pub struct MysqlError { + kind: MysqlErrorKind, pub status: StatusCode, pub backtrace: Backtrace, } #[derive(Debug, Error)] -pub enum DbErrorKind { +enum MysqlErrorKind { #[error("A database error occurred: {}", _0)] DieselQuery(#[from] diesel::result::Error), #[error("An error occurred while establishing a db connection: {}", _0)] DieselConnection(#[from] diesel::result::ConnectionError), - #[error("A database error occurred: {}", _0)] - SpannerGrpc(#[from] grpcio::Error), - - #[error("Spanner data load too large: {}", _0)] - SpannerTooLarge(String), - #[error("A database pool error occurred: {}", _0)] Pool(diesel::r2d2::PoolError), #[error("Error migrating the database: {}", _0)] Migration(diesel_migrations::RunMigrationsError), - - #[error("Specified collection does not exist")] - CollectionNotFound, - - #[error("Specified bso does not exist")] - BsoNotFound, - - #[error("Specified batch does not exist")] - BatchNotFound, - - #[error("An attempt at a conflicting write")] - Conflict, - - #[error("Database integrity error: {}", _0)] - Integrity(String), - - #[error("Invalid database URL: {}", _0)] - InvalidUrl(String), - - #[error("Unexpected error: {}", _0)] - Internal(String), - - #[error("User over quota")] - Quota, - - #[error("Connection expired")] - Expired, } -impl DbError { - pub fn internal(msg: &str) -> Self { - DbErrorKind::Internal(msg.to_owned()).into() - } - - pub fn is_sentry_event(&self) -> bool { - !matches!(&self.kind, DbErrorKind::Conflict) - } - - pub fn metric_label(&self) -> Option { - match &self.kind { - DbErrorKind::Conflict => Some("storage.conflict".to_owned()), - _ => None, - } - } - - pub fn is_collection_not_found(&self) -> bool { - matches!(self.kind, DbErrorKind::CollectionNotFound) - } - - pub fn is_conflict(&self) -> bool { - matches!(self.kind, DbErrorKind::Conflict) - } - - pub fn is_quota(&self) -> bool { - matches!(self.kind, DbErrorKind::Quota) - } - - pub fn is_bso_not_found(&self) -> bool { - matches!(self.kind, DbErrorKind::BsoNotFound) - } - - pub fn is_batch_not_found(&self) -> bool { - matches!(self.kind, DbErrorKind::BatchNotFound) - } -} - -impl From for DbError { - fn from(kind: DbErrorKind) -> Self { - let status = match kind { - DbErrorKind::CollectionNotFound | DbErrorKind::BsoNotFound => StatusCode::NOT_FOUND, - // Matching the Python code here (a 400 vs 404) - DbErrorKind::BatchNotFound | DbErrorKind::SpannerTooLarge(_) => StatusCode::BAD_REQUEST, - // NOTE: the protocol specification states that we should return a - // "409 Conflict" response here, but clients currently do not - // handle these respones very well: - // * desktop bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959034 - // * android bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959032 - DbErrorKind::Conflict => StatusCode::SERVICE_UNAVAILABLE, - DbErrorKind::Quota => StatusCode::FORBIDDEN, - _ => StatusCode::INTERNAL_SERVER_ERROR, - }; - +impl From for MysqlError { + fn from(kind: MysqlErrorKind) -> Self { Self { kind, - status, + status: StatusCode::INTERNAL_SERVER_ERROR, backtrace: Backtrace::new(), } } } -impl_fmt_display!(DbError, DbErrorKind); +impl_fmt_display!(MysqlError, MysqlErrorKind); -from_error!(diesel::result::Error, DbError, DbErrorKind::DieselQuery); +from_error!( + diesel::result::Error, + MysqlError, + MysqlErrorKind::DieselQuery +); from_error!( diesel::result::ConnectionError, - DbError, - DbErrorKind::DieselConnection + MysqlError, + MysqlErrorKind::DieselConnection ); -from_error!(grpcio::Error, DbError, |inner: grpcio::Error| { - // Convert ABORTED (typically due to a transaction abort) into 503s - match inner { - grpcio::Error::RpcFailure(ref status) | grpcio::Error::RpcFinished(Some(ref status)) - if status.code() == grpcio::RpcStatusCode::ABORTED => - { - DbErrorKind::Conflict - } - _ => DbErrorKind::SpannerGrpc(inner), - } -}); -from_error!(diesel::r2d2::PoolError, DbError, DbErrorKind::Pool); +from_error!(diesel::r2d2::PoolError, MysqlError, MysqlErrorKind::Pool); from_error!( diesel_migrations::RunMigrationsError, - DbError, - DbErrorKind::Migration + MysqlError, + MysqlErrorKind::Migration ); - -impl InternalError for DbError { - fn internal_error(message: String) -> Self { - DbErrorKind::Internal(message).into() - } -} diff --git a/syncserver-db-common/src/lib.rs b/syncserver-db-common/src/lib.rs index a850777d..5e227376 100644 --- a/syncserver-db-common/src/lib.rs +++ b/syncserver-db-common/src/lib.rs @@ -1,78 +1,18 @@ pub mod error; -pub mod params; -pub mod results; -pub mod util; +pub mod test; use std::fmt::Debug; -use async_trait::async_trait; -use futures::future::{self, LocalBoxFuture, TryFutureExt}; -use lazy_static::lazy_static; -use serde::Deserialize; +use futures::future::LocalBoxFuture; -use error::DbError; -use util::SyncTimestamp; - -lazy_static! { - /// For efficiency, it's possible to use fixed pre-determined IDs for - /// common collection names. This is the canonical list of such - /// names. Non-standard collections will be allocated IDs starting - /// from the highest ID in this collection. - pub static ref STD_COLLS: Vec<(i32, &'static str)> = { - vec![ - (1, "clients"), - (2, "crypto"), - (3, "forms"), - (4, "history"), - (5, "keys"), - (6, "meta"), - (7, "bookmarks"), - (8, "prefs"), - (9, "tabs"), - (10, "passwords"), - (11, "addons"), - (12, "addresses"), - (13, "creditcards"), - ] - }; -} - -/// Rough guesstimate of the maximum reasonable life span of a batch -pub const BATCH_LIFETIME: i64 = 2 * 60 * 60 * 1000; // 2 hours, in milliseconds - -/// The ttl to use for rows that are never supposed to expire (in seconds) -pub const DEFAULT_BSO_TTL: u32 = 2_100_000_000; - -/// Non-standard collections will be allocated IDs beginning with this value -pub const FIRST_CUSTOM_COLLECTION_ID: i32 = 101; - -pub type DbFuture<'a, T> = LocalBoxFuture<'a, Result>; - -#[async_trait] -pub trait DbPool: Sync + Send + Debug + GetPoolState { - async fn get(&self) -> Result>, DbError>; - - fn validate_batch_id(&self, params: params::ValidateBatchId) -> Result<(), DbError>; - - fn box_clone(&self) -> Box; -} - -impl Clone for Box { - fn clone(&self) -> Box { - self.box_clone() - } -} +pub type DbFuture<'a, T, E> = LocalBoxFuture<'a, Result>; +/// A trait to be implemented by database pool data structures. It provides an interface to +/// derive the current state of the pool, as represented by the `PoolState` struct. pub trait GetPoolState { fn state(&self) -> PoolState; } -impl GetPoolState for Box { - fn state(&self) -> PoolState { - (**self).state() - } -} - #[derive(Debug, Default)] /// A mockable r2d2::State pub struct PoolState { @@ -97,212 +37,18 @@ impl From for PoolState { } } -pub trait Db<'a>: Debug + 'a { - fn lock_for_read(&self, params: params::LockCollection) -> DbFuture<'_, ()>; - - fn lock_for_write(&self, params: params::LockCollection) -> DbFuture<'_, ()>; - - fn begin(&self, for_write: bool) -> DbFuture<'_, ()>; - - fn commit(&self) -> DbFuture<'_, ()>; - - fn rollback(&self) -> DbFuture<'_, ()>; - - fn get_collection_timestamps( - &self, - params: params::GetCollectionTimestamps, - ) -> DbFuture<'_, results::GetCollectionTimestamps>; - - fn get_collection_timestamp( - &self, - params: params::GetCollectionTimestamp, - ) -> DbFuture<'_, results::GetCollectionTimestamp>; - - fn get_collection_counts( - &self, - params: params::GetCollectionCounts, - ) -> DbFuture<'_, results::GetCollectionCounts>; - - fn get_collection_usage( - &self, - params: params::GetCollectionUsage, - ) -> DbFuture<'_, results::GetCollectionUsage>; - - fn get_storage_timestamp( - &self, - params: params::GetStorageTimestamp, - ) -> DbFuture<'_, results::GetStorageTimestamp>; - - fn get_storage_usage( - &self, - params: params::GetStorageUsage, - ) -> DbFuture<'_, results::GetStorageUsage>; - - fn get_quota_usage( - &self, - params: params::GetQuotaUsage, - ) -> DbFuture<'_, results::GetQuotaUsage>; - - fn delete_storage(&self, params: params::DeleteStorage) - -> DbFuture<'_, results::DeleteStorage>; - - fn delete_collection( - &self, - params: params::DeleteCollection, - ) -> DbFuture<'_, results::DeleteCollection>; - - fn delete_bsos(&self, params: params::DeleteBsos) -> DbFuture<'_, results::DeleteBsos>; - - fn get_bsos(&self, params: params::GetBsos) -> DbFuture<'_, results::GetBsos>; - - fn get_bso_ids(&self, params: params::GetBsos) -> DbFuture<'_, results::GetBsoIds>; - - fn post_bsos(&self, params: params::PostBsos) -> DbFuture<'_, results::PostBsos>; - - fn delete_bso(&self, params: params::DeleteBso) -> DbFuture<'_, results::DeleteBso>; - - fn get_bso(&self, params: params::GetBso) -> DbFuture<'_, Option>; - - fn get_bso_timestamp( - &self, - params: params::GetBsoTimestamp, - ) -> DbFuture<'_, results::GetBsoTimestamp>; - - fn put_bso(&self, params: params::PutBso) -> DbFuture<'_, results::PutBso>; - - fn create_batch(&self, params: params::CreateBatch) -> DbFuture<'_, results::CreateBatch>; - - fn validate_batch(&self, params: params::ValidateBatch) - -> DbFuture<'_, results::ValidateBatch>; - - fn append_to_batch( - &self, - params: params::AppendToBatch, - ) -> DbFuture<'_, results::AppendToBatch>; - - fn get_batch(&self, params: params::GetBatch) -> DbFuture<'_, Option>; - - fn commit_batch(&self, params: params::CommitBatch) -> DbFuture<'_, results::CommitBatch>; - - fn box_clone(&self) -> Box>; - - fn check(&self) -> DbFuture<'_, results::Check>; - - fn get_connection_info(&self) -> results::ConnectionInfo; - - /// Retrieve the timestamp for an item/collection - /// - /// Modeled on the Python `get_resource_timestamp` function. - fn extract_resource( - &self, - user_id: UserIdentifier, - collection: Option, - bso: Option, - ) -> DbFuture<'_, SyncTimestamp> { - // If there's no collection, we return the overall storage timestamp - let collection = match collection { - Some(collection) => collection, - None => return Box::pin(self.get_storage_timestamp(user_id)), - }; - // If there's no bso, return the collection - let bso = match bso { - Some(bso) => bso, - None => { - return Box::pin( - self.get_collection_timestamp(params::GetCollectionTimestamp { - user_id, - collection, - }) - .or_else(|e| { - if e.is_collection_not_found() { - future::ok(SyncTimestamp::from_seconds(0f64)) - } else { - future::err(e) - } - }), - ) - } - }; - Box::pin( - self.get_bso_timestamp(params::GetBsoTimestamp { - user_id, - collection, - id: bso, - }) - .or_else(|e| { - if e.is_collection_not_found() { - future::ok(SyncTimestamp::from_seconds(0f64)) - } else { - future::err(e) - } - }), - ) - } - - /// Internal methods used by the db tests - - fn get_collection_id(&self, name: String) -> DbFuture<'_, i32>; - - fn create_collection(&self, name: String) -> DbFuture<'_, i32>; - - fn update_collection(&self, params: params::UpdateCollection) -> DbFuture<'_, SyncTimestamp>; - - fn timestamp(&self) -> SyncTimestamp; - - fn set_timestamp(&self, timestamp: SyncTimestamp); - - fn delete_batch(&self, params: params::DeleteBatch) -> DbFuture<'_, ()>; - - fn clear_coll_cache(&self) -> DbFuture<'_, ()>; - - fn set_quota(&mut self, enabled: bool, limit: usize, enforce: bool); -} - -impl<'a> Clone for Box> { - fn clone(&self) -> Box> { - self.box_clone() - } -} - -#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Copy)] -#[serde(rename_all = "lowercase")] -pub enum Sorting { - None, - Newest, - Oldest, - Index, -} - -impl Default for Sorting { - fn default() -> Self { - Sorting::None - } -} - -#[derive(Clone, Debug, Default, Eq, Hash, PartialEq)] -pub struct UserIdentifier { - /// For MySQL database backends as the primary key - pub legacy_id: u64, - /// For NoSQL database backends that require randomly distributed primary keys - pub fxa_uid: String, - pub fxa_kid: String, -} - -impl UserIdentifier { - /// Create a new legacy id user identifier - pub fn new_legacy(user_id: u64) -> Self { - Self { - legacy_id: user_id, - ..Default::default() +#[macro_export] +macro_rules! sync_db_method { + ($name:ident, $sync_name:ident, $type:ident) => { + sync_db_method!($name, $sync_name, $type, results::$type); + }; + ($name:ident, $sync_name:ident, $type:ident, $result:ty) => { + fn $name(&self, params: params::$type) -> DbFuture<'_, $result, DbError> { + let db = self.clone(); + Box::pin( + self.blocking_threadpool + .spawn(move || db.$sync_name(params)), + ) } - } -} - -impl From for UserIdentifier { - fn from(val: u32) -> Self { - Self { - legacy_id: val.into(), - ..Default::default() - } - } + }; } diff --git a/syncserver-db-common/src/test.rs b/syncserver-db-common/src/test.rs new file mode 100644 index 00000000..351888f3 --- /dev/null +++ b/syncserver-db-common/src/test.rs @@ -0,0 +1,14 @@ +use diesel::{ + mysql::MysqlConnection, + r2d2::{CustomizeConnection, Error as PoolError}, + Connection, +}; + +#[derive(Debug)] +pub struct TestTransactionCustomizer; + +impl CustomizeConnection for TestTransactionCustomizer { + fn on_acquire(&self, conn: &mut MysqlConnection) -> Result<(), PoolError> { + conn.begin_test_transaction().map_err(PoolError::QueryError) + } +} diff --git a/syncserver-settings/src/lib.rs b/syncserver-settings/src/lib.rs index 9eb93331..f4fa887f 100644 --- a/syncserver-settings/src/lib.rs +++ b/syncserver-settings/src/lib.rs @@ -13,9 +13,9 @@ use syncstorage_settings::Settings as SyncstorageSettings; use tokenserver_settings::Settings as TokenserverSettings; use url::Url; -pub static PREFIX: &str = "sync"; +static PREFIX: &str = "sync"; -#[derive(Clone, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(default)] pub struct Settings { pub port: u16, @@ -134,6 +134,7 @@ impl Settings { } } + #[cfg(debug_assertions)] pub fn test_settings() -> Self { let mut settings = Self::with_env_and_config_file(None).expect("Could not get Settings in test_settings"); diff --git a/syncserver/Cargo.toml b/syncserver/Cargo.toml index 916969e4..1b13eb3d 100644 --- a/syncserver/Cargo.toml +++ b/syncserver/Cargo.toml @@ -13,47 +13,23 @@ default-run = "syncserver" [dependencies] actix-http = "2" actix-web = "3" -actix-rt = "1" # Pin to 1.0, due to dependencies on Tokio +actix-rt = "1" # Pin to 1.0, due to dependencies on Tokio actix-cors = "0.5" -actix-service = "1.0.6" async-trait = "0.1.40" backtrace = "0.3.61" base64 = "0.13" -bb8 = "0.4.1" # pin to 0.4 due to dependencies on Tokio -bytes = "1.0" cadence = "0.26" chrono = "0.4" -# Pin to 0.5 for now, to keep it under tokio 0.2 (issue977). -# Fix for #803 (deadpool#92) points to our fork for now -#deadpool = "0.5" # pin to 0.5 -deadpool = { git = "https://github.com/mozilla-services/deadpool", branch = "deadpool-v0.5.2-issue92" } -diesel = { version = "1.4", features = ["mysql", "r2d2"] } -diesel_logger = "0.1.1" -diesel_migrations = { version = "1.4.0", features = ["mysql"] } docopt = "1.1.0" dyn-clone = "1.0.4" env_logger = "0.9" futures = { version = "0.3", features = ["compat"] } -google-cloud-rust-raw = "0.11.0" -# Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause -# syncserver to either fail to either compile, or start. In those cases, try -# `cargo build --features grpcio/openssl ...` -grpcio = { version = "0.9" } +hostname = "0.3.1" lazy_static = "1.4.0" hawk = "3.2" hex = "0.4.3" -hostname = "0.3.1" hmac = "0.11" -http = "0.2.5" -log = { version = "0.4", features = [ - "max_level_debug", - "release_max_level_info", -] } mime = "0.3" -num_cpus = "1" -# must match what's used by googleapis-raw -protobuf = "2.20.0" -pyo3 = { version = "0.14", features = ["auto-initialize"] } rand = "0.8" regex = "1.4" reqwest = { version = "0.10.10", features = ["json", "rustls-tls"] } @@ -65,7 +41,6 @@ sentry-backtrace = "0.19" serde = "1.0" serde_derive = "1.0" serde_json = { version = "1.0", features = ["arbitrary_precision"] } -scheduled-thread-pool = "0.2" sha2 = "0.9" slog = { version = "2.5", features = [ "max_level_info", @@ -78,28 +53,24 @@ slog-mozlog-json = "0.1" slog-scope = "4.3" slog-stdlog = "4.1" slog-term = "2.6" -syncserver-settings = { path = "../syncserver-settings" } -syncserver-db-common = { path = "../syncserver-db-common" } syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } +syncserver-settings = { path = "../syncserver-settings" } +syncstorage-db = { path = "../syncstorage-db" } syncstorage-settings = { path = "../syncstorage-settings" } time = "^0.3" thiserror = "1.0.26" +tokenserver-auth = { path = "../tokenserver-auth" } tokenserver-common = { path = "../tokenserver-common" } +tokenserver-db = { path = "../tokenserver-db" } tokenserver-settings = { path = "../tokenserver-settings" } # pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.) tokio = { version = "0.2.4", features = ["macros", "sync"] } -url = "2.1" urlencoding = "2.1" -uuid = { version = "0.8.2", features = ["serde", "v4"] } validator = "0.14" validator_derive = "0.14" woothee = "0.11" -[dev-dependencies] -mockito = "0.30.0" - [features] +default = ["syncstorage-db/mysql"] no_auth = [] - -[[bin]] -name = "purge_ttl" diff --git a/syncserver/src/db/mysql/mod.rs b/syncserver/src/db/mysql/mod.rs deleted file mode 100644 index 82ea3a9b..00000000 --- a/syncserver/src/db/mysql/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -#[macro_use] -mod batch; -mod diesel_ext; -pub mod models; -pub mod pool; -mod schema; -#[cfg(test)] -mod test; - -pub use self::pool::MysqlDbPool; -#[cfg(test)] -pub use self::test::TestTransactionCustomizer; diff --git a/syncserver/src/db/spanner/manager/mod.rs b/syncserver/src/db/spanner/manager/mod.rs deleted file mode 100644 index b1ee0932..00000000 --- a/syncserver/src/db/spanner/manager/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -// mod bb8; -mod deadpool; -mod session; - -pub use self::deadpool::{Conn, SpannerSessionManager}; -pub use self::session::SpannerSession; diff --git a/syncserver/src/db/spanner/mod.rs b/syncserver/src/db/spanner/mod.rs deleted file mode 100644 index a5962eb3..00000000 --- a/syncserver/src/db/spanner/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -use std::time::SystemTime; - -#[macro_use] -mod macros; - -mod batch; -pub mod manager; -pub mod models; -pub mod pool; -mod support; - -pub use self::pool::SpannerDbPool; - -pub fn now() -> i64 { - SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as i64 -} diff --git a/syncserver/src/error.rs b/syncserver/src/error.rs index c81ea604..933b4de0 100644 --- a/syncserver/src/error.rs +++ b/syncserver/src/error.rs @@ -21,8 +21,9 @@ use serde::{ Serialize, }; -use syncserver_common::{from_error, impl_fmt_display, ReportableError}; -use syncserver_db_common::error::DbError; +use syncserver_common::{from_error, impl_fmt_display, MetricError, ReportableError}; +use syncstorage_db::{DbError, DbErrorIntrospect}; + use thiserror::Error; use crate::web::error::{HawkError, ValidationError}; @@ -57,7 +58,7 @@ pub const RETRY_AFTER: u8 = 10; #[derive(Debug)] pub struct ApiError { kind: ApiErrorKind, - pub(crate) backtrace: Backtrace, + pub(crate) backtrace: Box, status: StatusCode, } @@ -87,8 +88,8 @@ pub enum ApiErrorKind { impl ApiErrorKind { pub fn metric_label(&self) -> Option { match self { - ApiErrorKind::Db(err) => err.metric_label(), ApiErrorKind::Hawk(err) => err.metric_label(), + ApiErrorKind::Db(err) => err.metric_label(), ApiErrorKind::Validation(err) => err.metric_label(), _ => None, } @@ -96,6 +97,15 @@ impl ApiErrorKind { } impl ApiError { + pub fn is_sentry_event(&self) -> bool { + // Should we report this error to sentry? + self.status.is_server_error() + && match &self.kind { + ApiErrorKind::Db(dbe) => dbe.is_sentry_event(), + _ => self.kind.metric_label().is_none(), + } + } + fn weave_error_code(&self) -> WeaveError { match &self.kind { ApiErrorKind::Validation(ver) => ver.weave_error_code(), @@ -148,8 +158,8 @@ impl From for HttpResponse { } } -impl From for ApiError { - fn from(inner: cadence::MetricError) -> Self { +impl From for ApiError { + fn from(inner: MetricError) -> Self { ApiErrorKind::Internal(inner.to_string()).into() } } @@ -173,7 +183,7 @@ impl From for ApiError { Self { kind, - backtrace: Backtrace::new(), + backtrace: Box::new(Backtrace::new()), status, } } diff --git a/syncserver/src/lib.rs b/syncserver/src/lib.rs index 90284b15..ac5b979a 100644 --- a/syncserver/src/lib.rs +++ b/syncserver/src/lib.rs @@ -1,10 +1,6 @@ #![warn(rust_2018_idioms)] #![allow(clippy::try_err)] -#[macro_use] -extern crate diesel; -#[macro_use] -extern crate diesel_migrations; #[macro_use] extern crate slog_scope; #[macro_use] @@ -12,7 +8,6 @@ extern crate validator_derive; #[macro_use] pub mod error; -pub mod db; pub mod logging; pub mod server; pub mod tokenserver; diff --git a/syncserver/src/server/mod.rs b/syncserver/src/server/mod.rs index 7a0d80fd..fc377224 100644 --- a/syncserver/src/server/mod.rs +++ b/syncserver/src/server/mod.rs @@ -1,33 +1,27 @@ //! Main application server -use std::{ - env, fmt, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - time::Duration, -}; +use std::{env, sync::Arc, time::Duration}; use actix_cors::Cors; use actix_web::{ - dev, - error::BlockingError, + dev::{self, Payload}, http::StatusCode, http::{header::LOCATION, Method}, middleware::errhandlers::ErrorHandlers, - web, App, HttpRequest, HttpResponse, HttpServer, + web::{self, Data}, + App, FromRequest, HttpRequest, HttpResponse, HttpServer, }; use cadence::{Gauged, StatsdClient}; -use syncserver_common::InternalError; -use syncserver_db_common::{error::DbError, DbPool, GetPoolState, PoolState}; +use futures::future::{self, Ready}; +use syncserver_common::{BlockingThreadpool, Metrics}; +use syncserver_db_common::{GetPoolState, PoolState}; use syncserver_settings::Settings; +use syncstorage_db::{DbError, DbPool, DbPoolImpl}; use syncstorage_settings::{Deadman, ServerLimits}; use tokio::{sync::RwLock, time}; -use crate::db::pool_from_settings; use crate::error::ApiError; -use crate::server::metrics::Metrics; +use crate::server::tags::Taggable; use crate::tokenserver; use crate::web::{handlers, middleware}; @@ -38,7 +32,7 @@ pub const SYNC_DOCS_URL: &str = const MYSQL_UID_REGEX: &str = r"[0-9]{1,10}"; const SYNC_VERSION_PATH: &str = "1.5"; -pub mod metrics; +pub mod tags; #[cfg(test)] mod test; pub mod user_agent; @@ -46,7 +40,7 @@ pub mod user_agent; /// This is the global HTTP state object that will be made available to all /// HTTP API calls. pub struct ServerState { - pub db_pool: Box, + pub db_pool: Box>, /// Server-enforced limits for request payloads. pub limits: Arc, @@ -93,10 +87,10 @@ macro_rules! build_app { // These will wrap all outbound responses with matching status codes. .wrap(ErrorHandlers::new().handler(StatusCode::NOT_FOUND, ApiError::render_404)) // These are our wrappers - .wrap(middleware::weave::WeaveTimestamp::new()) - .wrap(tokenserver::logging::LoggingWrapper::new()) - .wrap(middleware::sentry::SentryWrapper::default()) - .wrap(middleware::rejectua::RejectUA::default()) + .wrap_fn(middleware::weave::set_weave_timestamp) + .wrap_fn(tokenserver::logging::handle_request_log_line) + .wrap_fn(middleware::sentry::report_error) + .wrap_fn(middleware::rejectua::reject_user_agent) .wrap($cors) .wrap_fn(middleware::emit_http_status_with_tokenserver_origin) .service( @@ -198,9 +192,9 @@ macro_rules! build_app_without_syncstorage { // These will wrap all outbound responses with matching status codes. .wrap(ErrorHandlers::new().handler(StatusCode::NOT_FOUND, ApiError::render_404)) // These are our wrappers - .wrap(middleware::sentry::SentryWrapper::default()) - .wrap(tokenserver::logging::LoggingWrapper::new()) - .wrap(middleware::rejectua::RejectUA::default()) + .wrap_fn(middleware::sentry::report_error) + .wrap_fn(tokenserver::logging::handle_request_log_line) + .wrap_fn(middleware::rejectua::reject_user_agent) // Followed by the "official middleware" so they run first. // actix is getting increasingly tighter about CORS headers. Our server is // not a huge risk but does deliver XHR JSON content. @@ -249,7 +243,7 @@ macro_rules! build_app_without_syncstorage { impl Server { pub async fn with_settings(settings: Settings) -> Result { let settings_copy = settings.clone(); - let metrics = metrics::metrics_from_opts( + let metrics = syncserver_common::metrics_from_opts( &settings.syncstorage.statsd_label, settings.statsd_host.as_deref(), settings.statsd_port, @@ -258,12 +252,11 @@ impl Server { let port = settings.port; let deadman = Arc::new(RwLock::new(Deadman::from(&settings.syncstorage))); let blocking_threadpool = Arc::new(BlockingThreadpool::default()); - let db_pool = pool_from_settings( + let db_pool = DbPoolImpl::new( &settings.syncstorage, &Metrics::from(&metrics), blocking_threadpool.clone(), - ) - .await?; + )?; let limits = Arc::new(settings.syncstorage.limits); let limits_json = serde_json::to_string(&*limits).expect("ServerLimits failed to serialize"); @@ -273,12 +266,12 @@ impl Server { let tokenserver_state = if settings.tokenserver.enabled { let state = tokenserver::ServerState::from_settings( &settings.tokenserver, - metrics::metrics_from_opts( + syncserver_common::metrics_from_opts( &settings.tokenserver.statsd_label, settings.statsd_host.as_deref(), settings.statsd_port, )?, - blocking_threadpool.clone(), + blocking_threadpool, )?; Some(state) @@ -290,7 +283,7 @@ impl Server { Duration::from_secs(10), metrics.clone(), db_pool.clone(), - blocking_threadpool.clone(), + blocking_threadpool, )?; None @@ -298,7 +291,7 @@ impl Server { let mut server = HttpServer::new(move || { let syncstorage_state = ServerState { - db_pool: db_pool.clone(), + db_pool: Box::new(db_pool.clone()), limits: Arc::clone(&limits), limits_json: limits_json.clone(), metrics: Box::new(metrics.clone()), @@ -337,7 +330,7 @@ impl Server { let blocking_threadpool = Arc::new(BlockingThreadpool::default()); let tokenserver_state = tokenserver::ServerState::from_settings( &settings.tokenserver, - metrics::metrics_from_opts( + syncserver_common::metrics_from_opts( &settings.tokenserver.statsd_label, settings.statsd_host.as_deref(), settings.statsd_port, @@ -405,6 +398,37 @@ fn build_cors(settings: &Settings) -> Cors { cors } +pub struct MetricsWrapper(pub Metrics); + +impl FromRequest for MetricsWrapper { + type Config = (); + type Error = (); + type Future = Ready>; + + fn from_request(req: &HttpRequest, _: &mut Payload) -> Self::Future { + let client = { + let syncstorage_metrics = req + .app_data::>() + .map(|state| state.metrics.clone()); + let tokenserver_metrics = req + .app_data::>() + .map(|state| state.metrics.clone()); + + syncstorage_metrics.or(tokenserver_metrics) + }; + + if client.is_none() { + warn!("⚠️ metric error: No App State"); + } + + future::ok(MetricsWrapper(Metrics { + client: client.as_deref().cloned(), + tags: req.get_tags(), + timer: None, + })) + } +} + /// Emit database pool and threadpool metrics periodically fn spawn_metric_periodic_reporter( interval: Duration, @@ -453,43 +477,3 @@ fn spawn_metric_periodic_reporter( Ok(()) } - -/// A threadpool on which callers can spawn non-CPU-bound tasks that block their thread (this is -/// mostly useful for running I/O tasks). `BlockingThreadpool` intentionally does not implement -/// `Clone`: `Arc`s are not used internally, so a `BlockingThreadpool` should be instantiated once -/// and shared by passing around `Arc`s. -#[derive(Debug, Default)] -pub struct BlockingThreadpool { - spawned_tasks: AtomicU64, -} - -impl BlockingThreadpool { - /// Runs a function as a task on the blocking threadpool. - /// - /// WARNING: Spawning a blocking task through means other than calling this method will - /// result in inaccurate threadpool metrics being reported. If you want to spawn a task on - /// the blocking threadpool, you **must** use this function. - pub async fn spawn(&self, f: F) -> Result - where - F: FnOnce() -> Result + Send + 'static, - T: Send + 'static, - E: fmt::Debug + Send + InternalError + 'static, - { - self.spawned_tasks.fetch_add(1, Ordering::Relaxed); - - let result = web::block(f).await.map_err(|e| match e { - BlockingError::Error(e) => e, - BlockingError::Canceled => { - E::internal_error("Blocking threadpool operation canceled".to_owned()) - } - }); - - self.spawned_tasks.fetch_sub(1, Ordering::Relaxed); - - result - } - - fn active_threads(&self) -> u64 { - self.spawned_tasks.load(Ordering::Relaxed) - } -} diff --git a/syncserver/src/web/tags.rs b/syncserver/src/server/tags.rs similarity index 89% rename from syncserver/src/web/tags.rs rename to syncserver/src/server/tags.rs index db6f6ab2..933644c9 100644 --- a/syncserver/src/web/tags.rs +++ b/syncserver/src/server/tags.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; use actix_web::HttpMessage; @@ -78,14 +78,3 @@ struct Tags(HashMap); // "Extras" are pieces of metadata with high cardinality to be included in Sentry errors. #[derive(Default)] struct Extras(HashMap); - -impl From for BTreeMap { - fn from(tags: Tags) -> BTreeMap { - let mut result = BTreeMap::new(); - - for (k, v) in tags.0 { - result.insert(k.clone(), v.clone()); - } - result - } -} diff --git a/syncserver/src/server/test.rs b/syncserver/src/server/test.rs index 6378e586..76bfd2a9 100644 --- a/syncserver/src/server/test.rs +++ b/syncserver/src/server/test.rs @@ -16,17 +16,16 @@ use serde::de::DeserializeOwned; use serde_json::json; use sha2::Sha256; use syncserver_common::{self, X_LAST_MODIFIED}; -use syncserver_db_common::{ +use syncserver_settings::{Secrets, Settings}; +use syncstorage_db::{ params, results::{DeleteBso, GetBso, PostBsos, PutBso}, - util::SyncTimestamp, + DbPoolImpl, SyncTimestamp, }; -use syncserver_settings::{Secrets, Settings}; use syncstorage_settings::ServerLimits; use super::*; use crate::build_app; -use crate::db::pool_from_settings; use crate::tokenserver; use crate::web::{auth::HawkPayload, extractors::BsoBody}; @@ -69,13 +68,14 @@ async fn get_test_state(settings: &Settings) -> ServerState { let blocking_threadpool = Arc::new(BlockingThreadpool::default()); ServerState { - db_pool: pool_from_settings( - &settings.syncstorage, - &Metrics::from(&metrics), - blocking_threadpool.clone(), - ) - .await - .expect("Could not get db_pool in get_test_state"), + db_pool: Box::new( + DbPoolImpl::new( + &settings.syncstorage, + &Metrics::from(&metrics), + blocking_threadpool, + ) + .expect("Could not get db_pool in get_test_state"), + ), limits: Arc::clone(&SERVER_LIMITS), limits_json: serde_json::to_string(&**SERVER_LIMITS).unwrap(), metrics: Box::new(metrics), diff --git a/syncserver/src/tokenserver/db/mod.rs b/syncserver/src/tokenserver/db/mod.rs deleted file mode 100644 index c068100c..00000000 --- a/syncserver/src/tokenserver/db/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod mock; -pub mod models; -pub mod params; -pub mod pool; -pub mod results; diff --git a/syncserver/src/tokenserver/extractors.rs b/syncserver/src/tokenserver/extractors.rs index 6118da19..5eb8cb0e 100644 --- a/syncserver/src/tokenserver/extractors.rs +++ b/syncserver/src/tokenserver/extractors.rs @@ -21,17 +21,11 @@ use regex::Regex; use serde::Deserialize; use sha2::Sha256; use syncserver_settings::Secrets; -use tokenserver_common::{ - error::{ErrorLocation, TokenserverError}, - NodeType, -}; +use tokenserver_common::{ErrorLocation, NodeType, TokenserverError}; +use tokenserver_db::{params, results, Db, DbPool}; -use super::{ - db::{models::Db, params, pool::DbPool, results}, - LogItemsMutator, ServerState, TokenserverMetrics, -}; -use crate::server::metrics::Metrics; -use crate::web::tags::Taggable; +use super::{LogItemsMutator, ServerState, TokenserverMetrics}; +use crate::server::{tags::Taggable, MetricsWrapper}; lazy_static! { static ref CLIENT_STATE_REGEX: Regex = Regex::new("^[a-zA-Z0-9._-]{1,32}$").unwrap(); @@ -218,7 +212,7 @@ impl FromRequest for TokenserverRequest { hash_device_id(&hashed_fxa_uid, device_id, fxa_metrics_hash_secret) }; - let db = >::extract(&req).await?; + let DbWrapper(db) = DbWrapper::extract(&req).await?; let service_id = { let path = req.match_info(); @@ -312,7 +306,10 @@ struct QueryParams { pub duration: Option, } -impl FromRequest for Box { +/// A local "newtype" that wraps `Box` so we can implement `FromRequest`. +pub struct DbWrapper(pub Box); + +impl FromRequest for DbWrapper { type Config = (); type Error = TokenserverError; type Future = LocalBoxFuture<'static, Result>; @@ -321,10 +318,12 @@ impl FromRequest for Box { let req = req.clone(); Box::pin(async move { - >::extract(&req) + DbPoolWrapper::extract(&req) .await? + .0 .get() .await + .map(Self) .map_err(|e| TokenserverError { context: format!("Couldn't acquire a database connection: {}", e), ..TokenserverError::internal_error() @@ -333,7 +332,9 @@ impl FromRequest for Box { } } -impl FromRequest for Box { +struct DbPoolWrapper(Box); + +impl FromRequest for DbPoolWrapper { type Config = (); type Error = TokenserverError; type Future = LocalBoxFuture<'static, Result>; @@ -344,7 +345,7 @@ impl FromRequest for Box { Box::pin(async move { let state = get_server_state(&req)?.as_ref(); - Ok(state.db_pool.clone()) + Ok(Self(state.db_pool.clone())) }) } } @@ -648,8 +649,12 @@ impl FromRequest for TokenserverMetrics { fn from_request(req: &HttpRequest, _payload: &mut Payload) -> Self::Future { let req = req.clone(); - // `Result::unwrap` is safe to use here, since Metrics::extract can never fail - Box::pin(async move { Ok(TokenserverMetrics(Metrics::extract(&req).await.unwrap())) }) + // `Result::unwrap` is safe to use here, since MetricsWrapper::extract can never fail + Box::pin(async move { + Ok(TokenserverMetrics( + MetricsWrapper::extract(&req).await.unwrap().0, + )) + }) } } @@ -706,14 +711,11 @@ mod tests { use serde_json; use syncserver_settings::Settings as GlobalSettings; use syncstorage_settings::ServerLimits; + use tokenserver_auth::{browserid, oauth, MockVerifier}; + use tokenserver_db::mock::MockDbPool as MockTokenserverPool; use tokenserver_settings::Settings as TokenserverSettings; - use crate::server::metrics; - use crate::tokenserver::{ - auth::{browserid, oauth, MockVerifier}, - db::mock::MockDbPool as MockTokenserverPool, - ServerState, - }; + use crate::tokenserver::ServerState; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; @@ -1339,7 +1341,7 @@ mod tests { node_capacity_release_rate: None, node_type: NodeType::default(), metrics: Box::new( - metrics::metrics_from_opts( + syncserver_common::metrics_from_opts( &tokenserver_settings.statsd_label, syncserver_settings.statsd_host.as_deref(), syncserver_settings.statsd_port, diff --git a/syncserver/src/tokenserver/handlers.rs b/syncserver/src/tokenserver/handlers.rs index 855efa51..7bea085f 100644 --- a/syncserver/src/tokenserver/handlers.rs +++ b/syncserver/src/tokenserver/handlers.rs @@ -6,15 +6,15 @@ use std::{ use actix_web::{http::StatusCode, Error, HttpResponse}; use serde::Serialize; use serde_json::Value; -use tokenserver_common::{error::TokenserverError, NodeType}; +use tokenserver_auth::{MakeTokenPlaintext, Tokenlib, TokenserverOrigin}; +use tokenserver_common::{NodeType, TokenserverError}; +use tokenserver_db::{ + params::{GetNodeId, PostUser, PutUser, ReplaceUsers}, + Db, +}; use super::{ - auth::{MakeTokenPlaintext, Tokenlib, TokenserverOrigin}, - db::{ - models::Db, - params::{GetNodeId, PostUser, PutUser, ReplaceUsers}, - }, - extractors::TokenserverRequest, + extractors::{DbWrapper, TokenserverRequest}, TokenserverMetrics, }; @@ -32,7 +32,7 @@ pub struct TokenserverResult { pub async fn get_tokenserver_result( req: TokenserverRequest, - db: Box, + DbWrapper(db): DbWrapper, TokenserverMetrics(mut metrics): TokenserverMetrics, ) -> Result { let updates = update_user(&req, db).await?; @@ -83,7 +83,7 @@ fn get_token_plaintext( context: format!("Failed to decode the client state hex: {}", e), ..TokenserverError::internal_error() })?; - let client_state_b64 = base64::encode_config(&client_state, base64::URL_SAFE_NO_PAD); + let client_state_b64 = base64::encode_config(client_state, base64::URL_SAFE_NO_PAD); format!( "{:013}-{:}", @@ -242,7 +242,7 @@ async fn update_user( } } -pub async fn heartbeat(db: Box) -> Result { +pub async fn heartbeat(DbWrapper(db): DbWrapper) -> Result { let mut checklist = HashMap::new(); checklist.insert( "version".to_owned(), diff --git a/syncserver/src/tokenserver/logging.rs b/syncserver/src/tokenserver/logging.rs index c7006c7d..577f0b55 100644 --- a/syncserver/src/tokenserver/logging.rs +++ b/syncserver/src/tokenserver/logging.rs @@ -1,74 +1,30 @@ -use std::task::Context; -use std::{cell::RefCell, rc::Rc}; - use actix_web::{ - dev::{Service, ServiceRequest, ServiceResponse, Transform}, - Error, HttpMessage, + dev::{Service, ServiceRequest, ServiceResponse}, + HttpMessage, }; -use futures::future::{self, LocalBoxFuture, TryFutureExt}; -use std::task::Poll; +use futures::future::Future; use super::LogItems; -#[derive(Default)] -pub struct LoggingWrapper; +pub fn handle_request_log_line( + request: ServiceRequest, + service: &mut impl Service< + Request = ServiceRequest, + Response = ServiceResponse, + Error = actix_web::Error, + >, +) -> impl Future> { + let items = LogItems::from(request.head()); + request.extensions_mut().insert(items); + let fut = service.call(request); -impl LoggingWrapper { - pub fn new() -> Self { - LoggingWrapper::default() - } -} - -impl Transform for LoggingWrapper -where - S: Service, Error = Error> + 'static, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = LoggingWrapperMiddleware; - type Future = LocalBoxFuture<'static, Result>; - - fn new_transform(&self, service: S) -> Self::Future { - Box::pin(future::ok(LoggingWrapperMiddleware { - service: Rc::new(RefCell::new(service)), - })) - } -} - -#[derive(Debug)] -pub struct LoggingWrapperMiddleware { - service: Rc>, -} - -impl Service for LoggingWrapperMiddleware -where - S: Service, Error = Error> + 'static, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = LocalBoxFuture<'static, Result>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, sreq: ServiceRequest) -> Self::Future { - let items = LogItems::from(sreq.head()); - sreq.extensions_mut().insert(items); - - Box::pin(self.service.call(sreq).and_then(move |sresp| { - if let Some(items) = sresp.request().extensions().get::() { - info!("{}", items); - } - - future::ok(sresp) - })) - } + Box::pin(async move { + let sresp = fut.await?; + + if let Some(items) = sresp.request().extensions().get::() { + info!("{}", items); + } + + Ok(sresp) + }) } diff --git a/syncserver/src/tokenserver/mod.rs b/syncserver/src/tokenserver/mod.rs index 52cb27fd..ff00392e 100644 --- a/syncserver/src/tokenserver/mod.rs +++ b/syncserver/src/tokenserver/mod.rs @@ -1,5 +1,3 @@ -pub mod auth; -pub mod db; pub mod extractors; pub mod handlers; pub mod logging; @@ -10,17 +8,15 @@ use serde::{ ser::{SerializeMap, Serializer}, Serialize, }; +use syncserver_common::{BlockingThreadpool, Metrics}; +use tokenserver_auth::{browserid, oauth, VerifyToken}; use tokenserver_common::NodeType; +use tokenserver_db::{params, DbPool, TokenserverPool}; use tokenserver_settings::Settings; use crate::{ - error::ApiError, - server::{metrics::Metrics, user_agent, BlockingThreadpool}, -}; -use auth::{browserid, oauth, VerifyToken}; -use db::{ - params, - pool::{DbPool, TokenserverPool}, + error::{ApiError, ApiErrorKind}, + server::user_agent, }; use std::{collections::HashMap, convert::TryFrom, fmt, sync::Arc}; @@ -49,7 +45,7 @@ impl ServerState { .expect("failed to create Tokenserver OAuth verifier"), ); let browserid_verifier = Box::new( - browserid::RemoteVerifier::try_from(settings) + browserid::Verifier::try_from(settings) .expect("failed to create Tokenserver BrowserID verifier"), ); let use_test_transactions = false; @@ -86,7 +82,7 @@ impl ServerState { token_duration: settings.token_duration, } }) - .map_err(Into::into) + .map_err(|_| ApiErrorKind::Internal("Failed to create Tokenserver pool".to_owned()).into()) } } diff --git a/syncserver/src/web/auth.rs b/syncserver/src/web/auth.rs index 958698ad..be39f6c4 100644 --- a/syncserver/src/web/auth.rs +++ b/syncserver/src/web/auth.rs @@ -16,6 +16,7 @@ use sha2::Sha256; use syncserver_common; use syncserver_settings::Secrets; use time::Duration; +use tokenserver_auth::TokenserverOrigin; use actix_web::dev::ConnectionInfo; use actix_web::http::Uri; @@ -26,7 +27,6 @@ use super::{ }; use crate::error::{ApiErrorKind, ApiResult}; use crate::label; -use crate::tokenserver::auth::TokenserverOrigin; /// A parsed and authenticated JSON payload /// extracted from the signed `id` property diff --git a/syncserver/src/web/extractors.rs b/syncserver/src/web/extractors.rs index 5bb2d2c7..e806a033 100644 --- a/syncserver/src/web/extractors.rs +++ b/syncserver/src/web/extractors.rs @@ -27,23 +27,23 @@ use serde::{ Deserialize, Serialize, }; use serde_json::Value; -use syncserver_common::X_WEAVE_RECORDS; -use syncserver_db_common::{ +use syncserver_common::{Metrics, X_WEAVE_RECORDS}; +use syncstorage_db::{ params::{self, PostCollectionBso}, - util::SyncTimestamp, - DbPool, Sorting, UserIdentifier, + DbError, DbPool, Sorting, SyncTimestamp, UserIdentifier, }; +use tokenserver_auth::TokenserverOrigin; use validator::{Validate, ValidationError}; -use crate::db::transaction::DbTransactionPool; use crate::error::{ApiError, ApiErrorKind}; use crate::label; -use crate::server::{metrics, ServerState, BSO_ID_REGEX, COLLECTION_ID_REGEX}; -use crate::tokenserver::auth::TokenserverOrigin; +use crate::server::{ + tags::Taggable, MetricsWrapper, ServerState, BSO_ID_REGEX, COLLECTION_ID_REGEX, +}; use crate::web::{ auth::HawkPayload, error::{HawkErrorKind, ValidationErrorKind}, - tags::Taggable, + transaction::DbTransactionPool, DOCKER_FLOW_ENDPOINTS, }; const BATCH_MAX_IDS: usize = 100; @@ -408,7 +408,6 @@ impl FromRequest for BsoBody { ) .into()); } - let state = match req.app_data::>() { Some(s) => s, None => { @@ -637,7 +636,7 @@ impl FromRequest for CollectionParam { pub struct MetaRequest { pub user_id: UserIdentifier, pub tokenserver_origin: TokenserverOrigin, - pub metrics: metrics::Metrics, + pub metrics: Metrics, } impl FromRequest for MetaRequest { @@ -655,7 +654,7 @@ impl FromRequest for MetaRequest { Ok(MetaRequest { tokenserver_origin: user_id.tokenserver_origin, user_id: user_id.into(), - metrics: metrics::Metrics::extract(&req).await?, + metrics: MetricsWrapper::extract(&req).await?.0, }) } .boxed_local() @@ -678,7 +677,7 @@ pub struct CollectionRequest { pub tokenserver_origin: TokenserverOrigin, pub query: BsoQueryParams, pub reply: ReplyFormat, - pub metrics: metrics::Metrics, + pub metrics: Metrics, } impl FromRequest for CollectionRequest { @@ -719,7 +718,7 @@ impl FromRequest for CollectionRequest { user_id: user_id.into(), query, reply, - metrics: metrics::Metrics::extract(&req).await?, + metrics: MetricsWrapper::extract(&req).await?.0, }) } .boxed_local() @@ -738,7 +737,7 @@ pub struct CollectionPostRequest { pub query: BsoQueryParams, pub bsos: BsoBodies, pub batch: Option, - pub metrics: metrics::Metrics, + pub metrics: Metrics, pub quota_enabled: bool, } @@ -817,7 +816,7 @@ impl FromRequest for CollectionPostRequest { query, bsos, batch: batch.opt, - metrics: metrics::Metrics::extract(&req).await?, + metrics: MetricsWrapper::extract(&req).await?.0, quota_enabled: state.quota_enabled, }) }) @@ -834,7 +833,7 @@ pub struct BsoRequest { pub tokenserver_origin: TokenserverOrigin, pub query: BsoQueryParams, pub bso: String, - pub metrics: metrics::Metrics, + pub metrics: Metrics, } impl FromRequest for BsoRequest { @@ -860,7 +859,7 @@ impl FromRequest for BsoRequest { user_id: user_id.into(), query, bso: bso.bso, - metrics: metrics::Metrics::extract(&req).await?, + metrics: MetricsWrapper::extract(&req).await?.0, }) }) } @@ -876,7 +875,7 @@ pub struct BsoPutRequest { pub query: BsoQueryParams, pub bso: String, pub body: BsoBody, - pub metrics: metrics::Metrics, + pub metrics: Metrics, } impl FromRequest for BsoPutRequest { @@ -889,7 +888,7 @@ impl FromRequest for BsoPutRequest { let mut payload = payload.take(); async move { - let metrics = metrics::Metrics::extract(&req).await?; + let metrics = MetricsWrapper::extract(&req).await?.0; let (user_id, collection, query, bso, body) = <( HawkIdentifier, @@ -938,7 +937,7 @@ pub struct QuotaInfo { #[derive(Clone, Debug)] pub struct HeartbeatRequest { pub headers: HeaderMap, - pub db_pool: Box, + pub db_pool: Box>, pub quota: QuotaInfo, } @@ -1755,13 +1754,12 @@ mod tests { use serde_json::{self, json}; use sha2::Sha256; use syncserver_common; - use syncserver_db_common::Db; use syncserver_settings::Settings as GlobalSettings; use syncstorage_settings::{Deadman, ServerLimits, Settings as SyncstorageSettings}; use tokio::sync::RwLock; - use crate::db::mock::{MockDb, MockDbPool}; - use crate::server::{metrics, ServerState}; + use crate::server::ServerState; + use syncstorage_db::mock::{MockDb, MockDbPool}; use crate::web::auth::HawkPayload; @@ -1779,8 +1777,8 @@ mod tests { const INVALID_BSO_NAME: &str = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; - fn make_db() -> Box> { - Box::new(MockDb::new()) + fn make_db() -> MockDb { + MockDb::new() } fn make_state() -> ServerState { @@ -1792,7 +1790,7 @@ mod tests { limits_json: serde_json::to_string(&**SERVER_LIMITS).unwrap(), port: 8000, metrics: Box::new( - metrics::metrics_from_opts( + syncserver_common::metrics_from_opts( &syncstorage_settings.statsd_label, syncserver_settings.statsd_host.as_deref(), syncserver_settings.statsd_port, diff --git a/syncserver/src/web/handlers.rs b/syncserver/src/web/handlers.rs index 762f5bca..7af9432e 100644 --- a/syncserver/src/web/handlers.rs +++ b/syncserver/src/web/handlers.rs @@ -6,21 +6,22 @@ use actix_web::{dev::HttpResponseBuilder, http::StatusCode, web::Data, HttpReque use serde::Serialize; use serde_json::{json, Value}; use syncserver_common::{X_LAST_MODIFIED, X_WEAVE_NEXT_OFFSET, X_WEAVE_RECORDS}; -use syncserver_db_common::{ - error::{DbError, DbErrorKind}, +use syncstorage_db::{ params, results::{CreateBatch, Paginated}, - Db, + Db, DbError, DbErrorIntrospect, }; use time; use crate::{ - db::transaction::DbTransactionPool, error::{ApiError, ApiErrorKind}, server::ServerState, - web::extractors::{ - BsoPutRequest, BsoRequest, CollectionPostRequest, CollectionRequest, EmitApiMetric, - HeartbeatRequest, MetaRequest, ReplyFormat, TestErrorRequest, + web::{ + extractors::{ + BsoPutRequest, BsoRequest, CollectionPostRequest, CollectionRequest, EmitApiMetric, + HeartbeatRequest, MetaRequest, ReplyFormat, TestErrorRequest, + }, + transaction::DbTransactionPool, }, }; @@ -189,7 +190,7 @@ pub async fn get_collection( async fn finish_get_collection( coll: &CollectionRequest, - db: Box + '_>, + db: Box>, result: Result, DbError>, ) -> Result where @@ -280,15 +281,16 @@ pub async fn post_collection( // the entire, accumulated if the `commit` flag is set. pub async fn post_collection_batch( coll: CollectionPostRequest, - db: Box + '_>, + db: Box>, ) -> Result { coll.emit_api_metric("request.post_collection_batch"); trace!("Batch: Post collection batch"); // Bail early if we have nonsensical arguments // TODO: issue932 may make these multi-level transforms easier - let breq = coll.batch.clone().ok_or_else(|| -> ApiError { - ApiErrorKind::Db(DbErrorKind::BatchNotFound.into()).into() - })?; + let breq = coll + .batch + .clone() + .ok_or_else(|| -> ApiError { ApiErrorKind::Db(DbError::batch_not_found()).into() })?; let new_batch = if let Some(id) = breq.id.clone() { trace!("Batch: Validating {}", &id); @@ -313,14 +315,13 @@ pub async fn post_collection_batch( CreateBatch { id: id.clone(), size: if coll.quota_enabled { - Some(usage.total_bytes as usize) + Some(usage.total_bytes) } else { None }, } } else { - let err: DbError = DbErrorKind::BatchNotFound.into(); - return Err(ApiError::from(err)); + return Err(ApiErrorKind::Db(DbError::batch_not_found()).into()); } } else { trace!("Batch: Creating new batch"); @@ -405,8 +406,7 @@ pub async fn post_collection_batch( }) .await? } else { - let err: DbError = DbErrorKind::BatchNotFound.into(); - return Err(ApiError::from(err)); + return Err(ApiErrorKind::Db(DbError::batch_not_found()).into()); }; // Then, write the BSOs contained in the commit request into the BSO table. @@ -594,7 +594,7 @@ pub async fn lbheartbeat(req: HttpRequest) -> Result { let db_state = if cfg!(test) { use actix_web::http::header::HeaderValue; use std::str::FromStr; - use syncserver_db_common::PoolState; + use syncstorage_db::PoolState; let test_pool = PoolState { connections: u32::from_str( diff --git a/syncserver/src/web/middleware/mod.rs b/syncserver/src/web/middleware/mod.rs index 78e3f4b6..f8c5cecf 100644 --- a/syncserver/src/web/middleware/mod.rs +++ b/syncserver/src/web/middleware/mod.rs @@ -13,10 +13,11 @@ use actix_web::{ dev::{Service, ServiceRequest, ServiceResponse}, web::Data, }; +use syncserver_common::Metrics; +use tokenserver_auth::TokenserverOrigin; use crate::error::{ApiError, ApiErrorKind}; -use crate::server::{metrics::Metrics, ServerState}; -use crate::tokenserver::auth::TokenserverOrigin; +use crate::server::ServerState; pub fn emit_http_status_with_tokenserver_origin( req: ServiceRequest, diff --git a/syncserver/src/web/middleware/rejectua.rs b/syncserver/src/web/middleware/rejectua.rs index 2e082ec7..d27115d0 100644 --- a/syncserver/src/web/middleware/rejectua.rs +++ b/syncserver/src/web/middleware/rejectua.rs @@ -1,17 +1,16 @@ #![allow(clippy::type_complexity)] -use std::task::{Context, Poll}; use actix_web::{ - dev::{Service, ServiceRequest, ServiceResponse, Transform}, + dev::{Service, ServiceRequest, ServiceResponse}, http::header::USER_AGENT, - Error, FromRequest, HttpResponse, + FromRequest, HttpResponse, }; -use futures::future::{self, LocalBoxFuture, Ready}; +use futures::future::LocalBoxFuture; use lazy_static::lazy_static; use regex::Regex; use crate::error::{ApiError, ApiErrorKind}; -use crate::server::metrics::Metrics; +use crate::server::MetricsWrapper; lazy_static! { // e.g. "Firefox-iOS-Sync/18.0b1 (iPhone; iPhone OS 13.2.2) (Fennec (synctesting))" @@ -32,67 +31,35 @@ $ .unwrap(); } -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug, Default)] -pub struct RejectUA; - -impl Transform for RejectUA -where - S: Service, Error = Error>, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = RejectUAMiddleware; - type Future = Ready>; - - fn new_transform(&self, service: S) -> Self::Future { - future::ok(RejectUAMiddleware { service }) - } -} -#[allow(clippy::upper_case_acronyms)] -pub struct RejectUAMiddleware { - service: S, -} - -impl Service for RejectUAMiddleware -where - S: Service, Error = Error>, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = LocalBoxFuture<'static, Result>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, sreq: ServiceRequest) -> Self::Future { - match sreq.headers().get(USER_AGENT).cloned() { - Some(header) if header.to_str().map_or(false, should_reject) => Box::pin(async move { - trace!("Rejecting User-Agent: {:?}", header); - let (req, payload) = sreq.into_parts(); - Metrics::extract(&req).await?.incr("error.rejectua"); - let sreq = ServiceRequest::from_parts(req, payload).map_err(|_| { - ApiError::from(ApiErrorKind::Internal( - "failed to reconstruct ServiceRequest from its parts".to_owned(), - )) - })?; - - Ok(sreq.into_response( - HttpResponse::ServiceUnavailable() - .body("0".to_owned()) - .into_body(), +pub fn reject_user_agent( + request: ServiceRequest, + service: &mut (impl Service< + Request = ServiceRequest, + Response = ServiceResponse, + Error = actix_web::Error, + > + 'static), +) -> LocalBoxFuture<'static, Result> { + match request.headers().get(USER_AGENT).cloned() { + Some(header) if header.to_str().map_or(false, should_reject) => Box::pin(async move { + trace!("Rejecting User-Agent: {:?}", header); + let (req, payload) = request.into_parts(); + MetricsWrapper::extract(&req) + .await? + .0 + .incr("error.rejectua"); + let sreq = ServiceRequest::from_parts(req, payload).map_err(|_| { + ApiError::from(ApiErrorKind::Internal( + "failed to reconstruct ServiceRequest from its parts".to_owned(), )) - }), - _ => Box::pin(self.service.call(sreq)), - } + })?; + + Ok(sreq.into_response( + HttpResponse::ServiceUnavailable() + .body("0".to_owned()) + .into_body(), + )) + }), + _ => Box::pin(service.call(request)), } } diff --git a/syncserver/src/web/middleware/sentry.rs b/syncserver/src/web/middleware/sentry.rs index 7af3741c..6fff74a1 100644 --- a/syncserver/src/web/middleware/sentry.rs +++ b/syncserver/src/web/middleware/sentry.rs @@ -1,63 +1,21 @@ use std::collections::HashMap; use std::error::Error as StdError; -use std::task::{Context, Poll}; -use std::{cell::RefCell, rc::Rc}; +use std::future::Future; use actix_http::HttpMessage; use actix_web::{ - dev::{Service, ServiceRequest, ServiceResponse, Transform}, + dev::{Service, ServiceRequest, ServiceResponse}, http::header::USER_AGENT, - Error, FromRequest, + FromRequest, }; -use futures::future::{self, LocalBoxFuture}; use sentry::protocol::Event; use sentry_backtrace::parse_stacktrace; use serde_json::value::Value; -use syncserver_common::ReportableError; -use tokenserver_common::error::TokenserverError; +use syncserver_common::{Metrics, ReportableError}; +use tokenserver_common::TokenserverError; use crate::error::ApiError; -use crate::server::{metrics::Metrics, user_agent}; -use crate::web::tags::Taggable; - -pub struct SentryWrapper; - -impl SentryWrapper { - pub fn new() -> Self { - SentryWrapper::default() - } -} - -impl Default for SentryWrapper { - fn default() -> Self { - Self - } -} - -impl Transform for SentryWrapper -where - S: Service, Error = Error> + 'static, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = SentryWrapperMiddleware; - type Future = LocalBoxFuture<'static, Result>; - - fn new_transform(&self, service: S) -> Self::Future { - Box::pin(future::ok(SentryWrapperMiddleware { - service: Rc::new(RefCell::new(service)), - })) - } -} - -#[derive(Debug)] -pub struct SentryWrapperMiddleware { - service: Rc>, -} +use crate::server::{tags::Taggable, user_agent, MetricsWrapper}; pub fn report( tags: HashMap, @@ -72,70 +30,61 @@ pub fn report( sentry::capture_event(event); } -impl Service for SentryWrapperMiddleware -where - S: Service, Error = Error> + 'static, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = LocalBoxFuture<'static, Result>; +pub fn report_error( + request: ServiceRequest, + service: &mut impl Service< + Request = ServiceRequest, + Response = ServiceResponse, + Error = actix_web::Error, + >, +) -> impl Future> { + add_initial_tags(&request, request.head().method.to_string()); + add_initial_extras(&request, request.head().uri.to_string()); - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } + let fut = service.call(request); - fn call(&mut self, sreq: ServiceRequest) -> Self::Future { - add_initial_tags(&sreq, sreq.head().method.to_string()); - add_initial_extras(&sreq, sreq.head().uri.to_string()); + Box::pin(async move { + let mut sresp = fut.await?; + let tags = sresp.request().get_tags(); + let extras = sresp.request().get_extras(); - let fut = self.service.call(sreq); - - Box::pin(async move { - let mut sresp = fut.await?; - let tags = sresp.request().get_tags(); - let extras = sresp.request().get_extras(); - - match sresp.response().error() { - None => { - // Middleware errors are eaten by current versions of Actix. Errors are now added - // to the extensions. Need to check both for any errors and report them. - if let Some(events) = sresp - .request() - .extensions_mut() - .remove::>>() - { - for event in events { - trace!("Sentry: found an error stored in request: {:?}", &event); - report(tags.clone(), extras.clone(), event); - } - } - if let Some(events) = sresp - .response_mut() - .extensions_mut() - .remove::>>() - { - for event in events { - trace!("Sentry: Found an error stored in response: {:?}", &event); - report(tags.clone(), extras.clone(), event); - } + match sresp.response().error() { + None => { + // Middleware errors are eaten by current versions of Actix. Errors are now added + // to the extensions. Need to check both for any errors and report them. + if let Some(events) = sresp + .request() + .extensions_mut() + .remove::>>() + { + for event in events { + trace!("Sentry: found an error stored in request: {:?}", &event); + report(tags.clone(), extras.clone(), event); } } - Some(e) => { - let metrics = Metrics::extract(sresp.request()).await.unwrap(); - - if let Some(apie) = e.as_error::() { - process_error(apie, metrics, tags, extras); - } else if let Some(tokenserver_error) = e.as_error::() { - process_error(tokenserver_error, metrics, tags, extras); + if let Some(events) = sresp + .response_mut() + .extensions_mut() + .remove::>>() + { + for event in events { + trace!("Sentry: Found an error stored in response: {:?}", &event); + report(tags.clone(), extras.clone(), event); } } } - Ok(sresp) - }) - } + Some(e) => { + let metrics = MetricsWrapper::extract(sresp.request()).await.unwrap().0; + + if let Some(apie) = e.as_error::() { + process_error(apie, metrics, tags, extras); + } else if let Some(tokenserver_error) = e.as_error::() { + process_error(tokenserver_error, metrics, tags, extras); + } + } + } + Ok(sresp) + }) } fn process_error( diff --git a/syncserver/src/web/middleware/weave.rs b/syncserver/src/web/middleware/weave.rs index 6f72e815..b4652b11 100644 --- a/syncserver/src/web/middleware/weave.rs +++ b/syncserver/src/web/middleware/weave.rs @@ -1,57 +1,44 @@ use std::fmt::Display; - -use std::task::{Context, Poll}; +use std::future::Future; use actix_web::{ - dev::{Service, ServiceRequest, ServiceResponse, Transform}, + dev::{Service, ServiceRequest, ServiceResponse}, http::header::{self, HeaderMap}, - Error, }; -use futures::future::{self, LocalBoxFuture}; use syncserver_common::{X_LAST_MODIFIED, X_WEAVE_TIMESTAMP}; -use syncserver_db_common::util::SyncTimestamp; +use syncstorage_db::SyncTimestamp; use crate::error::{ApiError, ApiErrorKind}; use crate::web::DOCKER_FLOW_ENDPOINTS; -pub struct WeaveTimestampMiddleware { - service: S, -} +/// Middleware to set the X-Weave-Timestamp header on all responses. +pub fn set_weave_timestamp( + request: ServiceRequest, + service: &mut impl Service< + Request = ServiceRequest, + Response = ServiceResponse, + Error = actix_web::Error, + >, +) -> impl Future> { + let request_path = request.uri().path().to_lowercase(); + let ts = SyncTimestamp::default().as_seconds(); + let fut = service.call(request); -impl Service for WeaveTimestampMiddleware -where - S: Service, Error = Error>, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type Future = LocalBoxFuture<'static, Result>; - - fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { - self.service.poll_ready(cx) - } - - fn call(&mut self, sreq: ServiceRequest) -> Self::Future { - if DOCKER_FLOW_ENDPOINTS.contains(&sreq.uri().path().to_lowercase().as_str()) { - return Box::pin(self.service.call(sreq)); + Box::pin(async move { + if DOCKER_FLOW_ENDPOINTS.contains(&request_path.as_str()) { + return fut.await; } - let ts = SyncTimestamp::default().as_seconds(); - let fut = self.service.call(sreq); - Box::pin(async move { - let mut resp = fut.await?; - set_weave_timestamp(resp.headers_mut(), ts)?; - Ok(resp) - }) - } + let mut resp = fut.await?; + insert_weave_timestamp_into_headers(resp.headers_mut(), ts)?; + Ok(resp) + }) } /// Set a X-Weave-Timestamp header on all responses (depending on the /// response's X-Last-Modified header) -fn set_weave_timestamp(headers: &mut HeaderMap, ts: f64) -> Result<(), ApiError> { +fn insert_weave_timestamp_into_headers(headers: &mut HeaderMap, ts: f64) -> Result<(), ApiError> { fn invalid_xlm(e: E) -> ApiError where E: Display, @@ -80,39 +67,6 @@ fn set_weave_timestamp(headers: &mut HeaderMap, ts: f64) -> Result<(), ApiError> Ok(()) } -/// Middleware to set the X-Weave-Timestamp header on all responses. -pub struct WeaveTimestamp; - -impl WeaveTimestamp { - pub fn new() -> Self { - WeaveTimestamp::default() - } -} - -impl Default for WeaveTimestamp { - fn default() -> Self { - Self - } -} - -impl Transform for WeaveTimestamp -where - S: Service, Error = Error>, - S::Future: 'static, - B: 'static, -{ - type Request = ServiceRequest; - type Response = ServiceResponse; - type Error = Error; - type InitError = (); - type Transform = WeaveTimestampMiddleware; - type Future = LocalBoxFuture<'static, Result>; - - fn new_transform(&self, service: S) -> Self::Future { - Box::pin(future::ok(WeaveTimestampMiddleware { service })) - } -} - #[cfg(test)] mod tests { use super::*; @@ -122,7 +76,11 @@ mod tests { #[test] fn test_no_modified_header() { let mut resp = HttpResponse::build(http::StatusCode::OK).finish(); - set_weave_timestamp(resp.headers_mut(), SyncTimestamp::default().as_seconds()).unwrap(); + insert_weave_timestamp_into_headers( + resp.headers_mut(), + SyncTimestamp::default().as_seconds(), + ) + .unwrap(); let weave_hdr = resp .headers() .get(X_WEAVE_TIMESTAMP) @@ -146,7 +104,7 @@ mod tests { let mut resp = HttpResponse::build(http::StatusCode::OK) .header(X_LAST_MODIFIED, hts.clone()) .finish(); - set_weave_timestamp(resp.headers_mut(), ts as f64).unwrap(); + insert_weave_timestamp_into_headers(resp.headers_mut(), ts as f64).unwrap(); let weave_hdr = resp .headers() .get(X_WEAVE_TIMESTAMP) @@ -166,7 +124,7 @@ mod tests { let mut resp = HttpResponse::build(http::StatusCode::OK) .header(X_LAST_MODIFIED, hts.clone()) .finish(); - set_weave_timestamp(resp.headers_mut(), ts as f64 / 1_000.0).unwrap(); + insert_weave_timestamp_into_headers(resp.headers_mut(), ts as f64 / 1_000.0).unwrap(); let weave_hdr = resp .headers() .get(X_WEAVE_TIMESTAMP) diff --git a/syncserver/src/web/mod.rs b/syncserver/src/web/mod.rs index f759eee5..671c8289 100644 --- a/syncserver/src/web/mod.rs +++ b/syncserver/src/web/mod.rs @@ -4,7 +4,7 @@ pub mod error; pub mod extractors; pub mod handlers; pub mod middleware; -pub mod tags; +mod transaction; // Known DockerFlow commands for Ops callbacks pub const DOCKER_FLOW_ENDPOINTS: [&str; 4] = [ diff --git a/syncserver/src/db/transaction.rs b/syncserver/src/web/transaction.rs similarity index 92% rename from syncserver/src/db/transaction.rs rename to syncserver/src/web/transaction.rs index 68c82c8d..dfbfb7b3 100644 --- a/syncserver/src/db/transaction.rs +++ b/syncserver/src/web/transaction.rs @@ -9,20 +9,18 @@ use actix_web::{FromRequest, HttpRequest, HttpResponse}; use futures::future::LocalBoxFuture; use futures::FutureExt; use syncserver_common::X_LAST_MODIFIED; -use syncserver_db_common::{params, Db, DbPool, UserIdentifier}; +use syncstorage_db::{params, results::ConnectionInfo, Db, DbError, DbPool, UserIdentifier}; -use crate::db::results::ConnectionInfo; use crate::error::{ApiError, ApiErrorKind}; -use crate::server::metrics::Metrics; -use crate::server::ServerState; +use crate::server::tags::Taggable; +use crate::server::{MetricsWrapper, ServerState}; use crate::web::extractors::{ BsoParam, CollectionParam, HawkIdentifier, PreConditionHeader, PreConditionHeaderOpt, }; -use crate::web::tags::Taggable; #[derive(Clone)] pub struct DbTransactionPool { - pool: Box, + pool: Box>, is_read: bool, user_id: UserIdentifier, collection: Option, @@ -51,10 +49,10 @@ impl DbTransactionPool { &'a self, request: HttpRequest, action: A, - ) -> Result<(R, Box>), ApiError> + ) -> Result<(R, Box>), ApiError> where - A: FnOnce(Box>) -> F, - F: Future> + 'a, + A: FnOnce(Box>) -> F, + F: Future>, { // Get connection from pool let db = self.pool.get().await?; @@ -88,7 +86,7 @@ impl DbTransactionPool { } } - pub fn get_pool(&self) -> Result, Error> { + pub fn get_pool(&self) -> Result>, Error> { Ok(self.pool.clone()) } @@ -99,7 +97,7 @@ impl DbTransactionPool { action: A, ) -> Result where - A: FnOnce(Box>) -> F, + A: FnOnce(Box>) -> F, F: Future> + 'a, { let (resp, db) = self.transaction_internal(request, action).await?; @@ -117,11 +115,11 @@ impl DbTransactionPool { action: A, ) -> Result where - A: FnOnce(Box>) -> F, + A: FnOnce(Box>) -> F, F: Future> + 'a, { let mreq = request.clone(); - let check_precondition = move |db: Box>| { + let check_precondition = move |db: Box>| { async move { // set the extra information for all requests so we capture default err handlers. set_extra(&mreq, db.get_connection_info()); @@ -233,9 +231,10 @@ impl FromRequest for DbTransactionPool { Err(e) => { // Semi-example to show how to use metrics inside of middleware. // `Result::unwrap` is safe to use here, since Metrics::extract can never fail - Metrics::extract(&req) + MetricsWrapper::extract(&req) .await .unwrap() + .0 .incr("sync.error.collectionParam"); warn!("⚠️ CollectionParam err: {:?}", e); return Err(e); diff --git a/syncstorage-db-common/Cargo.toml b/syncstorage-db-common/Cargo.toml new file mode 100644 index 00000000..31b37b9e --- /dev/null +++ b/syncstorage-db-common/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "syncstorage-db-common" +version = "0.13.6" +edition = "2021" + +[dependencies] +async-trait = "0.1.40" +backtrace = "0.3.61" +chrono = "0.4" +diesel = { version = "1.4", features = ["mysql", "r2d2"] } +diesel_migrations = { version = "1.4.0", features = ["mysql"] } +futures = { version = "0.3", features = ["compat"] } +http = "0.2.6" +lazy_static = "1.4.0" +serde = "1.0" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } +thiserror = "1.0.26" diff --git a/syncstorage-db-common/src/error.rs b/syncstorage-db-common/src/error.rs new file mode 100644 index 00000000..aa1716a5 --- /dev/null +++ b/syncstorage-db-common/src/error.rs @@ -0,0 +1,138 @@ +use std::fmt; + +use backtrace::Backtrace; +use http::StatusCode; +use syncserver_common::{impl_fmt_display, ReportableError}; +use thiserror::Error; + +/// Errors common to all supported syncstorage database backends. These errors can be thought of +/// as being related more to the syncstorage application logic as opposed to a particular +/// database backend. +#[derive(Debug)] +pub struct SyncstorageDbError { + kind: SyncstorageDbErrorKind, + pub status: StatusCode, + pub backtrace: Backtrace, +} + +#[derive(Debug, Error)] +enum SyncstorageDbErrorKind { + #[error("Specified collection does not exist")] + CollectionNotFound, + + #[error("Specified bso does not exist")] + BsoNotFound, + + #[error("Specified batch does not exist")] + BatchNotFound, + + #[error("An attempt at a conflicting write")] + Conflict, + + #[error("Unexpected error: {}", _0)] + Internal(String), + + #[error("User over quota")] + Quota, +} + +impl SyncstorageDbError { + pub fn batch_not_found() -> Self { + SyncstorageDbErrorKind::BatchNotFound.into() + } + + pub fn bso_not_found() -> Self { + SyncstorageDbErrorKind::BsoNotFound.into() + } + + pub fn collection_not_found() -> Self { + SyncstorageDbErrorKind::CollectionNotFound.into() + } + + pub fn conflict() -> Self { + SyncstorageDbErrorKind::Conflict.into() + } + + pub fn internal(msg: String) -> Self { + SyncstorageDbErrorKind::Internal(msg).into() + } + + pub fn quota() -> Self { + SyncstorageDbErrorKind::Quota.into() + } +} + +pub trait DbErrorIntrospect { + fn is_collection_not_found(&self) -> bool; + fn is_conflict(&self) -> bool; + fn is_quota(&self) -> bool; + fn is_bso_not_found(&self) -> bool; + fn is_batch_not_found(&self) -> bool; +} + +impl DbErrorIntrospect for SyncstorageDbError { + fn is_collection_not_found(&self) -> bool { + matches!(self.kind, SyncstorageDbErrorKind::CollectionNotFound) + } + + fn is_conflict(&self) -> bool { + matches!(self.kind, SyncstorageDbErrorKind::Conflict) + } + + fn is_quota(&self) -> bool { + matches!(self.kind, SyncstorageDbErrorKind::Quota) + } + + fn is_bso_not_found(&self) -> bool { + matches!(self.kind, SyncstorageDbErrorKind::BsoNotFound) + } + + fn is_batch_not_found(&self) -> bool { + matches!(self.kind, SyncstorageDbErrorKind::BatchNotFound) + } +} + +impl ReportableError for SyncstorageDbError { + fn is_sentry_event(&self) -> bool { + !matches!(&self.kind, SyncstorageDbErrorKind::Conflict) + } + + fn metric_label(&self) -> Option { + match &self.kind { + SyncstorageDbErrorKind::Conflict => Some("storage.conflict".to_owned()), + _ => None, + } + } + + fn error_backtrace(&self) -> String { + format!("{:#?}", self.backtrace) + } +} + +impl From for SyncstorageDbError { + fn from(kind: SyncstorageDbErrorKind) -> Self { + let status = match kind { + SyncstorageDbErrorKind::CollectionNotFound | SyncstorageDbErrorKind::BsoNotFound => { + StatusCode::NOT_FOUND + } + // Matching the Python code here (a 400 vs 404) + SyncstorageDbErrorKind::BatchNotFound => StatusCode::BAD_REQUEST, + // NOTE: the protocol specification states that we should return a + // "409 Conflict" response here, but clients currently do not + // handle these respones very well: + // * desktop bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959034 + // * android bug: https://bugzilla.mozilla.org/show_bug.cgi?id=959032 + SyncstorageDbErrorKind::Conflict => StatusCode::SERVICE_UNAVAILABLE, + SyncstorageDbErrorKind::Quota => StatusCode::FORBIDDEN, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + + Self { + kind, + status, + backtrace: Backtrace::new(), + } + } +} + +impl_fmt_display!(SyncstorageDbError, SyncstorageDbErrorKind); diff --git a/syncstorage-db-common/src/lib.rs b/syncstorage-db-common/src/lib.rs new file mode 100644 index 00000000..c1613614 --- /dev/null +++ b/syncstorage-db-common/src/lib.rs @@ -0,0 +1,304 @@ +pub mod error; +pub mod params; +pub mod results; +pub mod util; + +use std::fmt::Debug; + +use async_trait::async_trait; +use futures::{future, TryFutureExt}; +use lazy_static::lazy_static; +use serde::Deserialize; +use syncserver_db_common::{DbFuture, GetPoolState}; + +use error::DbErrorIntrospect; +use util::SyncTimestamp; + +lazy_static! { + /// For efficiency, it's possible to use fixed pre-determined IDs for + /// common collection names. This is the canonical list of such + /// names. Non-standard collections will be allocated IDs starting + /// from the highest ID in this collection. + pub static ref STD_COLLS: Vec<(i32, &'static str)> = { + vec![ + (1, "clients"), + (2, "crypto"), + (3, "forms"), + (4, "history"), + (5, "keys"), + (6, "meta"), + (7, "bookmarks"), + (8, "prefs"), + (9, "tabs"), + (10, "passwords"), + (11, "addons"), + (12, "addresses"), + (13, "creditcards"), + ] + }; +} + +/// Rough guesstimate of the maximum reasonable life span of a batch +pub const BATCH_LIFETIME: i64 = 2 * 60 * 60 * 1000; // 2 hours, in milliseconds + +/// The ttl to use for rows that are never supposed to expire (in seconds) +pub const DEFAULT_BSO_TTL: u32 = 2_100_000_000; + +/// Non-standard collections will be allocated IDs beginning with this value +pub const FIRST_CUSTOM_COLLECTION_ID: i32 = 101; + +#[async_trait] +pub trait DbPool: Sync + Send + Debug + GetPoolState { + type Error; + + async fn get(&self) -> Result>, Self::Error>; + + fn validate_batch_id(&self, params: params::ValidateBatchId) -> Result<(), Self::Error>; + + fn box_clone(&self) -> Box>; +} + +impl Clone for Box> { + fn clone(&self) -> Box> { + self.box_clone() + } +} + +pub trait Db: Debug { + type Error: DbErrorIntrospect + 'static; + + fn lock_for_read(&self, params: params::LockCollection) -> DbFuture<'_, (), Self::Error>; + + fn lock_for_write(&self, params: params::LockCollection) -> DbFuture<'_, (), Self::Error>; + + fn begin(&self, for_write: bool) -> DbFuture<'_, (), Self::Error>; + + fn commit(&self) -> DbFuture<'_, (), Self::Error>; + + fn rollback(&self) -> DbFuture<'_, (), Self::Error>; + + fn get_collection_timestamps( + &self, + params: params::GetCollectionTimestamps, + ) -> DbFuture<'_, results::GetCollectionTimestamps, Self::Error>; + + fn get_collection_timestamp( + &self, + params: params::GetCollectionTimestamp, + ) -> DbFuture<'_, results::GetCollectionTimestamp, Self::Error>; + + fn get_collection_counts( + &self, + params: params::GetCollectionCounts, + ) -> DbFuture<'_, results::GetCollectionCounts, Self::Error>; + + fn get_collection_usage( + &self, + params: params::GetCollectionUsage, + ) -> DbFuture<'_, results::GetCollectionUsage, Self::Error>; + + fn get_storage_timestamp( + &self, + params: params::GetStorageTimestamp, + ) -> DbFuture<'_, results::GetStorageTimestamp, Self::Error>; + + fn get_storage_usage( + &self, + params: params::GetStorageUsage, + ) -> DbFuture<'_, results::GetStorageUsage, Self::Error>; + + fn get_quota_usage( + &self, + params: params::GetQuotaUsage, + ) -> DbFuture<'_, results::GetQuotaUsage, Self::Error>; + + fn delete_storage( + &self, + params: params::DeleteStorage, + ) -> DbFuture<'_, results::DeleteStorage, Self::Error>; + + fn delete_collection( + &self, + params: params::DeleteCollection, + ) -> DbFuture<'_, results::DeleteCollection, Self::Error>; + + fn delete_bsos( + &self, + params: params::DeleteBsos, + ) -> DbFuture<'_, results::DeleteBsos, Self::Error>; + + fn get_bsos(&self, params: params::GetBsos) -> DbFuture<'_, results::GetBsos, Self::Error>; + + fn get_bso_ids(&self, params: params::GetBsos) + -> DbFuture<'_, results::GetBsoIds, Self::Error>; + + fn post_bsos(&self, params: params::PostBsos) -> DbFuture<'_, results::PostBsos, Self::Error>; + + fn delete_bso( + &self, + params: params::DeleteBso, + ) -> DbFuture<'_, results::DeleteBso, Self::Error>; + + fn get_bso(&self, params: params::GetBso) + -> DbFuture<'_, Option, Self::Error>; + + fn get_bso_timestamp( + &self, + params: params::GetBsoTimestamp, + ) -> DbFuture<'_, results::GetBsoTimestamp, Self::Error>; + + fn put_bso(&self, params: params::PutBso) -> DbFuture<'_, results::PutBso, Self::Error>; + + fn create_batch( + &self, + params: params::CreateBatch, + ) -> DbFuture<'_, results::CreateBatch, Self::Error>; + + fn validate_batch( + &self, + params: params::ValidateBatch, + ) -> DbFuture<'_, results::ValidateBatch, Self::Error>; + + fn append_to_batch( + &self, + params: params::AppendToBatch, + ) -> DbFuture<'_, results::AppendToBatch, Self::Error>; + + fn get_batch( + &self, + params: params::GetBatch, + ) -> DbFuture<'_, Option, Self::Error>; + + fn commit_batch( + &self, + params: params::CommitBatch, + ) -> DbFuture<'_, results::CommitBatch, Self::Error>; + + fn box_clone(&self) -> Box>; + + fn check(&self) -> DbFuture<'_, results::Check, Self::Error>; + + fn get_connection_info(&self) -> results::ConnectionInfo; + + /// Retrieve the timestamp for an item/collection + /// + /// Modeled on the Python `get_resource_timestamp` function. + fn extract_resource( + &self, + user_id: UserIdentifier, + collection: Option, + bso: Option, + ) -> DbFuture<'_, SyncTimestamp, Self::Error> { + // If there's no collection, we return the overall storage timestamp + let collection = match collection { + Some(collection) => collection, + None => return Box::pin(self.get_storage_timestamp(user_id)), + }; + // If there's no bso, return the collection + let bso = match bso { + Some(bso) => bso, + None => { + return Box::pin( + self.get_collection_timestamp(params::GetCollectionTimestamp { + user_id, + collection, + }) + .or_else(|e| { + if e.is_collection_not_found() { + future::ok(SyncTimestamp::from_seconds(0f64)) + } else { + future::err(e) + } + }), + ) + } + }; + Box::pin( + self.get_bso_timestamp(params::GetBsoTimestamp { + user_id, + collection, + id: bso, + }) + .or_else(|e| { + if e.is_collection_not_found() { + future::ok(SyncTimestamp::from_seconds(0f64)) + } else { + future::err(e) + } + }), + ) + } + + /// Internal methods used by the db tests + + fn get_collection_id(&self, name: String) -> DbFuture<'_, i32, Self::Error>; + + fn create_collection(&self, name: String) -> DbFuture<'_, i32, Self::Error>; + + fn update_collection( + &self, + params: params::UpdateCollection, + ) -> DbFuture<'_, SyncTimestamp, Self::Error>; + + fn timestamp(&self) -> SyncTimestamp; + + fn set_timestamp(&self, timestamp: SyncTimestamp); + + fn delete_batch(&self, params: params::DeleteBatch) -> DbFuture<'_, (), Self::Error>; + + fn clear_coll_cache(&self) -> DbFuture<'_, (), Self::Error>; + + fn set_quota(&mut self, enabled: bool, limit: usize, enforce: bool); +} + +impl Clone for Box> +where + E: DbErrorIntrospect + 'static, +{ + fn clone(&self) -> Box> { + self.box_clone() + } +} + +#[derive(Debug, Deserialize, Clone, PartialEq, Eq, Copy)] +#[serde(rename_all = "lowercase")] +pub enum Sorting { + None, + Newest, + Oldest, + Index, +} + +impl Default for Sorting { + fn default() -> Self { + Sorting::None + } +} + +#[derive(Clone, Debug, Default, Eq, Hash, PartialEq)] +pub struct UserIdentifier { + /// For MySQL database backends as the primary key + pub legacy_id: u64, + /// For NoSQL database backends that require randomly distributed primary keys + pub fxa_uid: String, + pub fxa_kid: String, +} + +impl UserIdentifier { + /// Create a new legacy id user identifier + pub fn new_legacy(user_id: u64) -> Self { + Self { + legacy_id: user_id, + ..Default::default() + } + } +} + +impl From for UserIdentifier { + fn from(val: u32) -> Self { + Self { + legacy_id: val.into(), + ..Default::default() + } + } +} diff --git a/syncserver-db-common/src/params.rs b/syncstorage-db-common/src/params.rs similarity index 100% rename from syncserver-db-common/src/params.rs rename to syncstorage-db-common/src/params.rs diff --git a/syncserver-db-common/src/results.rs b/syncstorage-db-common/src/results.rs similarity index 100% rename from syncserver-db-common/src/results.rs rename to syncstorage-db-common/src/results.rs diff --git a/syncserver-db-common/src/util.rs b/syncstorage-db-common/src/util.rs similarity index 85% rename from syncserver-db-common/src/util.rs rename to syncstorage-db-common/src/util.rs index 92da71be..1cee78d1 100644 --- a/syncserver-db-common/src/util.rs +++ b/syncstorage-db-common/src/util.rs @@ -12,7 +12,7 @@ use diesel::{ }; use serde::{ser, Deserialize, Deserializer, Serialize, Serializer}; -use super::error::{DbError, DbErrorKind}; +use super::error::SyncstorageDbError; /// Get the time since the UNIX epoch in milliseconds fn ms_since_epoch() -> i64 { @@ -53,15 +53,17 @@ impl SyncTimestamp { } /// Create a `SyncTimestamp` from an i64 - pub fn from_i64(val: i64) -> Result { + pub fn from_i64(val: i64) -> Result { if val < 0 { - return Err(DbErrorKind::Integrity("Invalid modified i64 (< 0)".to_owned()).into()); + return Err(SyncstorageDbError::internal( + "Invalid modified i64 (< 0)".to_owned(), + )); } Ok(SyncTimestamp::from_milliseconds(val as u64)) } /// Exposed separately for db tests - pub fn _from_i64(val: i64) -> Result { + pub fn _from_i64(val: i64) -> Result { SyncTimestamp::from_i64(val) } @@ -78,17 +80,19 @@ impl SyncTimestamp { /// Create a `SyncTimestamp` from an RFC 3339 and ISO 8601 date and time /// string such as 1996-12-19T16:39:57-08:00 - pub fn from_rfc3339(val: &str) -> Result { + pub fn from_rfc3339(val: &str) -> Result { let dt = DateTime::parse_from_rfc3339(val) - .map_err(|e| DbErrorKind::Integrity(format!("Invalid TIMESTAMP {}", e)))?; + .map_err(|e| SyncstorageDbError::internal(format!("Invalid TIMESTAMP {}", e)))?; Self::from_datetime(dt) } /// Create a `SyncTimestamp` from a chrono DateTime - fn from_datetime(val: DateTime) -> Result { + fn from_datetime(val: DateTime) -> Result { let millis = val.timestamp_millis(); if millis < 0 { - return Err(DbErrorKind::Integrity("Invalid DateTime (< 0)".to_owned()).into()); + return Err(SyncstorageDbError::internal( + "Invalid DateTime (< 0)".to_owned(), + )); } Ok(SyncTimestamp::from_milliseconds(millis as u64)) } @@ -105,7 +109,7 @@ impl SyncTimestamp { /// Return the timestamp as an RFC 3339 and ISO 8601 date and time string such as /// 1996-12-19T16:39:57-08:00 - pub fn as_rfc3339(self) -> Result { + pub fn as_rfc3339(self) -> Result { to_rfc3339(self.as_i64()) } } @@ -167,10 +171,10 @@ where /// Render a timestamp (as an i64 milliseconds since epoch) as an RFC 3339 and ISO 8601 /// date and time string such as 1996-12-19T16:39:57-08:00 -pub fn to_rfc3339(val: i64) -> Result { +pub fn to_rfc3339(val: i64) -> Result { let secs = val / 1000; let nsecs = ((val % 1000) * 1_000_000).try_into().map_err(|e| { - DbError::internal(&format!("Invalid timestamp (nanoseconds) {}: {}", val, e)) + SyncstorageDbError::internal(format!("Invalid timestamp (nanoseconds) {}: {}", val, e)) })?; Ok(Utc .timestamp(secs, nsecs) diff --git a/syncstorage-db/Cargo.toml b/syncstorage-db/Cargo.toml new file mode 100644 index 00000000..c691781e --- /dev/null +++ b/syncstorage-db/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "syncstorage-db" +version = "0.13.6" +edition = "2021" + +[dependencies] +async-trait = "0.1.40" +cadence = "0.26" +env_logger = "0.9" +futures = { version = "0.3", features = ["compat"] } +hostname = "0.3.1" +lazy_static = "1.4.0" +log = { version = "0.4", features = [ + "max_level_debug", + "release_max_level_info", +] } +rand = "0.8" +slog-scope = "4.3" +syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } +syncserver-settings = { path = "../syncserver-settings" } +syncstorage-db-common = { path = "../syncstorage-db-common" } +syncstorage-mysql = { path = "../syncstorage-mysql", optional = true } +syncstorage-settings = { path = "../syncstorage-settings" } +syncstorage-spanner = { path = "../syncstorage-spanner", optional = true } +# pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.) +tokio = { version = "0.2.4", features = ["macros", "sync"] } + +[features] +mysql = ['syncstorage-mysql'] +spanner = ['syncstorage-spanner'] diff --git a/syncstorage-db/src/lib.rs b/syncstorage-db/src/lib.rs new file mode 100644 index 00000000..f7007d1c --- /dev/null +++ b/syncstorage-db/src/lib.rs @@ -0,0 +1,77 @@ +//! Generic db abstration. + +#[cfg(test)] +#[macro_use] +extern crate slog_scope; + +pub mod mock; +#[cfg(test)] +mod tests; + +use std::time::Duration; + +use cadence::{Gauged, StatsdClient}; +use tokio::{self, time}; + +#[cfg(feature = "mysql")] +pub type DbPoolImpl = syncstorage_mysql::MysqlDbPool; +#[cfg(feature = "mysql")] +pub use syncstorage_mysql::DbError; +#[cfg(feature = "mysql")] +pub type DbImpl = syncstorage_mysql::MysqlDb; + +#[cfg(feature = "spanner")] +pub type DbPoolImpl = syncstorage_spanner::SpannerDbPool; +#[cfg(feature = "spanner")] +pub use syncstorage_spanner::DbError; +#[cfg(feature = "spanner")] +pub type DbImpl = syncstorage_spanner::SpannerDb; + +pub use syncserver_db_common::{GetPoolState, PoolState}; +pub use syncstorage_db_common::error::DbErrorIntrospect; + +pub use syncstorage_db_common::{ + params, results, + util::{to_rfc3339, SyncTimestamp}, + Db, DbPool, Sorting, UserIdentifier, +}; + +#[cfg(all(feature = "mysql", feature = "spanner"))] +compile_error!("only one of the \"mysql\" and \"spanner\" features can be enabled at a time"); + +#[cfg(not(any(feature = "mysql", feature = "spanner")))] +compile_error!("exactly one of the \"mysql\" and \"spanner\" features must be enabled"); + +/// Emit DbPool metrics periodically +pub fn spawn_pool_periodic_reporter( + interval: Duration, + metrics: StatsdClient, + pool: T, +) -> Result<(), DbError> { + let hostname = hostname::get() + .expect("Couldn't get hostname") + .into_string() + .expect("Couldn't get hostname"); + tokio::spawn(async move { + loop { + let PoolState { + connections, + idle_connections, + } = pool.state(); + metrics + .gauge_with_tags( + "storage.pool.connections.active", + (connections - idle_connections) as u64, + ) + .with_tag("hostname", &hostname) + .send(); + metrics + .gauge_with_tags("storage.pool.connections.idle", idle_connections as u64) + .with_tag("hostname", &hostname) + .send(); + time::delay_for(interval).await; + } + }); + + Ok(()) +} diff --git a/syncserver/src/db/mock.rs b/syncstorage-db/src/mock.rs similarity index 86% rename from syncserver/src/db/mock.rs rename to syncstorage-db/src/mock.rs index af13bfb7..2f5575b6 100644 --- a/syncserver/src/db/mock.rs +++ b/syncstorage-db/src/mock.rs @@ -2,10 +2,12 @@ #![allow(clippy::new_without_default)] use async_trait::async_trait; use futures::future; -use syncserver_db_common::{ - error::DbError, params, results, util::SyncTimestamp, Db, DbFuture, DbPool, GetPoolState, - PoolState, -}; +use syncserver_db_common::{GetPoolState, PoolState}; +use syncstorage_db_common::{params, results, util::SyncTimestamp, Db, DbPool}; + +use crate::DbError; + +type DbFuture<'a, T> = syncserver_db_common::DbFuture<'a, T, DbError>; #[derive(Clone, Debug)] pub struct MockDbPool; @@ -18,15 +20,17 @@ impl MockDbPool { #[async_trait] impl DbPool for MockDbPool { - async fn get<'a>(&'a self) -> Result>, DbError> { - Ok(Box::new(MockDb::new()) as Box>) + type Error = DbError; + + async fn get(&self) -> Result>, Self::Error> { + Ok(Box::new(MockDb::new())) } fn validate_batch_id(&self, _: params::ValidateBatchId) -> Result<(), DbError> { Ok(()) } - fn box_clone(&self) -> Box { + fn box_clone(&self) -> Box> { Box::new(self.clone()) } } @@ -58,7 +62,9 @@ macro_rules! mock_db_method { }; } -impl<'a> Db<'a> for MockDb { +impl Db for MockDb { + type Error = DbError; + fn commit(&self) -> DbFuture<'_, ()> { Box::pin(future::ok(())) } @@ -71,7 +77,7 @@ impl<'a> Db<'a> for MockDb { Box::pin(future::ok(())) } - fn box_clone(&self) -> Box> { + fn box_clone(&self) -> Box> { Box::new(self.clone()) } diff --git a/syncserver/src/db/tests/batch.rs b/syncstorage-db/src/tests/batch.rs similarity index 90% rename from syncserver/src/db/tests/batch.rs rename to syncstorage-db/src/tests/batch.rs index 8efa1613..ecc972fd 100644 --- a/syncserver/src/db/tests/batch.rs +++ b/syncstorage-db/src/tests/batch.rs @@ -1,8 +1,11 @@ use log::debug; -use syncserver_db_common::{params, results, util::SyncTimestamp, BATCH_LIFETIME}; use syncserver_settings::Settings; +use syncstorage_db_common::{ + error::DbErrorIntrospect, params, results, util::SyncTimestamp, BATCH_LIFETIME, +}; -use super::support::{db_pool, gbso, hid, pbso, postbso, test_db, Result}; +use super::support::{db_pool, gbso, hid, pbso, postbso, test_db}; +use crate::DbError; fn cb(user_id: u32, coll: &str, bsos: Vec) -> params::CreateBatch { params::CreateBatch { @@ -43,9 +46,9 @@ fn gb(user_id: u32, coll: &str, id: String) -> params::GetBatch { } #[tokio::test] -async fn create_delete() -> Result<()> { +async fn create_delete() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 1; let coll = "clients"; @@ -66,9 +69,9 @@ async fn create_delete() -> Result<()> { } #[tokio::test] -async fn expiry() -> Result<()> { +async fn expiry() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 1; let coll = "clients"; @@ -90,9 +93,9 @@ async fn expiry() -> Result<()> { } #[tokio::test] -async fn update() -> Result<()> { +async fn update() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 1; let coll = "clients"; @@ -114,9 +117,9 @@ async fn update() -> Result<()> { } #[tokio::test] -async fn append_commit() -> Result<()> { +async fn append_commit() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 1; let coll = "clients"; @@ -157,7 +160,7 @@ async fn append_commit() -> Result<()> { } #[tokio::test] -async fn quota_test_create_batch() -> Result<()> { +async fn quota_test_create_batch() -> Result<(), DbError> { let mut settings = Settings::test_settings().syncstorage; if !settings.enable_quota { @@ -169,7 +172,7 @@ async fn quota_test_create_batch() -> Result<()> { settings.limits.max_quota_limit = limit; let pool = db_pool(Some(settings.clone())).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 1; let coll = "clients"; @@ -199,7 +202,7 @@ async fn quota_test_create_batch() -> Result<()> { } #[tokio::test] -async fn quota_test_append_batch() -> Result<()> { +async fn quota_test_append_batch() -> Result<(), DbError> { let mut settings = Settings::test_settings().syncstorage; if !settings.enable_quota { @@ -211,7 +214,7 @@ async fn quota_test_append_batch() -> Result<()> { settings.limits.max_quota_limit = limit; let pool = db_pool(Some(settings.clone())).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 1; let coll = "clients"; @@ -244,10 +247,10 @@ async fn quota_test_append_batch() -> Result<()> { } #[tokio::test] -async fn test_append_async_w_null() -> Result<()> { +async fn test_append_async_w_null() -> Result<(), DbError> { let settings = Settings::test_settings().syncstorage; let pool = db_pool(Some(settings)).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; // Remember: TTL is seconds to live, not an expiry date let ttl_0 = 86_400; let ttl_1 = 86_400; diff --git a/syncserver/src/db/tests/db.rs b/syncstorage-db/src/tests/db.rs similarity index 89% rename from syncserver/src/db/tests/db.rs rename to syncstorage-db/src/tests/db.rs index 346501e1..e1a00a84 100644 --- a/syncserver/src/db/tests/db.rs +++ b/syncstorage-db/src/tests/db.rs @@ -3,10 +3,13 @@ use std::collections::HashMap; use lazy_static::lazy_static; use rand::{distributions::Alphanumeric, thread_rng, Rng}; -use syncserver_db_common::{params, util::SyncTimestamp, Sorting, UserIdentifier, DEFAULT_BSO_TTL}; use syncserver_settings::Settings; +use syncstorage_db_common::{ + error::DbErrorIntrospect, params, util::SyncTimestamp, Sorting, UserIdentifier, DEFAULT_BSO_TTL, +}; -use super::support::{db_pool, dbso, dbsos, gbso, gbsos, hid, pbso, postbso, test_db, Result}; +use super::support::{db_pool, dbso, dbsos, gbso, gbsos, hid, pbso, postbso, test_db}; +use crate::DbError; // distant future (year 2099) timestamp for tests const MAX_TIMESTAMP: u64 = 4_070_937_600_000; @@ -16,9 +19,9 @@ lazy_static! { } #[tokio::test] -async fn bso_successfully_updates_single_values() -> Result<()> { +async fn bso_successfully_updates_single_values() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -57,9 +60,9 @@ async fn bso_successfully_updates_single_values() -> Result<()> { } #[tokio::test] -async fn bso_modified_not_changed_on_ttl_touch() -> Result<()> { +async fn bso_modified_not_changed_on_ttl_touch() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -80,9 +83,9 @@ async fn bso_modified_not_changed_on_ttl_touch() -> Result<()> { } #[tokio::test] -async fn put_bso_updates() -> Result<()> { +async fn put_bso_updates() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -103,9 +106,9 @@ async fn put_bso_updates() -> Result<()> { } #[tokio::test] -async fn get_bsos_limit_offset() -> Result<()> { +async fn get_bsos_limit_offset() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -224,9 +227,9 @@ async fn get_bsos_limit_offset() -> Result<()> { } #[tokio::test] -async fn get_bsos_newer() -> Result<()> { +async fn get_bsos_newer() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -309,9 +312,9 @@ async fn get_bsos_newer() -> Result<()> { } #[tokio::test] -async fn get_bsos_sort() -> Result<()> { +async fn get_bsos_sort() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -382,9 +385,9 @@ async fn get_bsos_sort() -> Result<()> { } #[tokio::test] -async fn delete_bsos_in_correct_collection() -> Result<()> { +async fn delete_bsos_in_correct_collection() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let payload = "data"; @@ -399,9 +402,9 @@ async fn delete_bsos_in_correct_collection() -> Result<()> { } #[tokio::test] -async fn get_storage_timestamp() -> Result<()> { +async fn get_storage_timestamp() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; db.create_collection("NewCollection1".to_owned()).await?; @@ -422,17 +425,17 @@ async fn get_storage_timestamp() -> Result<()> { } #[tokio::test] -async fn get_collection_id() -> Result<()> { +async fn get_collection_id() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; db.get_collection_id("bookmarks".to_owned()).await?; Ok(()) } #[tokio::test] -async fn create_collection() -> Result<()> { +async fn create_collection() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let name = "NewCollection"; let cid = db.create_collection(name.to_owned()).await?; @@ -443,9 +446,9 @@ async fn create_collection() -> Result<()> { } #[tokio::test] -async fn update_collection() -> Result<()> { +async fn update_collection() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let collection = "test".to_owned(); let cid = db.create_collection(collection.clone()).await?; @@ -459,9 +462,9 @@ async fn update_collection() -> Result<()> { } #[tokio::test] -async fn delete_collection() -> Result<()> { +async fn delete_collection() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "NewCollection"; @@ -495,9 +498,9 @@ async fn delete_collection() -> Result<()> { } #[tokio::test] -async fn delete_collection_tombstone() -> Result<()> { +async fn delete_collection_tombstone() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "test"; @@ -555,9 +558,9 @@ async fn delete_collection_tombstone() -> Result<()> { } #[tokio::test] -async fn get_collection_timestamps() -> Result<()> { +async fn get_collection_timestamps() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "test".to_owned(); @@ -583,9 +586,9 @@ async fn get_collection_timestamps() -> Result<()> { } #[tokio::test] -async fn get_collection_timestamps_tombstone() -> Result<()> { +async fn get_collection_timestamps_tombstone() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "test".to_owned(); @@ -608,9 +611,9 @@ async fn get_collection_timestamps_tombstone() -> Result<()> { } #[tokio::test] -async fn get_collection_usage() -> Result<()> { +async fn get_collection_usage() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 5; let mut expected = HashMap::new(); @@ -625,7 +628,7 @@ async fn get_collection_usage() -> Result<()> { db.put_bso(pbso( uid, coll, - &format!("b{}", i as i32), + &format!("b{}", i), Some(&String::from_utf8_lossy(&payload)), None, None, @@ -660,7 +663,7 @@ async fn get_collection_usage() -> Result<()> { } #[tokio::test] -async fn test_quota() -> Result<()> { +async fn test_quota() -> Result<(), DbError> { let settings = Settings::test_settings(); if !settings.syncstorage.enable_quota { @@ -669,7 +672,7 @@ async fn test_quota() -> Result<()> { } let pool = db_pool(None).await?; - let mut db = test_db(pool.as_ref()).await?; + let mut db = test_db(pool).await?; let uid = 5; let coll = "bookmarks"; @@ -702,9 +705,9 @@ async fn test_quota() -> Result<()> { } #[tokio::test] -async fn get_collection_counts() -> Result<()> { +async fn get_collection_counts() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 4; let mut expected = HashMap::new(); @@ -725,9 +728,9 @@ async fn get_collection_counts() -> Result<()> { } #[tokio::test] -async fn put_bso() -> Result<()> { +async fn put_bso() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "NewCollection"; @@ -765,9 +768,9 @@ async fn put_bso() -> Result<()> { } #[tokio::test] -async fn post_bsos() -> Result<()> { +async fn post_bsos() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "NewCollection"; @@ -836,9 +839,9 @@ async fn post_bsos() -> Result<()> { } #[tokio::test] -async fn get_bso() -> Result<()> { +async fn get_bso() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -857,9 +860,9 @@ async fn get_bso() -> Result<()> { } #[tokio::test] -async fn get_bsos() -> Result<()> { +async fn get_bsos() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = 2; let coll = "clients"; @@ -928,9 +931,9 @@ async fn get_bsos() -> Result<()> { } #[tokio::test] -async fn get_bso_timestamp() -> Result<()> { +async fn get_bso_timestamp() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -949,9 +952,9 @@ async fn get_bso_timestamp() -> Result<()> { } #[tokio::test] -async fn delete_bso() -> Result<()> { +async fn delete_bso() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -965,9 +968,9 @@ async fn delete_bso() -> Result<()> { } #[tokio::test] -async fn delete_bsos() -> Result<()> { +async fn delete_bsos() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -1000,31 +1003,31 @@ async fn delete_bsos() -> Result<()> { /* #[tokio::test] -async fn usage_stats() -> Result<()> { +async fn usage_stats() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; Ok(()) } #[tokio::test] -async fn purge_expired() -> Result<()> { +async fn purge_expired() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; Ok(()) } #[tokio::test] -async fn optimize() -> Result<()> { +async fn optimize() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; Ok(()) } */ #[tokio::test] -async fn delete_storage() -> Result<()> { +async fn delete_storage() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let bid = "test"; @@ -1048,9 +1051,9 @@ async fn delete_storage() -> Result<()> { } #[tokio::test] -async fn collection_cache() -> Result<()> { +async fn collection_cache() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "test"; @@ -1069,9 +1072,9 @@ async fn collection_cache() -> Result<()> { } #[tokio::test] -async fn lock_for_read() -> Result<()> { +async fn lock_for_read() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -1087,9 +1090,9 @@ async fn lock_for_read() -> Result<()> { } #[tokio::test] -async fn lock_for_write() -> Result<()> { +async fn lock_for_write() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; let uid = *UID; let coll = "clients"; @@ -1105,9 +1108,9 @@ async fn lock_for_write() -> Result<()> { } #[tokio::test] -async fn heartbeat() -> Result<()> { +async fn heartbeat() -> Result<(), DbError> { let pool = db_pool(None).await?; - let db = test_db(pool.as_ref()).await?; + let db = test_db(pool).await?; assert!(db.check().await?); Ok(()) diff --git a/syncserver/src/db/tests/mod.rs b/syncstorage-db/src/tests/mod.rs similarity index 100% rename from syncserver/src/db/tests/mod.rs rename to syncstorage-db/src/tests/mod.rs diff --git a/syncserver/src/db/tests/support.rs b/syncstorage-db/src/tests/support.rs similarity index 86% rename from syncserver/src/db/tests/support.rs rename to syncstorage-db/src/tests/support.rs index 9590d7fa..a952799e 100644 --- a/syncserver/src/db/tests/support.rs +++ b/syncstorage-db/src/tests/support.rs @@ -1,17 +1,14 @@ use std::{str::FromStr, sync::Arc}; -use syncserver_db_common::{params, util::SyncTimestamp, Db, Sorting, UserIdentifier}; +use syncserver_common::{BlockingThreadpool, Metrics}; use syncserver_settings::Settings as SyncserverSettings; +use syncstorage_db_common::{params, util::SyncTimestamp, Db, DbPool, Sorting, UserIdentifier}; use syncstorage_settings::Settings as SyncstorageSettings; -use crate::db::DbPool; -use crate::error::ApiResult; -use crate::{db::pool_from_settings, db::BlockingThreadpool, error::ApiError, server::metrics}; - -pub type Result = std::result::Result; +use crate::{DbError, DbPoolImpl}; #[cfg(test)] -pub async fn db_pool(settings: Option) -> Result> { +pub async fn db_pool(settings: Option) -> Result { let _ = env_logger::try_init(); // The default for SYNC_SYNCSTORAGE__DATABASE_USE_TEST_TRANSACTIONS is // false, but we want the mysql default to be true, so let's check @@ -25,13 +22,12 @@ pub async fn db_pool(settings: Option) -> Result ApiResult>> { +pub async fn test_db(pool: DbPoolImpl) -> Result>, DbError> { let db = pool.get().await?; // Spanner won't have a timestamp until lock_for_xxx are called: fill one // in for it diff --git a/syncstorage-mysql/Cargo.toml b/syncstorage-mysql/Cargo.toml new file mode 100644 index 00000000..cf0f4a05 --- /dev/null +++ b/syncstorage-mysql/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "syncstorage-mysql" +version = "0.13.6" +edition = "2021" + +[dependencies] +async-trait = "0.1.40" +backtrace = "0.3.61" +base64 = "0.13" +diesel = { version = "1.4", features = ["mysql", "r2d2"] } +diesel_logger = "0.1.1" +diesel_migrations = { version = "1.4.0", features = ["mysql"] } +futures = { version = "0.3", features = ["compat"] } +http = "0.2.5" +slog-scope = "4.3" +syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } +syncstorage-db-common = { path = "../syncstorage-db-common" } +syncstorage-settings = { path = "../syncstorage-settings" } +thiserror = "1.0.26" +url = "2.1" + +[dev-dependencies] +syncserver-settings = { path = "../syncserver-settings" } +env_logger = "0.9" diff --git a/migrations/2018-08-28-010336_init/down.sql b/syncstorage-mysql/migrations/2018-08-28-010336_init/down.sql similarity index 100% rename from migrations/2018-08-28-010336_init/down.sql rename to syncstorage-mysql/migrations/2018-08-28-010336_init/down.sql diff --git a/migrations/2018-08-28-010336_init/up.sql b/syncstorage-mysql/migrations/2018-08-28-010336_init/up.sql similarity index 100% rename from migrations/2018-08-28-010336_init/up.sql rename to syncstorage-mysql/migrations/2018-08-28-010336_init/up.sql diff --git a/migrations/2019-09-11-164500/down.sql b/syncstorage-mysql/migrations/2019-09-11-164500/down.sql similarity index 100% rename from migrations/2019-09-11-164500/down.sql rename to syncstorage-mysql/migrations/2019-09-11-164500/down.sql diff --git a/migrations/2019-09-11-164500/up.sql b/syncstorage-mysql/migrations/2019-09-11-164500/up.sql similarity index 100% rename from migrations/2019-09-11-164500/up.sql rename to syncstorage-mysql/migrations/2019-09-11-164500/up.sql diff --git a/migrations/2019-09-25-174347_min_collection_id/down.sql b/syncstorage-mysql/migrations/2019-09-25-174347_min_collection_id/down.sql similarity index 100% rename from migrations/2019-09-25-174347_min_collection_id/down.sql rename to syncstorage-mysql/migrations/2019-09-25-174347_min_collection_id/down.sql diff --git a/migrations/2019-09-25-174347_min_collection_id/up.sql b/syncstorage-mysql/migrations/2019-09-25-174347_min_collection_id/up.sql similarity index 100% rename from migrations/2019-09-25-174347_min_collection_id/up.sql rename to syncstorage-mysql/migrations/2019-09-25-174347_min_collection_id/up.sql diff --git a/migrations/2020-04-03-102015_change_userid/down.sql b/syncstorage-mysql/migrations/2020-04-03-102015_change_userid/down.sql similarity index 100% rename from migrations/2020-04-03-102015_change_userid/down.sql rename to syncstorage-mysql/migrations/2020-04-03-102015_change_userid/down.sql diff --git a/migrations/2020-04-03-102015_change_userid/up.sql b/syncstorage-mysql/migrations/2020-04-03-102015_change_userid/up.sql similarity index 100% rename from migrations/2020-04-03-102015_change_userid/up.sql rename to syncstorage-mysql/migrations/2020-04-03-102015_change_userid/up.sql diff --git a/migrations/2020-06-12-231034_new_batch/down.sql b/syncstorage-mysql/migrations/2020-06-12-231034_new_batch/down.sql similarity index 100% rename from migrations/2020-06-12-231034_new_batch/down.sql rename to syncstorage-mysql/migrations/2020-06-12-231034_new_batch/down.sql diff --git a/migrations/2020-06-12-231034_new_batch/up.sql b/syncstorage-mysql/migrations/2020-06-12-231034_new_batch/up.sql similarity index 100% rename from migrations/2020-06-12-231034_new_batch/up.sql rename to syncstorage-mysql/migrations/2020-06-12-231034_new_batch/up.sql diff --git a/migrations/2020-08-24-091401_add_quota/down.sql b/syncstorage-mysql/migrations/2020-08-24-091401_add_quota/down.sql similarity index 100% rename from migrations/2020-08-24-091401_add_quota/down.sql rename to syncstorage-mysql/migrations/2020-08-24-091401_add_quota/down.sql diff --git a/migrations/2020-08-24-091401_add_quota/up.sql b/syncstorage-mysql/migrations/2020-08-24-091401_add_quota/up.sql similarity index 100% rename from migrations/2020-08-24-091401_add_quota/up.sql rename to syncstorage-mysql/migrations/2020-08-24-091401_add_quota/up.sql diff --git a/syncserver/src/db/mysql/batch.rs b/syncstorage-mysql/src/batch.rs similarity index 89% rename from syncserver/src/db/mysql/batch.rs rename to syncstorage-mysql/src/batch.rs index 4616c9b7..bde50c3e 100644 --- a/syncserver/src/db/mysql/batch.rs +++ b/syncstorage-mysql/src/batch.rs @@ -9,19 +9,18 @@ use diesel::{ sql_types::{BigInt, Integer}, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, }; -use syncserver_db_common::{ - error::{DbError, DbErrorKind}, - params, results, UserIdentifier, BATCH_LIFETIME, -}; +use syncstorage_db_common::{params, results, UserIdentifier, BATCH_LIFETIME}; use super::{ - models::{MysqlDb, Result}, + error::DbError, + models::MysqlDb, schema::{batch_upload_items, batch_uploads}, + DbResult, }; const MAXTTL: i32 = 2_100_000_000; -pub fn create(db: &MysqlDb, params: params::CreateBatch) -> Result { +pub fn create(db: &MysqlDb, params: params::CreateBatch) -> DbResult { let user_id = params.user_id.legacy_id as i64; let collection_id = db.get_collection_id(¶ms.collection)?; // Careful, there's some weirdness here! @@ -47,7 +46,7 @@ pub fn create(db: &MysqlDb, params: params::CreateBatch) -> Result DbError { match e { // The user tried to create two batches with the same timestamp - DieselError::DatabaseError(UniqueViolation, _) => DbErrorKind::Conflict.into(), + DieselError::DatabaseError(UniqueViolation, _) => DbError::conflict(), _ => e.into(), } })?; @@ -59,7 +58,7 @@ pub fn create(db: &MysqlDb, params: params::CreateBatch) -> Result Result { +pub fn validate(db: &MysqlDb, params: params::ValidateBatch) -> DbResult { let batch_id = decode_id(¶ms.id)?; // Avoid hitting the db for batches that are obviously too old. Recall // that the batchid is a millisecond timestamp. @@ -79,7 +78,7 @@ pub fn validate(db: &MysqlDb, params: params::ValidateBatch) -> Result { Ok(exists.is_some()) } -pub fn append(db: &MysqlDb, params: params::AppendToBatch) -> Result<()> { +pub fn append(db: &MysqlDb, params: params::AppendToBatch) -> DbResult<()> { let exists = validate( db, params::ValidateBatch { @@ -90,7 +89,7 @@ pub fn append(db: &MysqlDb, params: params::AppendToBatch) -> Result<()> { )?; if !exists { - Err(DbErrorKind::BatchNotFound)? + return Err(DbError::batch_not_found()); } let batch_id = decode_id(¶ms.batch.id)?; @@ -99,7 +98,7 @@ pub fn append(db: &MysqlDb, params: params::AppendToBatch) -> Result<()> { Ok(()) } -pub fn get(db: &MysqlDb, params: params::GetBatch) -> Result> { +pub fn get(db: &MysqlDb, params: params::GetBatch) -> DbResult> { let is_valid = validate( db, params::ValidateBatch { @@ -116,7 +115,7 @@ pub fn get(db: &MysqlDb, params: params::GetBatch) -> Result Result<()> { +pub fn delete(db: &MysqlDb, params: params::DeleteBatch) -> DbResult<()> { let batch_id = decode_id(¶ms.id)?; let user_id = params.user_id.legacy_id as i64; let collection_id = db.get_collection_id(¶ms.collection)?; @@ -133,19 +132,19 @@ pub fn delete(db: &MysqlDb, params: params::DeleteBatch) -> Result<()> { } /// Commits a batch to the bsos table, deleting the batch when succesful -pub fn commit(db: &MysqlDb, params: params::CommitBatch) -> Result { +pub fn commit(db: &MysqlDb, params: params::CommitBatch) -> DbResult { let batch_id = decode_id(¶ms.batch.id)?; let user_id = params.user_id.legacy_id as i64; let collection_id = db.get_collection_id(¶ms.collection)?; let timestamp = db.timestamp(); sql_query(include_str!("batch_commit.sql")) - .bind::(user_id as i64) + .bind::(user_id) .bind::(&collection_id) .bind::(&db.timestamp().as_i64()) .bind::(&db.timestamp().as_i64()) .bind::((MAXTTL as i64) * 1000) // XXX: .bind::(&batch_id) - .bind::(user_id as i64) + .bind::(user_id) .bind::(&db.timestamp().as_i64()) .bind::(&db.timestamp().as_i64()) .execute(&db.conn)?; @@ -169,7 +168,7 @@ pub fn do_append( user_id: UserIdentifier, _collection_id: i32, bsos: Vec, -) -> Result<()> { +) -> DbResult<()> { fn exist_idx(user_id: u64, batch_id: i64, bso_id: &str) -> String { // Construct something that matches the key for batch_upload_items format!( @@ -253,26 +252,25 @@ pub fn do_append( Ok(()) } -pub fn validate_batch_id(id: &str) -> Result<()> { +pub fn validate_batch_id(id: &str) -> DbResult<()> { decode_id(id).map(|_| ()) } fn encode_id(id: i64) -> String { - base64::encode(&id.to_string()) + base64::encode(id.to_string()) } -fn decode_id(id: &str) -> Result { +fn decode_id(id: &str) -> DbResult { let bytes = base64::decode(id).unwrap_or_else(|_| id.as_bytes().to_vec()); let decoded = std::str::from_utf8(&bytes).unwrap_or(id); decoded .parse::() - .map_err(|e| DbError::internal(&format!("Invalid batch_id: {}", e))) + .map_err(|e| DbError::internal(format!("Invalid batch_id: {}", e))) } -#[macro_export] macro_rules! batch_db_method { ($name:ident, $batch_name:ident, $type:ident) => { - pub fn $name(&self, params: params::$type) -> Result { + pub fn $name(&self, params: params::$type) -> DbResult { batch::$batch_name(self, params) } }; diff --git a/syncserver/src/db/mysql/batch_commit.sql b/syncstorage-mysql/src/batch_commit.sql similarity index 100% rename from syncserver/src/db/mysql/batch_commit.sql rename to syncstorage-mysql/src/batch_commit.sql diff --git a/syncserver/src/db/mysql/diesel_ext.rs b/syncstorage-mysql/src/diesel_ext.rs similarity index 100% rename from syncserver/src/db/mysql/diesel_ext.rs rename to syncstorage-mysql/src/diesel_ext.rs diff --git a/syncstorage-mysql/src/error.rs b/syncstorage-mysql/src/error.rs new file mode 100644 index 00000000..0c4c8b6a --- /dev/null +++ b/syncstorage-mysql/src/error.rs @@ -0,0 +1,144 @@ +use std::fmt; + +use backtrace::Backtrace; +use http::StatusCode; +use syncserver_common::{from_error, impl_fmt_display, InternalError, ReportableError}; +use syncserver_db_common::error::MysqlError; +use syncstorage_db_common::error::{DbErrorIntrospect, SyncstorageDbError}; +use thiserror::Error; + +/// An error type that represents any MySQL-related errors that may occur while processing a +/// syncstorage request. These errors may be application-specific or lower-level errors that arise +/// from the database backend. +#[derive(Debug)] +pub struct DbError { + kind: DbErrorKind, + pub status: StatusCode, + pub backtrace: Box, +} + +impl DbError { + pub fn batch_not_found() -> Self { + DbErrorKind::Common(SyncstorageDbError::batch_not_found()).into() + } + + pub fn bso_not_found() -> Self { + DbErrorKind::Common(SyncstorageDbError::bso_not_found()).into() + } + + pub fn collection_not_found() -> Self { + DbErrorKind::Common(SyncstorageDbError::collection_not_found()).into() + } + + pub fn conflict() -> Self { + DbErrorKind::Common(SyncstorageDbError::conflict()).into() + } + + pub fn internal(msg: String) -> Self { + DbErrorKind::Common(SyncstorageDbError::internal(msg)).into() + } + + pub fn quota() -> Self { + DbErrorKind::Common(SyncstorageDbError::quota()).into() + } +} + +#[derive(Debug, Error)] +enum DbErrorKind { + #[error("{}", _0)] + Common(SyncstorageDbError), + + #[error("{}", _0)] + Mysql(MysqlError), +} + +impl From for DbError { + fn from(kind: DbErrorKind) -> Self { + match &kind { + DbErrorKind::Common(dbe) => Self { + status: dbe.status, + backtrace: Box::new(dbe.backtrace.clone()), + kind, + }, + _ => Self { + kind, + status: StatusCode::INTERNAL_SERVER_ERROR, + backtrace: Box::new(Backtrace::new()), + }, + } + } +} + +impl DbErrorIntrospect for DbError { + fn is_batch_not_found(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_batch_not_found()) + } + + fn is_bso_not_found(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_bso_not_found()) + } + + fn is_collection_not_found(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_collection_not_found()) + } + + fn is_conflict(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_conflict()) + } + + fn is_quota(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_quota()) + } +} + +impl ReportableError for DbError { + fn is_sentry_event(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_sentry_event()) + } + + fn metric_label(&self) -> Option { + if let DbErrorKind::Common(e) = &self.kind { + e.metric_label() + } else { + None + } + } + + fn error_backtrace(&self) -> String { + format!("{:#?}", self.backtrace) + } +} + +impl InternalError for DbError { + fn internal_error(message: String) -> Self { + DbErrorKind::Common(SyncstorageDbError::internal(message)).into() + } +} + +impl_fmt_display!(DbError, DbErrorKind); + +from_error!(SyncstorageDbError, DbError, DbErrorKind::Common); +from_error!( + diesel::result::Error, + DbError, + |error: diesel::result::Error| DbError::from(DbErrorKind::Mysql(MysqlError::from(error))) +); +from_error!( + diesel::result::ConnectionError, + DbError, + |error: diesel::result::ConnectionError| DbError::from(DbErrorKind::Mysql(MysqlError::from( + error + ))) +); +from_error!( + diesel::r2d2::PoolError, + DbError, + |error: diesel::r2d2::PoolError| DbError::from(DbErrorKind::Mysql(MysqlError::from(error))) +); +from_error!( + diesel_migrations::RunMigrationsError, + DbError, + |error: diesel_migrations::RunMigrationsError| DbError::from(DbErrorKind::Mysql( + MysqlError::from(error) + )) +); diff --git a/syncstorage-mysql/src/lib.rs b/syncstorage-mysql/src/lib.rs new file mode 100644 index 00000000..4a933900 --- /dev/null +++ b/syncstorage-mysql/src/lib.rs @@ -0,0 +1,22 @@ +#[macro_use] +extern crate diesel; +#[macro_use] +extern crate diesel_migrations; +#[macro_use] +extern crate slog_scope; + +#[macro_use] +mod batch; +mod diesel_ext; +mod error; +mod models; +mod pool; +mod schema; +#[cfg(test)] +mod test; + +pub use error::DbError; +pub use models::MysqlDb; +pub use pool::MysqlDbPool; + +pub(crate) type DbResult = Result; diff --git a/syncserver/src/db/mysql/models.rs b/syncstorage-mysql/src/models.rs similarity index 86% rename from syncserver/src/db/mysql/models.rs rename to syncstorage-mysql/src/models.rs index 8b7fb82f..a5cb255c 100644 --- a/syncserver/src/db/mysql/models.rs +++ b/syncstorage-mysql/src/models.rs @@ -13,44 +13,43 @@ use diesel::{ sql_types::{BigInt, Integer, Nullable, Text}, Connection, ExpressionMethods, GroupByDsl, OptionalExtension, QueryDsl, RunQueryDsl, }; -#[cfg(test)] +#[cfg(debug_assertions)] use diesel_logger::LoggingConnection; -use syncserver_db_common::{ - error::{DbError, DbErrorKind}, - params, results, - util::SyncTimestamp, - Db, DbFuture, Sorting, UserIdentifier, DEFAULT_BSO_TTL, +use syncserver_common::{BlockingThreadpool, Metrics}; +use syncserver_db_common::{sync_db_method, DbFuture}; +use syncstorage_db_common::{ + error::DbErrorIntrospect, params, results, util::SyncTimestamp, Db, Sorting, UserIdentifier, + DEFAULT_BSO_TTL, }; use syncstorage_settings::{Quota, DEFAULT_MAX_TOTAL_RECORDS}; use super::{ batch, diesel_ext::LockInShareModeDsl, + error::DbError, pool::CollectionCache, schema::{bso, collections, user_collections}, + DbResult, }; -use crate::db::BlockingThreadpool; -use crate::server::metrics::Metrics; -pub type Result = std::result::Result; type Conn = PooledConnection>; // this is the max number of records we will return. -pub static DEFAULT_LIMIT: u32 = DEFAULT_MAX_TOTAL_RECORDS; +static DEFAULT_LIMIT: u32 = DEFAULT_MAX_TOTAL_RECORDS; -pub const TOMBSTONE: i32 = 0; +const TOMBSTONE: i32 = 0; /// SQL Variable remapping /// These names are the legacy values mapped to the new names. -pub const COLLECTION_ID: &str = "collection"; -pub const USER_ID: &str = "userid"; -pub const MODIFIED: &str = "modified"; -pub const EXPIRY: &str = "ttl"; -pub const LAST_MODIFIED: &str = "last_modified"; -pub const COUNT: &str = "count"; -pub const TOTAL_BYTES: &str = "total_bytes"; +const COLLECTION_ID: &str = "collection"; +const USER_ID: &str = "userid"; +const MODIFIED: &str = "modified"; +const EXPIRY: &str = "ttl"; +const LAST_MODIFIED: &str = "last_modified"; +const COUNT: &str = "count"; +const TOTAL_BYTES: &str = "total_bytes"; #[derive(Debug)] -pub enum CollectionLock { +enum CollectionLock { Read, Write, } @@ -71,8 +70,8 @@ struct MysqlDbSession { #[derive(Clone, Debug)] pub struct MysqlDb { - /// Synchronous Diesel calls are executed in tokio::task::spawn_blocking to satisfy - /// the Db trait's asynchronous interface. + /// Synchronous Diesel calls are executed in web::block to satisfy the Db trait's asynchronous + /// interface. /// /// Arc provides a Clone impl utilized for safely moving to /// the thread pool but does not provide Send as the underlying db @@ -94,9 +93,9 @@ pub struct MysqlDb { unsafe impl Send for MysqlDb {} pub struct MysqlDbInner { - #[cfg(not(test))] + #[cfg(not(debug_assertions))] pub(super) conn: Conn, - #[cfg(test)] + #[cfg(debug_assertions)] pub(super) conn: LoggingConnection, // display SQL when RUST_LOG="diesel_logger=trace" session: RefCell, @@ -117,7 +116,7 @@ impl Deref for MysqlDb { } impl MysqlDb { - pub fn new( + pub(super) fn new( conn: Conn, coll_cache: Arc, metrics: &Metrics, @@ -125,9 +124,9 @@ impl MysqlDb { blocking_threadpool: Arc, ) -> Self { let inner = MysqlDbInner { - #[cfg(not(test))] + #[cfg(not(debug_assertions))] conn, - #[cfg(test)] + #[cfg(debug_assertions)] conn: LoggingConnection::new(conn), session: RefCell::new(Default::default()), }; @@ -149,7 +148,7 @@ impl MysqlDb { /// In theory it would be possible to use serializable transactions rather /// than explicit locking, but our ops team have expressed concerns about /// the efficiency of that approach at scale. - pub fn lock_for_read_sync(&self, params: params::LockCollection) -> Result<()> { + fn lock_for_read_sync(&self, params: params::LockCollection) -> DbResult<()> { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection).or_else(|e| { if e.is_collection_not_found() { @@ -196,7 +195,7 @@ impl MysqlDb { Ok(()) } - pub fn lock_for_write_sync(&self, params: params::LockCollection) -> Result<()> { + fn lock_for_write_sync(&self, params: params::LockCollection) -> DbResult<()> { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_or_create_collection_id(¶ms.collection)?; if let Some(CollectionLock::Read) = self @@ -205,7 +204,9 @@ impl MysqlDb { .coll_locks .get(&(user_id as u32, collection_id)) { - Err(DbError::internal("Can't escalate read-lock to write-lock"))? + return Err(DbError::internal( + "Can't escalate read-lock to write-lock".to_owned(), + )); } // Lock the db @@ -221,7 +222,7 @@ impl MysqlDb { let modified = SyncTimestamp::from_i64(modified)?; // Forbid the write if it would not properly incr the timestamp if modified >= self.timestamp() { - Err(DbErrorKind::Conflict)? + return Err(DbError::conflict()); } self.session .borrow_mut() @@ -235,7 +236,7 @@ impl MysqlDb { Ok(()) } - pub(super) fn begin(&self, for_write: bool) -> Result<()> { + pub(super) fn begin(&self, for_write: bool) -> DbResult<()> { self.conn .transaction_manager() .begin_transaction(&self.conn)?; @@ -246,11 +247,11 @@ impl MysqlDb { Ok(()) } - pub async fn begin_async(&self, for_write: bool) -> Result<()> { + async fn begin_async(&self, for_write: bool) -> DbResult<()> { self.begin(for_write) } - pub fn commit_sync(&self) -> Result<()> { + fn commit_sync(&self) -> DbResult<()> { if self.session.borrow().in_transaction { self.conn .transaction_manager() @@ -259,7 +260,7 @@ impl MysqlDb { Ok(()) } - pub fn rollback_sync(&self) -> Result<()> { + fn rollback_sync(&self) -> DbResult<()> { if self.session.borrow().in_transaction { self.conn .transaction_manager() @@ -268,7 +269,7 @@ impl MysqlDb { Ok(()) } - fn erect_tombstone(&self, user_id: i32) -> Result<()> { + fn erect_tombstone(&self, user_id: i32) -> DbResult<()> { sql_query(format!( r#"INSERT INTO user_collections ({user_id}, {collection_id}, {modified}) VALUES (?, ?, ?) @@ -285,7 +286,7 @@ impl MysqlDb { Ok(()) } - pub fn delete_storage_sync(&self, user_id: UserIdentifier) -> Result<()> { + fn delete_storage_sync(&self, user_id: UserIdentifier) -> DbResult<()> { let user_id = user_id.legacy_id as i64; // Delete user data. delete(bso::table) @@ -301,10 +302,7 @@ impl MysqlDb { // Deleting the collection should result in: // - collection does not appear in /info/collections // - X-Last-Modified timestamp at the storage level changing - pub fn delete_collection_sync( - &self, - params: params::DeleteCollection, - ) -> Result { + fn delete_collection_sync(&self, params: params::DeleteCollection) -> DbResult { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection)?; let mut count = delete(bso::table) @@ -316,14 +314,14 @@ impl MysqlDb { .filter(user_collections::collection_id.eq(&collection_id)) .execute(&self.conn)?; if count == 0 { - Err(DbErrorKind::CollectionNotFound)? + return Err(DbError::collection_not_found()); } else { self.erect_tombstone(user_id as i32)?; } self.get_storage_timestamp_sync(params.user_id) } - pub(super) fn get_or_create_collection_id(&self, name: &str) -> Result { + pub(super) fn get_or_create_collection_id(&self, name: &str) -> DbResult { if let Some(id) = self.coll_cache.get_id(name)? { return Ok(id); } @@ -346,7 +344,7 @@ impl MysqlDb { Ok(id) } - pub(super) fn get_collection_id(&self, name: &str) -> Result { + pub(super) fn get_collection_id(&self, name: &str) -> DbResult { if let Some(id) = self.coll_cache.get_id(name)? { return Ok(id); } @@ -359,7 +357,7 @@ impl MysqlDb { .bind::(name) .get_result::(&self.conn) .optional()? - .ok_or(DbErrorKind::CollectionNotFound)? + .ok_or_else(DbError::collection_not_found)? .id; if !self.session.borrow().in_write_transaction { self.coll_cache.put(id, name.to_owned())?; @@ -367,7 +365,7 @@ impl MysqlDb { Ok(id) } - fn _get_collection_name(&self, id: i32) -> Result { + fn _get_collection_name(&self, id: i32) -> DbResult { let name = if let Some(name) = self.coll_cache.get_name(id)? { name } else { @@ -379,13 +377,13 @@ impl MysqlDb { .bind::(&id) .get_result::(&self.conn) .optional()? - .ok_or(DbErrorKind::CollectionNotFound)? + .ok_or_else(DbError::collection_not_found)? .name }; Ok(name) } - pub fn put_bso_sync(&self, bso: params::PutBso) -> Result { + fn put_bso_sync(&self, bso: params::PutBso) -> DbResult { /* if bso.payload.is_none() && bso.sortindex.is_none() && bso.ttl.is_none() { // XXX: go returns an error here (ErrNothingToDo), and is treated @@ -403,12 +401,12 @@ impl MysqlDb { collection: bso.collection.clone(), collection_id, })?; - if usage.total_bytes >= self.quota.size as usize { + if usage.total_bytes >= self.quota.size { let mut tags = HashMap::default(); tags.insert("collection".to_owned(), bso.collection.clone()); self.metrics.incr_with_tags("storage.quota.at_limit", tags); if self.quota.enforced { - return Err(DbErrorKind::Quota.into()); + return Err(DbError::quota()); } else { warn!("Quota at limit for user's collection ({} bytes)", usage.total_bytes; "collection"=>bso.collection.clone()); } @@ -476,7 +474,7 @@ impl MysqlDb { }) } - pub fn get_bsos_sync(&self, params: params::GetBsos) -> Result { + fn get_bsos_sync(&self, params: params::GetBsos) -> DbResult { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection)?; let now = self.timestamp().as_i64(); @@ -489,7 +487,7 @@ impl MysqlDb { bso::expiry, )) .filter(bso::user_id.eq(user_id)) - .filter(bso::collection_id.eq(collection_id as i32)) // XXX: + .filter(bso::collection_id.eq(collection_id)) .filter(bso::expiry.gt(now)) .into_boxed(); @@ -566,13 +564,13 @@ impl MysqlDb { }) } - pub fn get_bso_ids_sync(&self, params: params::GetBsos) -> Result { + fn get_bso_ids_sync(&self, params: params::GetBsos) -> DbResult { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection)?; let mut query = bso::table .select(bso::id) .filter(bso::user_id.eq(user_id)) - .filter(bso::collection_id.eq(collection_id as i32)) // XXX: + .filter(bso::collection_id.eq(collection_id)) .filter(bso::expiry.gt(self.timestamp().as_i64())) .into_boxed(); @@ -629,7 +627,7 @@ impl MysqlDb { }) } - pub fn get_bso_sync(&self, params: params::GetBso) -> Result> { + fn get_bso_sync(&self, params: params::GetBso) -> DbResult> { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection)?; Ok(bso::table @@ -648,7 +646,7 @@ impl MysqlDb { .optional()?) } - pub fn delete_bso_sync(&self, params: params::DeleteBso) -> Result { + fn delete_bso_sync(&self, params: params::DeleteBso) -> DbResult { let user_id = params.user_id.legacy_id; let collection_id = self.get_collection_id(¶ms.collection)?; let affected_rows = delete(bso::table) @@ -658,12 +656,12 @@ impl MysqlDb { .filter(bso::expiry.gt(&self.timestamp().as_i64())) .execute(&self.conn)?; if affected_rows == 0 { - Err(DbErrorKind::BsoNotFound)? + return Err(DbError::bso_not_found()); } self.update_collection(user_id as u32, collection_id) } - pub fn delete_bsos_sync(&self, params: params::DeleteBsos) -> Result { + fn delete_bsos_sync(&self, params: params::DeleteBsos) -> DbResult { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection)?; delete(bso::table) @@ -674,7 +672,7 @@ impl MysqlDb { self.update_collection(user_id as u32, collection_id) } - pub fn post_bsos_sync(&self, input: params::PostBsos) -> Result { + fn post_bsos_sync(&self, input: params::PostBsos) -> DbResult { let collection_id = self.get_or_create_collection_id(&input.collection)?; let mut result = results::PostBsos { modified: self.timestamp(), @@ -707,20 +705,20 @@ impl MysqlDb { Ok(result) } - pub fn get_storage_timestamp_sync(&self, user_id: UserIdentifier) -> Result { + fn get_storage_timestamp_sync(&self, user_id: UserIdentifier) -> DbResult { let user_id = user_id.legacy_id as i64; let modified = user_collections::table .select(max(user_collections::modified)) .filter(user_collections::user_id.eq(user_id)) .first::>(&self.conn)? .unwrap_or_default(); - SyncTimestamp::from_i64(modified) + SyncTimestamp::from_i64(modified).map_err(Into::into) } - pub fn get_collection_timestamp_sync( + fn get_collection_timestamp_sync( &self, params: params::GetCollectionTimestamp, - ) -> Result { + ) -> DbResult { let user_id = params.user_id.legacy_id as u32; let collection_id = self.get_collection_id(¶ms.collection)?; if let Some(modified) = self @@ -737,10 +735,10 @@ impl MysqlDb { .filter(user_collections::collection_id.eq(collection_id)) .first(&self.conn) .optional()? - .ok_or_else(|| DbErrorKind::CollectionNotFound.into()) + .ok_or_else(DbError::collection_not_found) } - pub fn get_bso_timestamp_sync(&self, params: params::GetBsoTimestamp) -> Result { + fn get_bso_timestamp_sync(&self, params: params::GetBsoTimestamp) -> DbResult { let user_id = params.user_id.legacy_id as i64; let collection_id = self.get_collection_id(¶ms.collection)?; let modified = bso::table @@ -751,13 +749,13 @@ impl MysqlDb { .first::(&self.conn) .optional()? .unwrap_or_default(); - SyncTimestamp::from_i64(modified) + SyncTimestamp::from_i64(modified).map_err(Into::into) } - pub fn get_collection_timestamps_sync( + fn get_collection_timestamps_sync( &self, user_id: UserIdentifier, - ) -> Result { + ) -> DbResult { let modifieds = sql_query(format!( "SELECT {collection_id}, {modified} FROM user_collections @@ -771,26 +769,29 @@ impl MysqlDb { .bind::(TOMBSTONE) .load::(&self.conn)? .into_iter() - .map(|cr| SyncTimestamp::from_i64(cr.last_modified).map(|ts| (cr.collection, ts))) - .collect::>>()?; + .map(|cr| { + SyncTimestamp::from_i64(cr.last_modified) + .map(|ts| (cr.collection, ts)) + .map_err(Into::into) + }) + .collect::>>()?; self.map_collection_names(modifieds) } - fn check_sync(&self) -> Result { + fn check_sync(&self) -> DbResult { // has the database been up for more than 0 seconds? let result = sql_query("SHOW STATUS LIKE \"Uptime\"").execute(&self.conn)?; Ok(result as u64 > 0) } - fn map_collection_names(&self, by_id: HashMap) -> Result> { + fn map_collection_names(&self, by_id: HashMap) -> DbResult> { let mut names = self.load_collection_names(by_id.keys())?; by_id .into_iter() .map(|(id, value)| { - names - .remove(&id) - .map(|name| (name, value)) - .ok_or_else(|| DbError::internal("load_collection_names unknown collection id")) + names.remove(&id).map(|name| (name, value)).ok_or_else(|| { + DbError::internal("load_collection_names unknown collection id".to_owned()) + }) }) .collect() } @@ -798,7 +799,7 @@ impl MysqlDb { fn load_collection_names<'a>( &self, collection_ids: impl Iterator, - ) -> Result> { + ) -> DbResult> { let mut names = HashMap::new(); let mut uncached = Vec::new(); for &id in collection_ids { @@ -830,7 +831,7 @@ impl MysqlDb { &self, user_id: u32, collection_id: i32, - ) -> Result { + ) -> DbResult { let quota = if self.quota.enabled { self.calc_quota_usage_sync(user_id, collection_id)? } else { @@ -869,10 +870,10 @@ impl MysqlDb { } // Perform a lighter weight "read only" storage size check - pub fn get_storage_usage_sync( + fn get_storage_usage_sync( &self, user_id: UserIdentifier, - ) -> Result { + ) -> DbResult { let uid = user_id.legacy_id as i64; let total_bytes = bso::table .select(sql::>("SUM(LENGTH(payload))")) @@ -883,10 +884,10 @@ impl MysqlDb { } // Perform a lighter weight "read only" quota storage check - pub fn get_quota_usage_sync( + fn get_quota_usage_sync( &self, params: params::GetQuotaUsage, - ) -> Result { + ) -> DbResult { let uid = params.user_id.legacy_id as i64; let (total_bytes, count): (i64, i32) = user_collections::table .select(( @@ -905,11 +906,11 @@ impl MysqlDb { } // perform a heavier weight quota calculation - pub fn calc_quota_usage_sync( + fn calc_quota_usage_sync( &self, user_id: u32, collection_id: i32, - ) -> Result { + ) -> DbResult { let (total_bytes, count): (i64, i32) = bso::table .select(( sql::(r#"COALESCE(SUM(LENGTH(COALESCE(payload, ""))),0)"#), @@ -927,10 +928,10 @@ impl MysqlDb { }) } - pub fn get_collection_usage_sync( + fn get_collection_usage_sync( &self, user_id: UserIdentifier, - ) -> Result { + ) -> DbResult { let counts = bso::table .select((bso::collection_id, sql::("SUM(LENGTH(payload))"))) .filter(bso::user_id.eq(user_id.legacy_id as i64)) @@ -942,10 +943,10 @@ impl MysqlDb { self.map_collection_names(counts) } - pub fn get_collection_counts_sync( + fn get_collection_counts_sync( &self, user_id: UserIdentifier, - ) -> Result { + ) -> DbResult { let counts = bso::table .select(( bso::collection_id, @@ -969,51 +970,34 @@ impl MysqlDb { batch_db_method!(commit_batch_sync, commit, CommitBatch); batch_db_method!(delete_batch_sync, delete, DeleteBatch); - pub fn get_batch_sync(&self, params: params::GetBatch) -> Result> { + fn get_batch_sync(&self, params: params::GetBatch) -> DbResult> { batch::get(self, params) } - pub fn timestamp(&self) -> SyncTimestamp { + pub(super) fn timestamp(&self) -> SyncTimestamp { self.session.borrow().timestamp } } -#[macro_export] -macro_rules! sync_db_method { - ($name:ident, $sync_name:ident, $type:ident) => { - sync_db_method!($name, $sync_name, $type, results::$type); - }; - ($name:ident, $sync_name:ident, $type:ident, $result:ty) => { - fn $name(&self, params: params::$type) -> DbFuture<'_, $result> { - let db = self.clone(); - Box::pin( - self.blocking_threadpool - .spawn(move || db.$sync_name(params)), - ) - } - }; -} -impl<'a> Db<'a> for MysqlDb { - fn commit(&self) -> DbFuture<'_, ()> { +impl Db for MysqlDb { + type Error = DbError; + + fn commit(&self) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(self.blocking_threadpool.spawn(move || db.commit_sync())) } - fn rollback(&self) -> DbFuture<'_, ()> { + fn rollback(&self) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(self.blocking_threadpool.spawn(move || db.rollback_sync())) } - fn begin(&self, for_write: bool) -> DbFuture<'_, ()> { + fn begin(&self, for_write: bool) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.begin_async(for_write).map_err(Into::into).await }) } - fn box_clone(&self) -> Box> { - Box::new(self.clone()) - } - - fn check(&self) -> DbFuture<'_, results::Check> { + fn check(&self) -> DbFuture<'_, results::Check, Self::Error> { let db = self.clone(); Box::pin(self.blocking_threadpool.spawn(move || db.check_sync())) } @@ -1073,7 +1057,7 @@ impl<'a> Db<'a> for MysqlDb { ); sync_db_method!(commit_batch, commit_batch_sync, CommitBatch); - fn get_collection_id(&self, name: String) -> DbFuture<'_, i32> { + fn get_collection_id(&self, name: String) -> DbFuture<'_, i32, Self::Error> { let db = self.clone(); Box::pin( self.blocking_threadpool @@ -1085,7 +1069,7 @@ impl<'a> Db<'a> for MysqlDb { results::ConnectionInfo::default() } - fn create_collection(&self, name: String) -> DbFuture<'_, i32> { + fn create_collection(&self, name: String) -> DbFuture<'_, i32, Self::Error> { let db = self.clone(); Box::pin( self.blocking_threadpool @@ -1093,7 +1077,10 @@ impl<'a> Db<'a> for MysqlDb { ) } - fn update_collection(&self, param: params::UpdateCollection) -> DbFuture<'_, SyncTimestamp> { + fn update_collection( + &self, + param: params::UpdateCollection, + ) -> DbFuture<'_, SyncTimestamp, Self::Error> { let db = self.clone(); Box::pin(self.blocking_threadpool.spawn(move || { db.update_collection(param.user_id.legacy_id as u32, param.collection_id) @@ -1110,7 +1097,7 @@ impl<'a> Db<'a> for MysqlDb { sync_db_method!(delete_batch, delete_batch_sync, DeleteBatch); - fn clear_coll_cache(&self) -> DbFuture<'_, ()> { + fn clear_coll_cache(&self) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(self.blocking_threadpool.spawn(move || { db.coll_cache.clear(); @@ -1125,6 +1112,10 @@ impl<'a> Db<'a> for MysqlDb { enforced, } } + + fn box_clone(&self) -> Box> { + Box::new(self.clone()) + } } #[derive(Debug, QueryableByName)] diff --git a/syncserver/src/db/mysql/pool.rs b/syncstorage-mysql/src/pool.rs similarity index 76% rename from syncserver/src/db/mysql/pool.rs rename to syncstorage-mysql/src/pool.rs index a90034db..f2bf0d14 100644 --- a/syncserver/src/db/mysql/pool.rs +++ b/syncstorage-mysql/src/pool.rs @@ -12,16 +12,16 @@ use diesel::{ r2d2::{ConnectionManager, Pool}, Connection, }; -#[cfg(test)] +#[cfg(debug_assertions)] use diesel_logger::LoggingConnection; -use syncserver_db_common::{error::DbError, Db, DbPool, GetPoolState, PoolState, STD_COLLS}; +use syncserver_common::{BlockingThreadpool, Metrics}; +#[cfg(debug_assertions)] +use syncserver_db_common::test::TestTransactionCustomizer; +use syncserver_db_common::{GetPoolState, PoolState}; +use syncstorage_db_common::{Db, DbPool, STD_COLLS}; use syncstorage_settings::{Quota, Settings}; -use super::models::{MysqlDb, Result}; -#[cfg(test)] -use super::test::TestTransactionCustomizer; -use crate::db::BlockingThreadpool; -use crate::server::metrics::Metrics; +use super::{error::DbError, models::MysqlDb, DbResult}; embed_migrations!(); @@ -29,13 +29,13 @@ embed_migrations!(); /// /// Mysql DDL statements implicitly commit which could disrupt MysqlPool's /// begin_test_transaction during tests. So this runs on its own separate conn. -pub fn run_embedded_migrations(database_url: &str) -> Result<()> { +fn run_embedded_migrations(database_url: &str) -> DbResult<()> { let conn = MysqlConnection::establish(database_url)?; - #[cfg(test)] + #[cfg(debug_assertions)] // XXX: this doesn't show the DDL statements // https://github.com/shssoichiro/diesel-logger/issues/1 embedded_migrations::run(&LoggingConnection::new(conn))?; - #[cfg(not(test))] + #[cfg(not(debug_assertions))] embedded_migrations::run(&conn)?; Ok(()) } @@ -61,7 +61,7 @@ impl MysqlDbPool { settings: &Settings, metrics: &Metrics, blocking_threadpool: Arc, - ) -> Result { + ) -> DbResult { run_embedded_migrations(&settings.database_url)?; Self::new_without_migrations(settings, metrics, blocking_threadpool) } @@ -70,7 +70,7 @@ impl MysqlDbPool { settings: &Settings, metrics: &Metrics, blocking_threadpool: Arc, - ) -> Result { + ) -> DbResult { let manager = ConnectionManager::::new(settings.database_url.clone()); let builder = Pool::builder() .max_size(settings.database_pool_max_size) @@ -79,7 +79,7 @@ impl MysqlDbPool { )) .min_idle(settings.database_pool_min_idle); - #[cfg(test)] + #[cfg(debug_assertions)] let builder = if settings.database_use_test_transactions { builder.connection_customizer(Box::new(TestTransactionCustomizer)) } else { @@ -99,7 +99,7 @@ impl MysqlDbPool { }) } - pub fn get_sync(&self) -> Result { + pub fn get_sync(&self) -> DbResult { Ok(MysqlDb::new( self.pool.get()?, Arc::clone(&self.coll_cache), @@ -112,31 +112,25 @@ impl MysqlDbPool { #[async_trait] impl DbPool for MysqlDbPool { - async fn get<'a>(&'a self) -> Result>> { - let pool = self.clone(); - let db = self - .blocking_threadpool - .spawn(move || pool.get_sync()) - .await?; + type Error = DbError; - Ok(Box::new(db) as Box>) + async fn get<'a>(&'a self) -> DbResult>> { + let pool = self.clone(); + self.blocking_threadpool + .spawn(move || pool.get_sync()) + .await + .map(|db| Box::new(db) as Box>) } - fn validate_batch_id(&self, id: String) -> Result<()> { + fn validate_batch_id(&self, id: String) -> DbResult<()> { super::batch::validate_batch_id(&id) } - fn box_clone(&self) -> Box { + fn box_clone(&self) -> Box> { Box::new(self.clone()) } } -impl GetPoolState for MysqlDbPool { - fn state(&self) -> PoolState { - self.pool.state().into() - } -} - impl fmt::Debug for MysqlDbPool { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt.debug_struct("MysqlDbPool") @@ -145,42 +139,48 @@ impl fmt::Debug for MysqlDbPool { } } +impl GetPoolState for MysqlDbPool { + fn state(&self) -> PoolState { + self.pool.state().into() + } +} + #[derive(Debug)] -pub struct CollectionCache { +pub(super) struct CollectionCache { pub by_name: RwLock>, pub by_id: RwLock>, } impl CollectionCache { - pub fn put(&self, id: i32, name: String) -> Result<()> { + pub fn put(&self, id: i32, name: String) -> DbResult<()> { // XXX: should this emit a metric? // XXX: should probably either lock both simultaneously during // writes or use an RwLock alternative self.by_name .write() - .map_err(|_| DbError::internal("by_name write"))? + .map_err(|_| DbError::internal("by_name write".to_owned()))? .insert(name.clone(), id); self.by_id .write() - .map_err(|_| DbError::internal("by_id write"))? + .map_err(|_| DbError::internal("by_id write".to_owned()))? .insert(id, name); Ok(()) } - pub fn get_id(&self, name: &str) -> Result> { + pub fn get_id(&self, name: &str) -> DbResult> { Ok(self .by_name .read() - .map_err(|_| DbError::internal("by_name read"))? + .map_err(|_| DbError::internal("by_name read".to_owned()))? .get(name) .cloned()) } - pub fn get_name(&self, id: i32) -> Result> { + pub fn get_name(&self, id: i32) -> DbResult> { Ok(self .by_id .read() - .map_err(|_| DbError::internal("by_id read"))? + .map_err(|_| DbError::internal("by_id read".to_owned()))? .get(&id) .cloned()) } diff --git a/syncserver/src/db/mysql/schema.rs b/syncstorage-mysql/src/schema.rs similarity index 100% rename from syncserver/src/db/mysql/schema.rs rename to syncstorage-mysql/src/schema.rs diff --git a/syncserver/src/db/mysql/test.rs b/syncstorage-mysql/src/test.rs similarity index 74% rename from syncserver/src/db/mysql/test.rs rename to syncstorage-mysql/src/test.rs index fdeaf1ad..55b98ea1 100644 --- a/syncserver/src/db/mysql/test.rs +++ b/syncstorage-mysql/src/test.rs @@ -1,49 +1,32 @@ -use std::{collections::HashMap, result::Result as StdResult, sync::Arc}; +use std::{collections::HashMap, sync::Arc}; use diesel::{ // expression_methods::TextExpressionMethods, // See note below about `not_like` becoming swedish - mysql::MysqlConnection, - r2d2::{CustomizeConnection, Error as PoolError}, - Connection, ExpressionMethods, QueryDsl, RunQueryDsl, }; +use syncserver_common::{BlockingThreadpool, Metrics}; use syncserver_settings::Settings as SyncserverSettings; use syncstorage_settings::Settings as SyncstorageSettings; use url::Url; -use crate::db::mysql::{ - models::{MysqlDb, Result}, - pool::MysqlDbPool, - schema::collections, -}; -use crate::db::BlockingThreadpool; -use crate::server::metrics; +use crate::{models::MysqlDb, pool::MysqlDbPool, schema::collections, DbResult}; -#[derive(Debug)] -pub struct TestTransactionCustomizer; - -impl CustomizeConnection for TestTransactionCustomizer { - fn on_acquire(&self, conn: &mut MysqlConnection) -> StdResult<(), PoolError> { - conn.begin_test_transaction().map_err(PoolError::QueryError) - } -} - -pub fn db(settings: &SyncstorageSettings) -> Result { +pub fn db(settings: &SyncstorageSettings) -> DbResult { let _ = env_logger::try_init(); // inherit SYNC_SYNCSTORAGE__DATABASE_URL from the env let pool = MysqlDbPool::new( settings, - &metrics::Metrics::noop(), + &Metrics::noop(), Arc::new(BlockingThreadpool::default()), )?; pool.get_sync() } #[test] -fn static_collection_id() -> Result<()> { +fn static_collection_id() -> DbResult<()> { let settings = SyncserverSettings::test_settings().syncstorage; if Url::parse(&settings.database_url).unwrap().scheme() != "mysql" { // Skip this test if we're not using mysql diff --git a/syncstorage-settings/src/lib.rs b/syncstorage-settings/src/lib.rs index 6f03ec53..3011e5c8 100644 --- a/syncstorage-settings/src/lib.rs +++ b/syncstorage-settings/src/lib.rs @@ -75,6 +75,7 @@ pub struct Settings { pub database_pool_connection_lifespan: Option, /// Max time a connection should sit idle before being dropped. pub database_pool_connection_max_idle: Option, + #[cfg(debug_assertions)] pub database_use_test_transactions: bool, /// Server-enforced limits for request payloads. @@ -105,6 +106,7 @@ impl Default for Settings { database_pool_connection_lifespan: None, database_pool_connection_max_idle: None, database_pool_connection_timeout: Some(30), + #[cfg(debug_assertions)] database_use_test_transactions: false, limits: ServerLimits::default(), statsd_label: "syncstorage".to_string(), diff --git a/syncstorage-spanner/Cargo.toml b/syncstorage-spanner/Cargo.toml new file mode 100644 index 00000000..b08ad0f1 --- /dev/null +++ b/syncstorage-spanner/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "syncstorage-spanner" +version = "0.13.6" +edition = "2021" + +[dependencies] +async-trait = "0.1.40" +backtrace = "0.3.61" +cadence = "0.26" +# Pin to 0.5 for now, to keep it under tokio 0.2 (issue977). +# Fix for #803 (deadpool#92) points to our fork for now +#deadpool = "0.5" # pin to 0.5 +deadpool = { git = "https://github.com/mozilla-services/deadpool", branch = "deadpool-v0.5.2-issue92" } +env_logger = "0.9" +futures = { version = "0.3", features = ["compat"] } +google-cloud-rust-raw = "0.11.0" +# Some versions of OpenSSL 1.1.1 conflict with grpcio's built-in boringssl which can cause +# syncserver to either fail to either compile, or start. In those cases, try +# `cargo build --features grpcio/openssl ...` +grpcio = { version = "0.9" } +http = "0.2.5" +log = { version = "0.4", features = [ + "max_level_debug", + "release_max_level_info", +] } +# must match what's used by googleapis-raw +protobuf = "2.20.0" +slog-scope = "4.3" +syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } +syncstorage-db-common = { path = "../syncstorage-db-common" } +syncstorage-settings = { path = "../syncstorage-settings" } +thiserror = "1.0.26" +# pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.) +tokio = { version = "0.2.4", features = ["macros", "sync"] } +url = "2.1" +uuid = { version = "0.8.2", features = ["serde", "v4"] } + +[[bin]] +name = "purge_ttl" +path = "src/bin/purge_ttl.rs" diff --git a/syncserver/src/db/spanner/BATCH_COMMIT.txt b/syncstorage-spanner/src/BATCH_COMMIT.txt similarity index 100% rename from syncserver/src/db/spanner/BATCH_COMMIT.txt rename to syncstorage-spanner/src/BATCH_COMMIT.txt diff --git a/syncserver/src/db/spanner/batch.rs b/syncstorage-spanner/src/batch.rs similarity index 96% rename from syncserver/src/db/spanner/batch.rs rename to syncstorage-spanner/src/batch.rs index 74acfd02..bc69acdc 100644 --- a/syncserver/src/db/spanner/batch.rs +++ b/syncstorage-spanner/src/batch.rs @@ -8,24 +8,24 @@ use protobuf::{ well_known_types::{ListValue, Value}, RepeatedField, }; -use syncserver_db_common::{ - error::{DbError, DbErrorKind}, - params, results, - util::to_rfc3339, - UserIdentifier, BATCH_LIFETIME, DEFAULT_BSO_TTL, +use syncstorage_db_common::{ + params, results, util::to_rfc3339, UserIdentifier, BATCH_LIFETIME, DEFAULT_BSO_TTL, }; use uuid::Uuid; -use super::models::{Result, SpannerDb, PRETOUCH_TS}; +use crate::error::DbError; + +use super::models::{SpannerDb, PRETOUCH_TS}; use super::support::{as_type, null_value, struct_type_field, IntoSpannerValue}; +use super::DbResult; pub async fn create_async( db: &SpannerDb, params: params::CreateBatch, -) -> Result { +) -> DbResult { let batch_id = Uuid::new_v4().to_simple().to_string(); let collection_id = db.get_collection_id_async(¶ms.collection).await?; - let timestamp = db.timestamp()?.as_i64(); + let timestamp = db.checked_timestamp()?.as_i64(); // Ensure a parent record exists in user_collections before writing to batches // (INTERLEAVE IN PARENT user_collections) @@ -66,13 +66,13 @@ pub async fn create_async( Ok(new_batch) } -pub async fn validate_async(db: &SpannerDb, params: params::ValidateBatch) -> Result { +pub async fn validate_async(db: &SpannerDb, params: params::ValidateBatch) -> DbResult { let exists = get_async(db, params.into()).await?; Ok(exists.is_some()) } // Append a collection to a pending batch (`create_batch` creates a new batch) -pub async fn append_async(db: &SpannerDb, params: params::AppendToBatch) -> Result<()> { +pub async fn append_async(db: &SpannerDb, params: params::AppendToBatch) -> DbResult<()> { let mut metrics = db.metrics.clone(); metrics.start_timer("storage.spanner.append_items_to_batch", None); let collection_id = db.get_collection_id_async(¶ms.collection).await?; @@ -98,7 +98,7 @@ pub async fn append_async(db: &SpannerDb, params: params::AppendToBatch) -> Resu if !exists { // NOTE: db tests expects this but it doesn't seem necessary w/ the // handler validating the batch before appends - Err(DbErrorKind::BatchNotFound)? + return Err(DbError::batch_not_found()); } do_append_async( @@ -116,7 +116,7 @@ pub async fn append_async(db: &SpannerDb, params: params::AppendToBatch) -> Resu pub async fn get_async( db: &SpannerDb, params: params::GetBatch, -) -> Result> { +) -> DbResult> { let collection_id = db.get_collection_id_async(¶ms.collection).await?; let (sqlparams, sqlparam_types) = params! { "fxa_uid" => params.user_id.fxa_uid.clone(), @@ -143,7 +143,7 @@ pub async fn get_async( Ok(batch) } -pub async fn delete_async(db: &SpannerDb, params: params::DeleteBatch) -> Result<()> { +pub async fn delete_async(db: &SpannerDb, params: params::DeleteBatch) -> DbResult<()> { let collection_id = db.get_collection_id_async(¶ms.collection).await?; let (sqlparams, sqlparam_types) = params! { "fxa_uid" => params.user_id.fxa_uid.clone(), @@ -170,7 +170,7 @@ pub async fn delete_async(db: &SpannerDb, params: params::DeleteBatch) -> Result pub async fn commit_async( db: &SpannerDb, params: params::CommitBatch, -) -> Result { +) -> DbResult { let mut metrics = db.metrics.clone(); metrics.start_timer("storage.spanner.apply_batch", None); let collection_id = db.get_collection_id_async(¶ms.collection).await?; @@ -249,7 +249,7 @@ pub async fn do_append_async( batch: results::CreateBatch, bsos: Vec, collection: &str, -) -> Result<()> { +) -> DbResult<()> { // Pass an array of struct objects as @values (for UNNEST), e.g.: // [("", "", 101, "ba1", "bso1", NULL, "payload1", NULL), // ("", "", 101, "ba1", "bso2", NULL, "payload2", NULL)] @@ -528,7 +528,7 @@ async fn pretouch_collection_async( db: &SpannerDb, user_id: &UserIdentifier, collection_id: i32, -) -> Result<()> { +) -> DbResult<()> { let (mut sqlparams, mut sqlparam_types) = params! { "fxa_uid" => user_id.fxa_uid.clone(), "fxa_kid" => user_id.fxa_kid.clone(), @@ -569,8 +569,8 @@ async fn pretouch_collection_async( Ok(()) } -pub fn validate_batch_id(id: &str) -> Result<()> { +pub fn validate_batch_id(id: &str) -> DbResult<()> { Uuid::from_str(id) .map(|_| ()) - .map_err(|e| DbError::internal(&format!("Invalid batch_id: {}", e))) + .map_err(|e| DbError::internal(format!("Invalid batch_id: {}", e))) } diff --git a/syncserver/src/db/spanner/batch_commit_insert.sql b/syncstorage-spanner/src/batch_commit_insert.sql similarity index 100% rename from syncserver/src/db/spanner/batch_commit_insert.sql rename to syncstorage-spanner/src/batch_commit_insert.sql diff --git a/syncserver/src/db/spanner/batch_commit_update.sql b/syncstorage-spanner/src/batch_commit_update.sql similarity index 100% rename from syncserver/src/db/spanner/batch_commit_update.sql rename to syncstorage-spanner/src/batch_commit_update.sql diff --git a/syncserver/src/db/spanner/batch_index.sql b/syncstorage-spanner/src/batch_index.sql similarity index 100% rename from syncserver/src/db/spanner/batch_index.sql rename to syncstorage-spanner/src/batch_index.sql diff --git a/syncserver/src/bin/purge_ttl.rs b/syncstorage-spanner/src/bin/purge_ttl.rs similarity index 100% rename from syncserver/src/bin/purge_ttl.rs rename to syncstorage-spanner/src/bin/purge_ttl.rs diff --git a/syncstorage-spanner/src/error.rs b/syncstorage-spanner/src/error.rs new file mode 100644 index 00000000..34a951de --- /dev/null +++ b/syncstorage-spanner/src/error.rs @@ -0,0 +1,151 @@ +use std::fmt; + +use backtrace::Backtrace; +use http::StatusCode; +use syncserver_common::{from_error, impl_fmt_display, InternalError, ReportableError}; +use syncstorage_db_common::error::{DbErrorIntrospect, SyncstorageDbError}; +use thiserror::Error; + +/// An error type that represents any Spanner-related errors that may occur while processing a +/// syncstorage request. These errors may be application-specific or lower-level errors that arise +/// from the database backend. +#[derive(Debug)] +pub struct DbError { + kind: DbErrorKind, + pub status: StatusCode, + pub backtrace: Box, +} + +impl DbError { + pub fn batch_not_found() -> Self { + DbErrorKind::Common(SyncstorageDbError::batch_not_found()).into() + } + + pub fn bso_not_found() -> Self { + DbErrorKind::Common(SyncstorageDbError::bso_not_found()).into() + } + + pub fn collection_not_found() -> Self { + DbErrorKind::Common(SyncstorageDbError::collection_not_found()).into() + } + + pub fn conflict() -> Self { + DbErrorKind::Common(SyncstorageDbError::conflict()).into() + } + + pub fn expired() -> Self { + DbErrorKind::Expired.into() + } + + pub fn integrity(msg: String) -> Self { + DbErrorKind::Integrity(msg).into() + } + + pub fn internal(msg: String) -> Self { + DbErrorKind::Common(SyncstorageDbError::internal(msg)).into() + } + + pub fn quota() -> Self { + DbErrorKind::Common(SyncstorageDbError::quota()).into() + } + + pub fn too_large(msg: String) -> Self { + DbErrorKind::TooLarge(msg).into() + } +} + +#[derive(Debug, Error)] +enum DbErrorKind { + #[error("{}", _0)] + Common(SyncstorageDbError), + + #[error("Connection expired")] + Expired, + + #[error("A database error occurred: {}", _0)] + Grpc(#[from] grpcio::Error), + + #[error("Database integrity error: {}", _0)] + Integrity(String), + + #[error("Spanner data load too large: {}", _0)] + TooLarge(String), +} + +impl From for DbError { + fn from(kind: DbErrorKind) -> Self { + let status = match &kind { + DbErrorKind::Common(e) => e.status, + // Matching the Python code here (a 400 vs 404) + DbErrorKind::TooLarge(_) => StatusCode::BAD_REQUEST, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + + Self { + kind, + status, + backtrace: Box::new(Backtrace::new()), + } + } +} + +impl DbErrorIntrospect for DbError { + fn is_batch_not_found(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_batch_not_found()) + } + + fn is_bso_not_found(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_bso_not_found()) + } + + fn is_collection_not_found(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_collection_not_found()) + } + + fn is_conflict(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_conflict()) + } + + fn is_quota(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_quota()) + } +} + +impl ReportableError for DbError { + fn is_sentry_event(&self) -> bool { + matches!(&self.kind, DbErrorKind::Common(e) if e.is_sentry_event()) + } + + fn metric_label(&self) -> Option { + if let DbErrorKind::Common(e) = &self.kind { + e.metric_label() + } else { + None + } + } + + fn error_backtrace(&self) -> String { + format!("{:#?}", self.backtrace) + } +} + +impl InternalError for DbError { + fn internal_error(message: String) -> Self { + DbErrorKind::Common(SyncstorageDbError::internal(message)).into() + } +} + +impl_fmt_display!(DbError, DbErrorKind); + +from_error!(grpcio::Error, DbError, |inner: grpcio::Error| { + // Convert ABORTED (typically due to a transaction abort) into 503s + match inner { + grpcio::Error::RpcFailure(ref status) | grpcio::Error::RpcFinished(Some(ref status)) + if status.code() == grpcio::RpcStatusCode::ABORTED => + { + DbErrorKind::Common(SyncstorageDbError::conflict()) + } + _ => DbErrorKind::Grpc(inner), + } +}); +from_error!(SyncstorageDbError, DbError, DbErrorKind::Common); diff --git a/syncserver/src/db/spanner/insert_standard_collections.sql b/syncstorage-spanner/src/insert_standard_collections.sql similarity index 100% rename from syncserver/src/db/spanner/insert_standard_collections.sql rename to syncstorage-spanner/src/insert_standard_collections.sql diff --git a/syncstorage-spanner/src/lib.rs b/syncstorage-spanner/src/lib.rs new file mode 100644 index 00000000..560c3f64 --- /dev/null +++ b/syncstorage-spanner/src/lib.rs @@ -0,0 +1,27 @@ +use std::time::SystemTime; + +#[macro_use] +extern crate slog_scope; + +#[macro_use] +mod macros; + +mod batch; +mod error; +mod manager; +mod models; +mod pool; +mod support; + +pub use error::DbError; +pub use models::SpannerDb; +pub use pool::SpannerDbPool; + +type DbResult = Result; + +fn now() -> i64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} diff --git a/syncserver/src/db/spanner/macros.rs b/syncstorage-spanner/src/macros.rs similarity index 98% rename from syncserver/src/db/spanner/macros.rs rename to syncstorage-spanner/src/macros.rs index 651265a3..6785c4e6 100644 --- a/syncserver/src/db/spanner/macros.rs +++ b/syncstorage-spanner/src/macros.rs @@ -19,7 +19,7 @@ macro_rules! params { #[test] fn test_params_macro() { - use crate::db::spanner::support::IntoSpannerValue; + use super::support::IntoSpannerValue; use google_cloud_rust_raw::spanner::v1::type_pb::{Type, TypeCode}; use protobuf::{ well_known_types::{ListValue, Value}, diff --git a/syncserver/src/db/spanner/manager/bb8.rs b/syncstorage-spanner/src/manager/bb8.rs similarity index 96% rename from syncserver/src/db/spanner/manager/bb8.rs rename to syncstorage-spanner/src/manager/bb8.rs index 7ad6d7f8..cb551e21 100644 --- a/syncserver/src/db/spanner/manager/bb8.rs +++ b/syncstorage-spanner/src/manager/bb8.rs @@ -10,7 +10,7 @@ use crate::{ error::{DbError, DbErrorKind}, PoolState, }, - server::metrics::Metrics, + server::Metrics, settings::Settings, }; @@ -19,7 +19,7 @@ use super::session::{create_spanner_session, recycle_spanner_session, SpannerSes #[allow(dead_code)] pub type Conn<'a> = PooledConnection<'a, SpannerSessionManager>; -pub struct SpannerSessionManager { +pub(super) struct SpannerSessionManager { database_name: String, /// The gRPC environment env: Arc, diff --git a/syncserver/src/db/spanner/manager/deadpool.rs b/syncstorage-spanner/src/manager/deadpool.rs similarity index 88% rename from syncserver/src/db/spanner/manager/deadpool.rs rename to syncstorage-spanner/src/manager/deadpool.rs index 4433d655..ededa3fa 100644 --- a/syncserver/src/db/spanner/manager/deadpool.rs +++ b/syncstorage-spanner/src/manager/deadpool.rs @@ -3,17 +3,15 @@ use std::{fmt, sync::Arc}; use async_trait::async_trait; use deadpool::managed::{Manager, RecycleError, RecycleResult}; use grpcio::{EnvBuilder, Environment}; -use syncserver_db_common::error::{DbError, DbErrorKind}; +use syncserver_common::{BlockingThreadpool, Metrics}; use syncstorage_settings::Settings; -use crate::db::BlockingThreadpool; -use crate::server::metrics::Metrics; - use super::session::{create_spanner_session, recycle_spanner_session, SpannerSession}; +use crate::error::DbError; -pub type Conn = deadpool::managed::Object; +pub(crate) type Conn = deadpool::managed::Object; -pub struct SpannerSessionManager { +pub(crate) struct SpannerSessionManager { database_name: String, /// The gRPC environment env: Arc, @@ -42,7 +40,9 @@ impl SpannerSessionManager { ) -> Result { let database_name = settings .spanner_database_name() - .ok_or_else(|| DbErrorKind::InvalidUrl(settings.database_url.to_owned()))? + .ok_or_else(|| { + DbError::internal(format!("invalid database url: {}", settings.database_url)) + })? .to_owned(); let env = Arc::new(EnvBuilder::new().build()); diff --git a/syncstorage-spanner/src/manager/mod.rs b/syncstorage-spanner/src/manager/mod.rs new file mode 100644 index 00000000..b1e20c53 --- /dev/null +++ b/syncstorage-spanner/src/manager/mod.rs @@ -0,0 +1,6 @@ +// mod bb8; +mod deadpool; +mod session; + +pub(super) use self::deadpool::{Conn, SpannerSessionManager}; +pub(super) use self::session::SpannerSession; diff --git a/syncserver/src/db/spanner/manager/session.rs b/syncstorage-spanner/src/manager/session.rs similarity index 94% rename from syncserver/src/db/spanner/manager/session.rs rename to syncstorage-spanner/src/manager/session.rs index b577c9de..68a123a6 100644 --- a/syncserver/src/db/spanner/manager/session.rs +++ b/syncstorage-spanner/src/manager/session.rs @@ -1,13 +1,13 @@ +use std::sync::Arc; + use google_cloud_rust_raw::spanner::v1::{ spanner::{CreateSessionRequest, GetSessionRequest, Session}, spanner_grpc::SpannerClient, }; use grpcio::{CallOption, ChannelBuilder, ChannelCredentials, Environment, MetadataBuilder}; -use std::sync::Arc; -use syncserver_db_common::error::{DbError, DbErrorKind}; +use syncserver_common::{BlockingThreadpool, Metrics}; -use crate::db::{spanner::now, BlockingThreadpool}; -use crate::server::metrics::Metrics; +use crate::error::DbError; const SPANNER_ADDRESS: &str = "spanner.googleapis.com:443"; @@ -21,11 +21,11 @@ pub struct SpannerSession { pub session: Session, /// The underlying client (Connection/Channel) for interacting with spanner pub client: SpannerClient, - pub(in crate::db::spanner) use_test_transactions: bool, + pub(crate) use_test_transactions: bool, /// A second based UTC for SpannerSession creation. /// Session has a similar `create_time` value that is managed by protobuf, /// but some clock skew issues are possible. - pub(in crate::db::spanner) create_time: i64, + pub(crate) create_time: i64, /// Whether we are using the Spanner emulator pub using_spanner_emulator: bool, } @@ -71,7 +71,7 @@ pub async fn create_spanner_session( session, client, use_test_transactions, - create_time: now(), + create_time: crate::now(), using_spanner_emulator, }) } @@ -84,7 +84,7 @@ pub async fn recycle_spanner_session( max_lifetime: Option, max_idle: Option, ) -> Result<(), DbError> { - let now = now(); + let now = crate::now(); let mut req = GetSessionRequest::new(); req.set_name(conn.session.get_name().to_owned()); /* @@ -128,7 +128,7 @@ pub async fn recycle_spanner_session( if age > max_life as i64 { metrics.incr("db.connection.max_life"); dbg!("### aging out", this_session.get_name()); - return Err(DbErrorKind::Expired.into()); + return Err(DbError::expired()); } } // check how long that this has been idle... @@ -145,7 +145,7 @@ pub async fn recycle_spanner_session( if idle > max_idle as i64 { metrics.incr("db.connection.max_idle"); dbg!("### idling out", this_session.get_name()); - return Err(DbErrorKind::Expired.into()); + return Err(DbError::expired()); } // and update the connection's reference session info conn.session = this_session; diff --git a/syncserver/src/db/spanner/models.rs b/syncstorage-spanner/src/models.rs similarity index 87% rename from syncserver/src/db/spanner/models.rs rename to syncstorage-spanner/src/models.rs index 50eef358..fd7c73a8 100644 --- a/syncserver/src/db/spanner/models.rs +++ b/syncstorage-spanner/src/models.rs @@ -22,37 +22,33 @@ use protobuf::{ well_known_types::{ListValue, Value}, Message, RepeatedField, }; -use syncserver_common::MAX_SPANNER_LOAD_SIZE; -use syncserver_db_common::{ - error::{DbError, DbErrorKind}, - params, results, - util::SyncTimestamp, - Db, DbFuture, Sorting, UserIdentifier, DEFAULT_BSO_TTL, FIRST_CUSTOM_COLLECTION_ID, +use syncserver_common::{Metrics, MAX_SPANNER_LOAD_SIZE}; +use syncserver_db_common::DbFuture; +use syncstorage_db_common::{ + error::DbErrorIntrospect, params, results, util::SyncTimestamp, Db, Sorting, UserIdentifier, + DEFAULT_BSO_TTL, FIRST_CUSTOM_COLLECTION_ID, }; use syncstorage_settings::Quota; -use crate::{db::spanner::now, server::metrics::Metrics}; - use super::{ batch, + error::DbError, pool::{CollectionCache, Conn}, support::{ as_type, bso_from_row, bso_to_insert_row, bso_to_update_row, ExecuteSqlRequestBuilder, IntoSpannerValue, StreamedResultSetAsync, }, + DbResult, }; #[derive(Debug, Eq, PartialEq)] -pub enum CollectionLock { +enum CollectionLock { Read, Write, } -pub type Result = std::result::Result; - -pub const TOMBSTONE: i32 = 0; - -pub const PRETOUCH_TS: &str = "0001-01-01T00:00:00.00Z"; +const TOMBSTONE: i32 = 0; +pub(super) const PRETOUCH_TS: &str = "0001-01-01T00:00:00.00Z"; /// Per session Db metadata #[derive(Debug, Default)] @@ -106,7 +102,7 @@ impl Deref for SpannerDb { } impl SpannerDb { - pub fn new( + pub(super) fn new( conn: Conn, coll_cache: Arc, metrics: &Metrics, @@ -128,7 +124,7 @@ impl SpannerDb { self.coll_cache.get_name(id).await } - pub(super) async fn get_collection_id_async(&self, name: &str) -> Result { + pub(super) async fn get_collection_id_async(&self, name: &str) -> DbResult { if let Some(id) = self.coll_cache.get_id(name).await { return Ok(id); } @@ -144,23 +140,25 @@ impl SpannerDb { .execute_async(&self.conn)? .one_or_none() .await? - .ok_or(DbErrorKind::CollectionNotFound)?; + .ok_or_else(DbError::collection_not_found)?; let id = result[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; if !self.in_write_transaction() { self.coll_cache.put(id, name.to_owned()).await; } Ok(id) } - pub(super) async fn create_collection_async(&self, name: &str) -> Result { + pub(super) async fn create_collection_async(&self, name: &str) -> DbResult { // This should always run within a r/w transaction, so that: "If a // transaction successfully commits, then no other writer modified the // data that was read in the transaction after it was read." if !cfg!(test) && !self.in_write_transaction() { - Err(DbError::internal("Can't escalate read-lock to write-lock"))? + return Err(DbError::internal( + "Can't escalate read-lock to write-lock".to_owned(), + )); } let result = self .sql( @@ -173,7 +171,7 @@ impl SpannerDb { let max = result[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; let id = FIRST_CUSTOM_COLLECTION_ID.max(max + 1); let (sqlparams, sqlparam_types) = params! { "name" => name.to_string(), @@ -191,14 +189,14 @@ impl SpannerDb { Ok(id) } - async fn get_or_create_collection_id_async(&self, name: &str) -> Result { + async fn get_or_create_collection_id_async(&self, name: &str) -> DbResult { match self.get_collection_id_async(name).await { Err(err) if err.is_collection_not_found() => self.create_collection_async(name).await, result => result, } } - pub async fn lock_for_read_async(&self, params: params::LockCollection) -> Result<()> { + async fn lock_for_read_async(&self, params: params::LockCollection) -> DbResult<()> { // Begin a transaction self.begin_async(false).await?; @@ -235,7 +233,7 @@ impl SpannerDb { Ok(()) } - pub async fn lock_for_write_async(&self, params: params::LockCollection) -> Result<()> { + async fn lock_for_write_async(&self, params: params::LockCollection) -> DbResult<()> { // Begin a transaction self.begin_async(true).await?; let collection_id = self @@ -248,7 +246,9 @@ impl SpannerDb { .coll_locks .get(&(params.user_id.clone(), collection_id)) { - Err(DbError::internal("Can't escalate read-lock to write-lock"))? + return Err(DbError::internal( + "Can't escalate read-lock to write-lock".to_owned(), + )); } let (sqlparams, mut sqlparam_types) = params! { "fxa_uid" => params.user_id.fxa_uid.clone(), @@ -274,12 +274,12 @@ impl SpannerDb { .await?; let timestamp = if let Some(result) = result { - let modified = SyncTimestamp::from_rfc3339(result[1].get_string_value())?; - let now = SyncTimestamp::from_rfc3339(result[0].get_string_value())?; + let modified = sync_timestamp_from_rfc3339(result[1].get_string_value())?; + let now = sync_timestamp_from_rfc3339(result[0].get_string_value())?; // Forbid the write if it would not properly incr the modified // timestamp if modified >= now { - Err(DbErrorKind::Conflict)? + return Err(DbError::conflict()); } self.session .borrow_mut() @@ -292,7 +292,7 @@ impl SpannerDb { .execute_async(&self.conn)? .one() .await?; - SyncTimestamp::from_rfc3339(result[0].get_string_value())? + sync_timestamp_from_rfc3339(result[0].get_string_value())? }; self.set_timestamp(timestamp); @@ -308,7 +308,7 @@ impl SpannerDb { self.session.borrow_mut().timestamp = Some(timestamp); } - pub(super) fn begin(&self, for_write: bool) -> Result<()> { + pub(super) fn begin(&self, for_write: bool) -> DbResult<()> { let spanner = &self.conn; let mut options = TransactionOptions::new(); if for_write { @@ -328,7 +328,7 @@ impl SpannerDb { Ok(()) } - pub(super) async fn begin_async(&self, for_write: bool) -> Result<()> { + pub(super) async fn begin_async(&self, for_write: bool) -> DbResult<()> { let spanner = &self.conn; let mut options = TransactionOptions::new(); if for_write { @@ -349,7 +349,7 @@ impl SpannerDb { } /// Return the current transaction metadata (TransactionSelector) if one is active. - fn get_transaction(&self) -> Result> { + fn get_transaction(&self) -> DbResult> { if self.session.borrow().transaction.is_none() { self.begin(true)?; } @@ -358,7 +358,7 @@ impl SpannerDb { } /// Return the current transaction metadata (TransactionSelector) if one is active. - async fn get_transaction_async(&self) -> Result> { + async fn get_transaction_async(&self) -> DbResult> { if self.session.borrow().transaction.is_none() { self.begin_async(true).await?; } @@ -366,7 +366,7 @@ impl SpannerDb { Ok(self.session.borrow().transaction.clone()) } - fn sql_request(&self, sql: &str) -> Result { + fn sql_request(&self, sql: &str) -> DbResult { let mut sqlr = ExecuteSqlRequest::new(); sqlr.set_sql(sql.to_owned()); if let Some(transaction) = self.get_transaction()? { @@ -375,16 +375,17 @@ impl SpannerDb { sqlr.seqno = session .execute_sql_count .try_into() - .map_err(|_| DbError::internal("seqno overflow"))?; + .map_err(|_| DbError::internal("seqno overflow".to_owned()))?; session.execute_sql_count += 1; } Ok(sqlr) } - pub(super) fn sql(&self, sql: &str) -> Result { + pub(super) fn sql(&self, sql: &str) -> DbResult { Ok(ExecuteSqlRequestBuilder::new(self.sql_request(sql)?)) } + #[allow(unused)] pub(super) fn insert(&self, table: &str, columns: &[&str], values: Vec) { let mut mutation = Mutation::new(); mutation.set_insert(self.mutation_write(table, columns, values)); @@ -395,6 +396,7 @@ impl SpannerDb { .push(mutation); } + #[allow(unused)] pub(super) fn update(&self, table: &str, columns: &[&str], values: Vec) { let mut mutation = Mutation::new(); mutation.set_update(self.mutation_write(table, columns, values)); @@ -435,7 +437,7 @@ impl SpannerDb { self.session.borrow().in_write_transaction } - pub fn commit(&self) -> Result<()> { + pub fn commit_sync(&self) -> DbResult<()> { if !self.in_write_transaction() { // read-only return Ok(()); @@ -458,11 +460,11 @@ impl SpannerDb { spanner.client.commit(&req)?; Ok(()) } else { - Err(DbError::internal("No transaction to commit"))? + Err(DbError::internal("No transaction to commit".to_owned())) } } - pub async fn commit_async(&self) -> Result<()> { + async fn commit_async(&self) -> DbResult<()> { if !self.in_write_transaction() { // read-only return Ok(()); @@ -485,11 +487,11 @@ impl SpannerDb { spanner.client.commit_async(&req)?.await?; Ok(()) } else { - Err(DbError::internal("No transaction to commit"))? + Err(DbError::internal("No transaction to commit".to_owned())) } } - pub fn rollback(&self) -> Result<()> { + pub fn rollback_sync(&self) -> DbResult<()> { if !self.in_write_transaction() { // read-only return Ok(()); @@ -503,11 +505,11 @@ impl SpannerDb { spanner.client.rollback(&req)?; Ok(()) } else { - Err(DbError::internal("No transaction to rollback"))? + Err(DbError::internal("No transaction to rollback".to_owned())) } } - pub async fn rollback_async(&self) -> Result<()> { + async fn rollback_async(&self) -> DbResult<()> { if !self.in_write_transaction() { // read-only return Ok(()); @@ -521,14 +523,14 @@ impl SpannerDb { spanner.client.rollback_async(&req)?.await?; Ok(()) } else { - Err(DbError::internal("No transaction to rollback"))? + Err(DbError::internal("No transaction to rollback".to_owned())) } } - pub async fn get_collection_timestamp_async( + async fn get_collection_timestamp_async( &self, params: params::GetCollectionTimestamp, - ) -> Result { + ) -> DbResult { let collection_id = self.get_collection_id_async(¶ms.collection).await?; if let Some(modified) = self .session @@ -560,15 +562,15 @@ impl SpannerDb { .execute_async(&self.conn)? .one_or_none() .await? - .ok_or(DbErrorKind::CollectionNotFound)?; - let modified = SyncTimestamp::from_rfc3339(result[0].get_string_value())?; + .ok_or_else(DbError::collection_not_found)?; + let modified = sync_timestamp_from_rfc3339(result[0].get_string_value())?; Ok(modified) } - pub async fn get_collection_timestamps_async( + async fn get_collection_timestamps_async( &self, user_id: params::GetCollectionTimestamps, - ) -> Result { + ) -> DbResult { let (sqlparams, mut sqlparam_types) = params! { "fxa_uid" => user_id.fxa_uid, "fxa_kid" => user_id.fxa_kid, @@ -594,14 +596,17 @@ impl SpannerDb { let collection_id = row[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; - let modified = SyncTimestamp::from_rfc3339(row[1].get_string_value())?; + .map_err(|e| DbError::integrity(e.to_string()))?; + let modified = sync_timestamp_from_rfc3339(row[1].get_string_value())?; results.insert(collection_id, modified); } self.map_collection_names(results).await } - async fn map_collection_names(&self, by_id: HashMap) -> Result> { + async fn map_collection_names( + &self, + by_id: HashMap, + ) -> DbResult> { let mut names = self.load_collection_names(by_id.keys()).await?; by_id .into_iter() @@ -609,7 +614,7 @@ impl SpannerDb { names .remove(&id) .map(|name| (name, value)) - .ok_or_else(|| DbError::internal("load_collection_names get")) + .ok_or_else(|| DbError::internal("load_collection_names get".to_owned())) }) .collect() } @@ -617,7 +622,7 @@ impl SpannerDb { async fn load_collection_names( &self, collection_ids: impl Iterator, - ) -> Result> { + ) -> DbResult> { let (mut names, uncached) = self .coll_cache .get_names(&collection_ids.cloned().collect::>()) @@ -646,7 +651,7 @@ impl SpannerDb { let id = row[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; let name = row[1].take_string_value(); names.insert(id, name.clone()); if !self.in_write_transaction() { @@ -658,10 +663,10 @@ impl SpannerDb { Ok(names) } - pub async fn get_collection_counts_async( + async fn get_collection_counts_async( &self, user_id: params::GetCollectionCounts, - ) -> Result { + ) -> DbResult { let (sqlparams, sqlparam_types) = params! { "fxa_uid" => user_id.fxa_uid, "fxa_kid" => user_id.fxa_kid, @@ -684,20 +689,20 @@ impl SpannerDb { let collection_id = row[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; let count = row[1] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; counts.insert(collection_id, count); } self.map_collection_names(counts).await } - pub async fn get_collection_usage_async( + async fn get_collection_usage_async( &self, user_id: params::GetCollectionUsage, - ) -> Result { + ) -> DbResult { let (sqlparams, sqlparam_types) = params! { "fxa_uid" => user_id.fxa_uid, "fxa_kid" => user_id.fxa_kid @@ -720,20 +725,20 @@ impl SpannerDb { let collection_id = row[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; let usage = row[1] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; usages.insert(collection_id, usage); } self.map_collection_names(usages).await } - pub async fn get_storage_timestamp( + async fn get_storage_timestamp( &self, user_id: params::GetStorageTimestamp, - ) -> Result { + ) -> DbResult { let (sqlparams, mut sqlparam_types) = params! { "fxa_uid" => user_id.fxa_uid, "fxa_kid" => user_id.fxa_kid, @@ -754,16 +759,17 @@ impl SpannerDb { .one() .await?; if row[0].has_null_value() { - SyncTimestamp::from_i64(0) + SyncTimestamp::from_i64(0).map_err(|e| DbError::integrity(e.to_string())) } else { - SyncTimestamp::from_rfc3339(row[0].get_string_value()) + sync_timestamp_from_rfc3339(row[0].get_string_value()) } + .map_err(Into::into) } - pub async fn get_storage_usage_async( + async fn get_storage_usage_async( &self, user_id: params::GetStorageUsage, - ) -> Result { + ) -> DbResult { let (sqlparams, sqlparam_types) = params! { "fxa_uid" => user_id.fxa_uid, "fxa_kid" => user_id.fxa_kid @@ -786,17 +792,17 @@ impl SpannerDb { let usage = result[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; Ok(usage as u64) } else { Ok(0) } } - pub async fn get_quota_usage_async( + async fn get_quota_usage_async( &self, params: params::GetQuotaUsage, - ) -> Result { + ) -> DbResult { if !self.quota.enabled { return Ok(results::GetQuotaUsage::default()); } @@ -822,31 +828,31 @@ impl SpannerDb { result[0] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))? + .map_err(|e| DbError::integrity(e.to_string()))? } else { 0 }; let count = result[1] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?; + .map_err(|e| DbError::integrity(e.to_string()))?; Ok(results::GetQuotaUsage { total_bytes, count }) } else { Ok(results::GetQuotaUsage::default()) } } - pub async fn update_user_collection_quotas( + pub(super) async fn update_user_collection_quotas( &self, user: &UserIdentifier, collection_id: i32, - ) -> Result { + ) -> DbResult { // This will also update the counts in user_collections, since `update_collection_sync` // is called very early to ensure the record exists, and return the timestamp. // This will also write the tombstone if there are no records and we're explicitly // specifying a TOMBSTONE collection_id. // This function should be called after any write operation. - let timestamp = self.timestamp()?; + let timestamp = self.checked_timestamp()?; let (mut sqlparams, mut sqltypes) = params! { "fxa_uid" => user.fxa_uid.clone(), "fxa_kid" => user.fxa_kid.clone(), @@ -903,12 +909,9 @@ impl SpannerDb { ); sqltypes.insert( "total_bytes".to_owned(), - crate::db::spanner::support::as_type(TypeCode::INT64), - ); - sqltypes.insert( - "count".to_owned(), - crate::db::spanner::support::as_type(TypeCode::INT64), + super::support::as_type(TypeCode::INT64), ); + sqltypes.insert("count".to_owned(), super::support::as_type(TypeCode::INT64)); "UPDATE user_collections SET modified = @modified, count = @count, @@ -965,13 +968,13 @@ impl SpannerDb { Ok(timestamp) } - async fn erect_tombstone(&self, user_id: &UserIdentifier) -> Result { + async fn erect_tombstone(&self, user_id: &UserIdentifier) -> DbResult { // Delete the old tombstone (if it exists) let (params, mut param_types) = params! { "fxa_uid" => user_id.fxa_uid.clone(), "fxa_kid" => user_id.fxa_kid.clone(), "collection_id" => TOMBSTONE, - "modified" => self.timestamp()?.as_rfc3339()? + "modified" => self.checked_timestamp()?.as_rfc3339()? }; param_types.insert("modified".to_owned(), as_type(TypeCode::TIMESTAMP)); self.sql( @@ -988,10 +991,10 @@ impl SpannerDb { .await?; // Return timestamp, because sometimes there's a delay between writing and // reading the database. - self.timestamp() + self.checked_timestamp() } - pub async fn delete_storage_async(&self, user_id: params::DeleteStorage) -> Result<()> { + async fn delete_storage_async(&self, user_id: params::DeleteStorage) -> DbResult<()> { // Also deletes child bsos/batch rows (INTERLEAVE IN PARENT // user_collections ON DELETE CASCADE) let (sqlparams, sqlparam_types) = params! { @@ -1010,17 +1013,17 @@ impl SpannerDb { Ok(()) } - pub fn timestamp(&self) -> Result { + pub fn checked_timestamp(&self) -> DbResult { self.session .borrow() .timestamp - .ok_or_else(|| DbError::internal("CURRENT_TIMESTAMP() not read yet")) + .ok_or_else(|| DbError::internal("CURRENT_TIMESTAMP() not read yet".to_owned())) } - pub async fn delete_collection_async( + async fn delete_collection_async( &self, params: params::DeleteCollection, - ) -> Result { + ) -> DbResult { // Also deletes child bsos/batch rows (INTERLEAVE IN PARENT // user_collections ON DELETE CASCADE) let collection_id = self.get_collection_id_async(¶ms.collection).await?; @@ -1059,7 +1062,7 @@ impl SpannerDb { user_id: &UserIdentifier, collection_id: i32, collection: &str, - ) -> Result { + ) -> DbResult { // NOTE: Spanner supports upserts via its InsertOrUpdate mutation but // lacks a SQL equivalent. This call could be 1 InsertOrUpdate instead // of 2 queries but would require put/post_bsos to also use mutations. @@ -1069,7 +1072,7 @@ impl SpannerDb { // Mutations don't run in the same order as ExecuteSql calls, they are // buffered on the client side and only issued to Spanner in the final // transaction Commit. - let timestamp = self.timestamp()?; + let timestamp = self.checked_timestamp()?; if !cfg!(test) && self.session.borrow().updated_collection { // No need to touch it again (except during tests where we // currently reuse Dbs for multiple requests) @@ -1130,7 +1133,7 @@ impl SpannerDb { Ok(timestamp) } - pub async fn delete_bso_async(&self, params: params::DeleteBso) -> Result { + async fn delete_bso_async(&self, params: params::DeleteBso) -> DbResult { let collection_id = self.get_collection_id_async(¶ms.collection).await?; let user_id = params.user_id.clone(); let (sqlparams, sqlparam_types) = params! { @@ -1152,7 +1155,7 @@ impl SpannerDb { .execute_dml_async(&self.conn) .await?; if affected_rows == 0 { - Err(DbErrorKind::BsoNotFound)? + Err(DbError::bso_not_found()) } else { self.metrics.incr("storage.spanner.delete_bso"); Ok(self @@ -1161,10 +1164,7 @@ impl SpannerDb { } } - pub async fn delete_bsos_async( - &self, - params: params::DeleteBsos, - ) -> Result { + async fn delete_bsos_async(&self, params: params::DeleteBsos) -> DbResult { let user_id = params.user_id.clone(); let collection_id = self.get_collection_id_async(¶ms.collection).await?; @@ -1197,7 +1197,7 @@ impl SpannerDb { &self, query_str: &str, params: params::GetBsos, - ) -> Result { + ) -> DbResult { let mut query = query_str.to_owned(); let (mut sqlparams, mut sqlparam_types) = params! { "fxa_uid" => params.user_id.fxa_uid, @@ -1353,7 +1353,7 @@ impl SpannerDb { */ } - pub async fn get_bsos_async(&self, params: params::GetBsos) -> Result { + async fn get_bsos_async(&self, params: params::GetBsos) -> DbResult { let query = "\ SELECT bso_id, sortindex, payload, modified, expiry FROM bsos @@ -1393,7 +1393,7 @@ impl SpannerDb { }) } - pub async fn get_bso_ids_async(&self, params: params::GetBsos) -> Result { + async fn get_bso_ids_async(&self, params: params::GetBsos) -> DbResult { let limit = params.limit.map(i64::from).unwrap_or(-1); let params::Offset { offset, timestamp } = params.offset.clone().unwrap_or_default(); let sort = params.sort; @@ -1412,7 +1412,7 @@ impl SpannerDb { while let Some(row) = stream.next_async().await { let mut row = row?; ids.push(row[0].take_string_value()); - modifieds.push(SyncTimestamp::from_rfc3339(row[1].get_string_value())?.as_i64()); + modifieds.push(sync_timestamp_from_rfc3339(row[1].get_string_value())?.as_i64()); } // NOTE: when bsos.len() == 0, server-syncstorage (the Python impl) // makes an additional call to get_collection_timestamp to potentially @@ -1435,7 +1435,7 @@ impl SpannerDb { }) } - pub async fn get_bso_async(&self, params: params::GetBso) -> Result> { + async fn get_bso_async(&self, params: params::GetBso) -> DbResult> { let collection_id = self.get_collection_id_async(¶ms.collection).await?; let (sqlparams, sqlparam_types) = params! { "fxa_uid" => params.user_id.fxa_uid, @@ -1461,10 +1461,10 @@ impl SpannerDb { .transpose() } - pub async fn get_bso_timestamp_async( + async fn get_bso_timestamp_async( &self, params: params::GetBsoTimestamp, - ) -> Result { + ) -> DbResult { let collection_id = self.get_collection_id_async(¶ms.collection).await?; let (sqlparams, sqlparam_types) = params! { "fxa_uid" => params.user_id.fxa_uid, @@ -1489,13 +1489,15 @@ impl SpannerDb { .one_or_none() .await?; if let Some(result) = result { - SyncTimestamp::from_rfc3339(result[0].get_string_value()) + sync_timestamp_from_rfc3339(result[0].get_string_value()) } else { - SyncTimestamp::from_i64(0) + SyncTimestamp::from_i64(0).map_err(|e| DbError::integrity(e.to_string())) } + .map_err(Into::into) } - pub async fn put_bso_async(&self, params: params::PutBso) -> Result { + #[allow(unused)] + async fn put_bso_async(&self, params: params::PutBso) -> DbResult { let bsos = vec![params::PostCollectionBso { id: params.id, sortindex: params.sortindex, @@ -1515,7 +1517,7 @@ impl SpannerDb { Ok(result.modified) } - pub async fn post_bsos_async(&self, params: params::PostBsos) -> Result { + async fn post_bsos_async(&self, params: params::PostBsos) -> DbResult { let user_id = params.user_id; let collection_id = self .get_or_create_collection_id_async(¶ms.collection) @@ -1581,11 +1583,10 @@ impl SpannerDb { "⚠️Attempted to load too much data into Spanner: {:?} bytes", load_size ); - return Err(DbErrorKind::SpannerTooLarge(format!( + return Err(DbError::too_large(format!( "Committed data too large: {}", load_size - )) - .into()); + ))); } if !inserts.is_empty() { @@ -1621,7 +1622,7 @@ impl SpannerDb { Ok(result) } - async fn check_async(&self) -> Result { + async fn check_async(&self) -> DbResult { // TODO: is there a better check than just fetching UTC? self.sql("SELECT CURRENT_TIMESTAMP()")? .execute_async(&self.conn)? @@ -1635,15 +1636,15 @@ impl SpannerDb { let mut tags = HashMap::default(); tags.insert("collection".to_owned(), collection.to_owned()); self.metrics.incr_with_tags("storage.quota.at_limit", tags); - DbErrorKind::Quota.into() + DbError::quota() } - pub async fn check_quota( + pub(super) async fn check_quota( &self, user_id: &UserIdentifier, collection: &str, collection_id: i32, - ) -> Result> { + ) -> DbResult> { // duplicate quota trap in test func below. if !self.quota.enabled { return Ok(None); @@ -1662,13 +1663,14 @@ impl SpannerDb { warn!("Quota at limit for user's collection: ({} bytes)", usage.total_bytes; "collection"=>collection); } } - Ok(Some(usage.total_bytes as usize)) + Ok(Some(usage.total_bytes)) } // NOTE: Currently this put_bso_async_test impl. is only used during db tests, // see above for the non-tests version - pub async fn put_bso_async_test(&self, bso: params::PutBso) -> Result { - use syncserver_db_common::util::to_rfc3339; + #[allow(unused)] + async fn put_bso_async_test(&self, bso: params::PutBso) -> DbResult { + use syncstorage_db_common::util::to_rfc3339; let collection_id = self .get_or_create_collection_id_async(&bso.collection) .await?; @@ -1685,7 +1687,7 @@ impl SpannerDb { // prewarm the collections table by ensuring that the row is added if not present. self.update_collection_async(&bso.user_id, collection_id, &bso.collection) .await?; - let timestamp = self.timestamp()?; + let timestamp = self.checked_timestamp()?; let result = self .sql( @@ -1841,12 +1843,13 @@ impl SpannerDb { // NOTE: Currently this post_bso_async_test impl. is only used during db tests, // see above for the non-tests version - pub async fn post_bsos_async_test(&self, input: params::PostBsos) -> Result { + #[allow(unused)] + async fn post_bsos_async_test(&self, input: params::PostBsos) -> DbResult { let collection_id = self .get_or_create_collection_id_async(&input.collection) .await?; let mut result = results::PostBsos { - modified: self.timestamp()?, + modified: self.checked_timestamp()?, success: Default::default(), failed: input.failed, }; @@ -1870,28 +1873,30 @@ impl SpannerDb { } } -impl<'a> Db<'a> for SpannerDb { - fn commit(&self) -> DbFuture<'_, ()> { +impl Db for SpannerDb { + type Error = DbError; + + fn commit(&self) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.commit_async().map_err(Into::into).await }) } - fn rollback(&self) -> DbFuture<'_, ()> { + fn rollback(&self) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.rollback_async().map_err(Into::into).await }) } - fn lock_for_read(&self, param: params::LockCollection) -> DbFuture<'_, ()> { + fn lock_for_read(&self, param: params::LockCollection) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.lock_for_read_async(param).map_err(Into::into).await }) } - fn lock_for_write(&self, param: params::LockCollection) -> DbFuture<'_, ()> { + fn lock_for_write(&self, param: params::LockCollection) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.lock_for_write_async(param).map_err(Into::into).await }) } - fn begin(&self, for_write: bool) -> DbFuture<'_, ()> { + fn begin(&self, for_write: bool) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.begin_async(for_write).map_err(Into::into).await }) } @@ -1899,7 +1904,7 @@ impl<'a> Db<'a> for SpannerDb { fn get_collection_timestamp( &self, param: params::GetCollectionTimestamp, - ) -> DbFuture<'_, results::GetCollectionTimestamp> { + ) -> DbFuture<'_, results::GetCollectionTimestamp, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_collection_timestamp_async(param) @@ -1911,7 +1916,7 @@ impl<'a> Db<'a> for SpannerDb { fn get_storage_timestamp( &self, param: params::GetStorageTimestamp, - ) -> DbFuture<'_, results::GetStorageTimestamp> { + ) -> DbFuture<'_, results::GetStorageTimestamp, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_storage_timestamp(param).map_err(Into::into).await }) } @@ -1919,16 +1924,12 @@ impl<'a> Db<'a> for SpannerDb { fn delete_collection( &self, param: params::DeleteCollection, - ) -> DbFuture<'_, results::DeleteCollection> { + ) -> DbFuture<'_, results::DeleteCollection, Self::Error> { let db = self.clone(); Box::pin(async move { db.delete_collection_async(param).map_err(Into::into).await }) } - fn box_clone(&self) -> Box> { - Box::new(self.clone()) - } - - fn check(&self) -> DbFuture<'_, results::Check> { + fn check(&self) -> DbFuture<'_, results::Check, Self::Error> { let db = self.clone(); Box::pin(async move { db.check_async().map_err(Into::into).await }) } @@ -1936,7 +1937,7 @@ impl<'a> Db<'a> for SpannerDb { fn get_collection_timestamps( &self, user_id: params::GetCollectionTimestamps, - ) -> DbFuture<'_, results::GetCollectionTimestamps> { + ) -> DbFuture<'_, results::GetCollectionTimestamps, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_collection_timestamps_async(user_id) @@ -1948,7 +1949,7 @@ impl<'a> Db<'a> for SpannerDb { fn get_collection_counts( &self, user_id: params::GetCollectionCounts, - ) -> DbFuture<'_, results::GetCollectionCounts> { + ) -> DbFuture<'_, results::GetCollectionCounts, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_collection_counts_async(user_id) @@ -1960,7 +1961,7 @@ impl<'a> Db<'a> for SpannerDb { fn get_collection_usage( &self, user_id: params::GetCollectionUsage, - ) -> DbFuture<'_, results::GetCollectionUsage> { + ) -> DbFuture<'_, results::GetCollectionUsage, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_collection_usage_async(user_id) @@ -1972,7 +1973,7 @@ impl<'a> Db<'a> for SpannerDb { fn get_storage_usage( &self, param: params::GetStorageUsage, - ) -> DbFuture<'_, results::GetStorageUsage> { + ) -> DbFuture<'_, results::GetStorageUsage, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_storage_usage_async(param).map_err(Into::into).await }) } @@ -1980,37 +1981,49 @@ impl<'a> Db<'a> for SpannerDb { fn get_quota_usage( &self, param: params::GetQuotaUsage, - ) -> DbFuture<'_, results::GetQuotaUsage> { + ) -> DbFuture<'_, results::GetQuotaUsage, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_quota_usage_async(param).map_err(Into::into).await }) } - fn delete_storage(&self, param: params::DeleteStorage) -> DbFuture<'_, results::DeleteStorage> { + fn delete_storage( + &self, + param: params::DeleteStorage, + ) -> DbFuture<'_, results::DeleteStorage, Self::Error> { let db = self.clone(); Box::pin(async move { db.delete_storage_async(param).map_err(Into::into).await }) } - fn delete_bso(&self, param: params::DeleteBso) -> DbFuture<'_, results::DeleteBso> { + fn delete_bso( + &self, + param: params::DeleteBso, + ) -> DbFuture<'_, results::DeleteBso, Self::Error> { let db = self.clone(); Box::pin(async move { db.delete_bso_async(param).map_err(Into::into).await }) } - fn delete_bsos(&self, param: params::DeleteBsos) -> DbFuture<'_, results::DeleteBsos> { + fn delete_bsos( + &self, + param: params::DeleteBsos, + ) -> DbFuture<'_, results::DeleteBsos, Self::Error> { let db = self.clone(); Box::pin(async move { db.delete_bsos_async(param).map_err(Into::into).await }) } - fn get_bsos(&self, param: params::GetBsos) -> DbFuture<'_, results::GetBsos> { + fn get_bsos(&self, param: params::GetBsos) -> DbFuture<'_, results::GetBsos, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_bsos_async(param).map_err(Into::into).await }) } - fn get_bso_ids(&self, param: params::GetBsoIds) -> DbFuture<'_, results::GetBsoIds> { + fn get_bso_ids( + &self, + param: params::GetBsoIds, + ) -> DbFuture<'_, results::GetBsoIds, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_bso_ids_async(param).map_err(Into::into).await }) } - fn get_bso(&self, param: params::GetBso) -> DbFuture<'_, Option> { + fn get_bso(&self, param: params::GetBso) -> DbFuture<'_, Option, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_bso_async(param).map_err(Into::into).await }) } @@ -2018,41 +2031,47 @@ impl<'a> Db<'a> for SpannerDb { fn get_bso_timestamp( &self, param: params::GetBsoTimestamp, - ) -> DbFuture<'_, results::GetBsoTimestamp> { + ) -> DbFuture<'_, results::GetBsoTimestamp, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_bso_timestamp_async(param).map_err(Into::into).await }) } #[cfg(not(test))] - fn put_bso(&self, param: params::PutBso) -> DbFuture<'_, results::PutBso> { + fn put_bso(&self, param: params::PutBso) -> DbFuture<'_, results::PutBso, Self::Error> { let db = self.clone(); Box::pin(async move { db.put_bso_async(param).map_err(Into::into).await }) } #[cfg(test)] - fn put_bso(&self, param: params::PutBso) -> DbFuture<'_, results::PutBso> { + fn put_bso(&self, param: params::PutBso) -> DbFuture<'_, results::PutBso, Self::Error> { let db = self.clone(); Box::pin(async move { db.put_bso_async_test(param).map_err(Into::into).await }) } #[cfg(not(test))] - fn post_bsos(&self, param: params::PostBsos) -> DbFuture<'_, results::PostBsos> { + fn post_bsos(&self, param: params::PostBsos) -> DbFuture<'_, results::PostBsos, Self::Error> { let db = self.clone(); Box::pin(async move { db.post_bsos_async(param).map_err(Into::into).await }) } #[cfg(test)] - fn post_bsos(&self, param: params::PostBsos) -> DbFuture<'_, results::PostBsos> { + fn post_bsos(&self, param: params::PostBsos) -> DbFuture<'_, results::PostBsos, Self::Error> { let db = self.clone(); Box::pin(async move { db.post_bsos_async_test(param).map_err(Into::into).await }) } - fn create_batch(&self, param: params::CreateBatch) -> DbFuture<'_, results::CreateBatch> { + fn create_batch( + &self, + param: params::CreateBatch, + ) -> DbFuture<'_, results::CreateBatch, Self::Error> { let db = self.clone(); Box::pin(async move { batch::create_async(&db, param).map_err(Into::into).await }) } - fn validate_batch(&self, param: params::ValidateBatch) -> DbFuture<'_, results::ValidateBatch> { + fn validate_batch( + &self, + param: params::ValidateBatch, + ) -> DbFuture<'_, results::ValidateBatch, Self::Error> { let db = self.clone(); Box::pin(async move { batch::validate_async(&db, param).map_err(Into::into).await }) } @@ -2060,29 +2079,35 @@ impl<'a> Db<'a> for SpannerDb { fn append_to_batch( &self, param: params::AppendToBatch, - ) -> DbFuture<'_, results::AppendToBatch> { + ) -> DbFuture<'_, results::AppendToBatch, Self::Error> { let db = self.clone(); Box::pin(async move { batch::append_async(&db, param).map_err(Into::into).await }) } - fn get_batch(&self, param: params::GetBatch) -> DbFuture<'_, Option> { + fn get_batch( + &self, + param: params::GetBatch, + ) -> DbFuture<'_, Option, Self::Error> { let db = self.clone(); Box::pin(async move { batch::get_async(&db, param).map_err(Into::into).await }) } - fn commit_batch(&self, param: params::CommitBatch) -> DbFuture<'_, results::CommitBatch> { + fn commit_batch( + &self, + param: params::CommitBatch, + ) -> DbFuture<'_, results::CommitBatch, Self::Error> { let db = self.clone(); Box::pin(async move { batch::commit_async(&db, param).map_err(Into::into).await }) } - fn get_collection_id(&self, name: String) -> DbFuture<'_, i32> { + fn get_collection_id(&self, name: String) -> DbFuture<'_, i32, Self::Error> { let db = self.clone(); Box::pin(async move { db.get_collection_id_async(&name).map_err(Into::into).await }) } fn get_connection_info(&self) -> results::ConnectionInfo { let session = self.conn.session.clone(); - let now = now(); + let now = super::now(); results::ConnectionInfo { spanner_age: session .create_time @@ -2098,12 +2123,15 @@ impl<'a> Db<'a> for SpannerDb { } } - fn create_collection(&self, name: String) -> DbFuture<'_, i32> { + fn create_collection(&self, name: String) -> DbFuture<'_, i32, Self::Error> { let db = self.clone(); Box::pin(async move { db.create_collection_async(&name).map_err(Into::into).await }) } - fn update_collection(&self, param: params::UpdateCollection) -> DbFuture<'_, SyncTimestamp> { + fn update_collection( + &self, + param: params::UpdateCollection, + ) -> DbFuture<'_, SyncTimestamp, Self::Error> { let db = self.clone(); Box::pin(async move { db.update_collection_async(¶m.user_id, param.collection_id, ¶m.collection) @@ -2113,7 +2141,7 @@ impl<'a> Db<'a> for SpannerDb { } fn timestamp(&self) -> SyncTimestamp { - self.timestamp() + self.checked_timestamp() .expect("set_timestamp() not called yet for SpannerDb") } @@ -2121,12 +2149,15 @@ impl<'a> Db<'a> for SpannerDb { SpannerDb::set_timestamp(self, timestamp) } - fn delete_batch(&self, param: params::DeleteBatch) -> DbFuture<'_, results::DeleteBatch> { + fn delete_batch( + &self, + param: params::DeleteBatch, + ) -> DbFuture<'_, results::DeleteBatch, Self::Error> { let db = self.clone(); Box::pin(async move { batch::delete_async(&db, param).map_err(Into::into).await }) } - fn clear_coll_cache(&self) -> DbFuture<'_, ()> { + fn clear_coll_cache(&self) -> DbFuture<'_, (), Self::Error> { let db = self.clone(); Box::pin(async move { db.coll_cache.clear().await; @@ -2141,4 +2172,12 @@ impl<'a> Db<'a> for SpannerDb { enforced, }; } + + fn box_clone(&self) -> Box> { + Box::new(self.clone()) + } +} + +fn sync_timestamp_from_rfc3339(val: &str) -> Result { + SyncTimestamp::from_rfc3339(val).map_err(|e| DbError::integrity(e.to_string())) } diff --git a/syncserver/src/db/spanner/pool.rs b/syncstorage-spanner/src/pool.rs similarity index 75% rename from syncserver/src/db/spanner/pool.rs rename to syncstorage-spanner/src/pool.rs index f1d474d6..35de15dd 100644 --- a/syncserver/src/db/spanner/pool.rs +++ b/syncstorage-spanner/src/pool.rs @@ -1,32 +1,20 @@ use std::{collections::HashMap, fmt, sync::Arc, time::Duration}; use async_trait::async_trait; -use bb8::ErrorSink; -use syncserver_db_common::{error::DbError, Db, DbPool, GetPoolState, PoolState, STD_COLLS}; +use syncserver_common::{BlockingThreadpool, Metrics}; +use syncserver_db_common::{GetPoolState, PoolState}; +use syncstorage_db_common::{Db, DbPool, STD_COLLS}; use syncstorage_settings::{Quota, Settings}; use tokio::sync::RwLock; -use crate::db::BlockingThreadpool; -use crate::server::metrics::Metrics; - -pub use super::manager::Conn; +pub(super) use super::manager::Conn; use super::{ + error::DbError, manager::{SpannerSession, SpannerSessionManager}, - models::Result, models::SpannerDb, + DbResult, }; -embed_migrations!(); - -/// Run the diesel embedded migrations -/// -/// Mysql DDL statements implicitly commit which could disrupt MysqlPool's -/// begin_test_transaction during tests. So this runs on its own separate conn. -//pub fn run_embedded_migrations(settings: &Settings) -> Result<()> { -// let conn = MysqlConnection::establish(&settings.database_url)?; -// Ok(embedded_migrations::run(&conn)?) -//} - #[derive(Clone)] pub struct SpannerDbPool { /// Pool of db connections @@ -40,20 +28,20 @@ pub struct SpannerDbPool { impl SpannerDbPool { /// Creates a new pool of Spanner db connections. - pub async fn new( + pub fn new( settings: &Settings, metrics: &Metrics, blocking_threadpool: Arc, - ) -> Result { + ) -> DbResult { //run_embedded_migrations(settings)?; - Self::new_without_migrations(settings, metrics, blocking_threadpool).await + Self::new_without_migrations(settings, metrics, blocking_threadpool) } - pub async fn new_without_migrations( + pub fn new_without_migrations( settings: &Settings, metrics: &Metrics, blocking_threadpool: Arc, - ) -> Result { + ) -> DbResult { let max_size = settings.database_pool_max_size as usize; let wait = settings .database_pool_connection_timeout @@ -78,11 +66,11 @@ impl SpannerDbPool { }) } - pub async fn get_async(&self) -> Result { + pub async fn get_async(&self) -> DbResult { let conn = self.pool.get().await.map_err(|e| match e { deadpool::managed::PoolError::Backend(dbe) => dbe, deadpool::managed::PoolError::Timeout(timeout_type) => { - DbError::internal(&format!("deadpool Timeout: {:?}", timeout_type)) + DbError::internal(format!("deadpool Timeout: {:?}", timeout_type)) } })?; Ok(SpannerDb::new( @@ -96,21 +84,23 @@ impl SpannerDbPool { #[async_trait] impl DbPool for SpannerDbPool { - async fn get<'a>(&'a self) -> Result>> { + type Error = DbError; + + async fn get(&self) -> DbResult>> { let mut metrics = self.metrics.clone(); metrics.start_timer("storage.spanner.get_pool", None); self.get_async() .await - .map(|db| Box::new(db) as Box>) + .map(|db| Box::new(db) as Box>) .map_err(Into::into) } - fn validate_batch_id(&self, id: String) -> Result<()> { - super::batch::validate_batch_id(&id) + fn validate_batch_id(&self, id: String) -> DbResult<()> { + super::batch::validate_batch_id(&id).map_err(Into::into) } - fn box_clone(&self) -> Box { + fn box_clone(&self) -> Box> { Box::new(self.clone()) } } @@ -130,7 +120,7 @@ impl fmt::Debug for SpannerDbPool { } #[derive(Debug)] -pub struct CollectionCache { +pub(super) struct CollectionCache { pub by_name: RwLock>, pub by_id: RwLock>, } @@ -194,19 +184,3 @@ impl Default for CollectionCache { } } } - -/// Logs internal bb8 errors -#[derive(Debug, Clone, Copy)] -pub struct LoggingErrorSink; - -impl ErrorSink for LoggingErrorSink { - fn sink(&self, e: E) { - error!("bb8 Error: {}", e); - let event = sentry::event_from_error(&e); - sentry::capture_event(event); - } - - fn boxed_clone(&self) -> Box> { - Box::new(*self) - } -} diff --git a/syncserver/src/db/spanner/schema.ddl b/syncstorage-spanner/src/schema.ddl similarity index 100% rename from syncserver/src/db/spanner/schema.ddl rename to syncstorage-spanner/src/schema.ddl diff --git a/syncserver/src/db/spanner/support.rs b/syncstorage-spanner/src/support.rs similarity index 87% rename from syncserver/src/db/spanner/support.rs rename to syncstorage-spanner/src/support.rs index 5abeddcc..3fac678f 100644 --- a/syncserver/src/db/spanner/support.rs +++ b/syncstorage-spanner/src/support.rs @@ -1,7 +1,6 @@ use std::{ collections::{HashMap, VecDeque}, mem, - result::Result as StdResult, }; use futures::stream::{StreamExt, StreamFuture}; @@ -15,15 +14,11 @@ use protobuf::{ well_known_types::{ListValue, NullValue, Struct, Value}, RepeatedField, }; -use syncserver_db_common::{ - error::{DbError, DbErrorKind}, - params, results, - util::to_rfc3339, - util::SyncTimestamp, - UserIdentifier, DEFAULT_BSO_TTL, +use syncstorage_db_common::{ + params, results, util::to_rfc3339, util::SyncTimestamp, UserIdentifier, DEFAULT_BSO_TTL, }; -use super::{models::Result, pool::Conn}; +use super::{error::DbError, pool::Conn, DbResult}; pub trait IntoSpannerValue { const TYPE_CODE: TypeCode; @@ -169,7 +164,7 @@ impl ExecuteSqlRequestBuilder { } /// Execute a SQL read statement but return a non-blocking streaming result - pub fn execute_async(self, conn: &Conn) -> Result { + pub fn execute_async(self, conn: &Conn) -> DbResult { let stream = conn .client .execute_streaming_sql(&self.prepare_request(conn))?; @@ -177,7 +172,7 @@ impl ExecuteSqlRequestBuilder { } /// Execute a DML statement, returning the exact count of modified rows - pub async fn execute_dml_async(self, conn: &Conn) -> Result { + pub async fn execute_dml_async(self, conn: &Conn) -> DbResult { let rs = conn .client .execute_sql_async(&self.prepare_request(conn))? @@ -230,20 +225,24 @@ impl StreamedResultSetAsync { } } - pub async fn one(&mut self) -> Result> { + pub async fn one(&mut self) -> DbResult> { if let Some(result) = self.one_or_none().await? { Ok(result) } else { - Err(DbError::internal("No rows matched the given query."))? + Err(DbError::internal( + "No rows matched the given query.".to_owned(), + )) } } - pub async fn one_or_none(&mut self) -> Result>> { + pub async fn one_or_none(&mut self) -> DbResult>> { let result = self.next_async().await; if result.is_none() { Ok(None) } else if self.next_async().await.is_some() { - Err(DbError::internal("Expected one result; got more."))? + Err(DbError::internal( + "Expected one result; got more.".to_owned(), + )) } else { result.transpose() } @@ -252,7 +251,7 @@ impl StreamedResultSetAsync { /// Pull and process the next values from the Stream /// /// Returns false when the stream is finished - async fn consume_next(&mut self) -> Result { + async fn consume_next(&mut self) -> DbResult { let (result, stream) = self .stream .take() @@ -286,9 +285,9 @@ impl StreamedResultSetAsync { let fields = self.fields(); let current_row_i = self.current_row.len(); if fields.len() <= current_row_i { - Err(DbErrorKind::Integrity( + return Err(DbError::integrity( "Invalid PartialResultSet fields".to_owned(), - ))?; + )); } let field = &fields[current_row_i]; values[0] = merge_by_type(pending_chunk, &values[0], field.get_field_type())?; @@ -314,7 +313,7 @@ impl StreamedResultSetAsync { // We could implement Stream::poll_next instead of this, but // this is easier for now and we can refactor into the trait later - pub async fn next_async(&mut self) -> Option>> { + pub async fn next_async(&mut self) -> Option>> { while self.rows.is_empty() { match self.consume_next().await { Ok(true) => (), @@ -329,7 +328,7 @@ impl StreamedResultSetAsync { } } -fn merge_by_type(lhs: Value, rhs: &Value, field_type: &Type) -> Result { +fn merge_by_type(lhs: Value, rhs: &Value, field_type: &Type) -> DbResult { // We only support merging basic string types as that's all we currently use. // The python client also supports: float64, array, struct. The go client // only additionally supports array (claiming structs are only returned as @@ -344,25 +343,28 @@ fn merge_by_type(lhs: Value, rhs: &Value, field_type: &Type) -> Result { } } -fn unsupported_merge(field_type: &Type) -> Result { - Err(DbError::internal(&format!( +fn unsupported_merge(field_type: &Type) -> DbResult { + Err(DbError::internal(format!( "merge not supported, type: {:?}", field_type ))) } -fn merge_string(mut lhs: Value, rhs: &Value) -> Result { +fn merge_string(mut lhs: Value, rhs: &Value) -> DbResult { if !lhs.has_string_value() || !rhs.has_string_value() { - Err(DbError::internal("merge_string has no string value"))? + return Err(DbError::internal( + "merge_string has no string value".to_owned(), + )); } let mut merged = lhs.take_string_value(); merged.push_str(rhs.get_string_value()); Ok(merged.into_spanner_value()) } -pub fn bso_from_row(mut row: Vec) -> Result { +pub fn bso_from_row(mut row: Vec) -> DbResult { let modified_string = &row[3].get_string_value(); - let modified = SyncTimestamp::from_rfc3339(modified_string)?; + let modified = SyncTimestamp::from_rfc3339(modified_string) + .map_err(|e| DbError::integrity(e.to_string()))?; Ok(results::GetBso { id: row[0].take_string_value(), sortindex: if row[1].has_null_value() { @@ -372,12 +374,14 @@ pub fn bso_from_row(mut row: Vec) -> Result { row[1] .get_string_value() .parse::() - .map_err(|e| DbErrorKind::Integrity(e.to_string()))?, + .map_err(|e| DbError::integrity(e.to_string()))?, ) }, payload: row[2].take_string_value(), modified, - expiry: SyncTimestamp::from_rfc3339(row[4].get_string_value())?.as_i64(), + expiry: SyncTimestamp::from_rfc3339(row[4].get_string_value()) + .map_err(|e| DbError::integrity(e.to_string()))? + .as_i64(), }) } @@ -386,7 +390,7 @@ pub fn bso_to_insert_row( collection_id: i32, bso: params::PostCollectionBso, now: SyncTimestamp, -) -> Result { +) -> DbResult { let sortindex = bso .sortindex .map(|sortindex| sortindex.into_spanner_value()) @@ -413,7 +417,7 @@ pub fn bso_to_update_row( collection_id: i32, bso: params::PostCollectionBso, now: SyncTimestamp, -) -> Result<(Vec<&'static str>, ListValue)> { +) -> DbResult<(Vec<&'static str>, ListValue)> { let mut columns = vec!["fxa_uid", "fxa_kid", "collection_id", "bso_id"]; let mut values = vec![ user_id.fxa_uid.clone().into_spanner_value(), @@ -454,10 +458,10 @@ pub struct MapAndThenIterator { impl Iterator for MapAndThenIterator where - F: FnMut(A) -> StdResult, - I: Iterator>, + F: FnMut(A) -> Result, + I: Iterator>, { - type Item = StdResult; + type Item = Result; fn next(&mut self) -> Option { self.iter.next().map(|result| result.and_then(&mut self.f)) @@ -466,13 +470,13 @@ where pub trait MapAndThenTrait { /// Return an iterator adaptor that applies the provided closure to every - /// Result::Ok value. Result::Err values are unchanged. + /// DbResult::Ok value. DbResult::Err values are unchanged. /// /// The closure can be used for control flow based on result values fn map_and_then(self, func: F) -> MapAndThenIterator where - Self: Sized + Iterator>, - F: FnMut(A) -> StdResult, + Self: Sized + Iterator>, + F: FnMut(A) -> Result, { MapAndThenIterator { iter: self, @@ -481,4 +485,4 @@ pub trait MapAndThenTrait { } } -impl MapAndThenTrait for I where I: Sized + Iterator> {} +impl MapAndThenTrait for I where I: Sized + Iterator> {} diff --git a/tokenserver-auth/Cargo.toml b/tokenserver-auth/Cargo.toml new file mode 100644 index 00000000..d0fa9fcc --- /dev/null +++ b/tokenserver-auth/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "tokenserver-auth" +version = "0.13.6" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-trait = "0.1.40" +dyn-clone = "1.0.4" +futures = { version = "0.3", features = ["compat"] } +pyo3 = { version = "0.14", features = ["auto-initialize"] } +reqwest = { version = "0.10.10", features = ["json", "rustls-tls"] } +serde = "1.0" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +syncserver-common = { path = "../syncserver-common" } +tokenserver-common = { path = "../tokenserver-common" } +tokenserver-settings = { path = "../tokenserver-settings" } +# pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.) +tokio = { version = "0.2.4", features = ["blocking"] } + +[dev-dependencies] +mockito = "0.30.0" diff --git a/syncserver/src/tokenserver/auth/browserid.rs b/tokenserver-auth/src/browserid.rs similarity index 98% rename from syncserver/src/tokenserver/auth/browserid.rs rename to tokenserver-auth/src/browserid.rs index 8be9bb0f..2757cf42 100644 --- a/syncserver/src/tokenserver/auth/browserid.rs +++ b/tokenserver-auth/src/browserid.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use reqwest::{Client as ReqwestClient, StatusCode}; use serde::{de::Deserializer, Deserialize, Serialize}; -use tokenserver_common::error::{ErrorLocation, TokenType, TokenserverError}; +use tokenserver_common::{ErrorLocation, TokenType, TokenserverError}; use tokenserver_settings::Settings; use super::VerifyToken; @@ -19,7 +19,7 @@ pub struct VerifyOutput { /// The verifier used to verify BrowserID assertions. #[derive(Clone)] -pub struct RemoteVerifier { +pub struct Verifier { audience: String, issuer: String, fxa_verifier_url: String, @@ -28,7 +28,7 @@ pub struct RemoteVerifier { request_client: ReqwestClient, } -impl TryFrom<&Settings> for RemoteVerifier { +impl TryFrom<&Settings> for Verifier { type Error = &'static str; fn try_from(settings: &Settings) -> Result { @@ -47,7 +47,7 @@ impl TryFrom<&Settings> for RemoteVerifier { } #[async_trait] -impl VerifyToken for RemoteVerifier { +impl VerifyToken for Verifier { type Output = VerifyOutput; /// Verifies a BrowserID assertion. Returns `VerifyOutput` for valid assertions and a @@ -288,7 +288,7 @@ mod tests { use mockito::{self, Mock}; use serde_json::json; - #[actix_rt::test] + #[tokio::test] async fn test_browserid_verifier_success() { let body = json!({ "status": "okay", @@ -305,7 +305,7 @@ mod tests { .with_header("content-type", "application/json") .with_body(body.to_string()) .create(); - let verifier = RemoteVerifier::try_from(&Settings { + let verifier = Verifier::try_from(&Settings { fxa_browserid_audience: "https://test.com".to_owned(), fxa_browserid_issuer: "accounts.firefox.com".to_owned(), fxa_browserid_server_url: format!("{}/v2", mockito::server_url()), @@ -326,11 +326,11 @@ mod tests { assert_eq!(expected_result, result); } - #[actix_rt::test] + #[tokio::test] async fn test_browserid_verifier_failure_cases() { const AUDIENCE: &str = "https://test.com"; - let verifier = RemoteVerifier::try_from(&Settings { + let verifier = Verifier::try_from(&Settings { fxa_browserid_audience: AUDIENCE.to_owned(), fxa_browserid_server_url: format!("{}/v2", mockito::server_url()), ..Default::default() @@ -446,7 +446,7 @@ mod tests { } } - #[actix_rt::test] + #[tokio::test] async fn test_browserid_verifier_rejects_unissuers() { const AUDIENCE: &str = "https://test.com"; const ISSUER: &str = "accounts.firefox.com"; @@ -470,7 +470,7 @@ mod tests { token_type: TokenType::BrowserId, ..TokenserverError::invalid_credentials("Unauthorized".to_owned()) }; - let verifier = RemoteVerifier::try_from(&Settings { + let verifier = Verifier::try_from(&Settings { fxa_browserid_audience: AUDIENCE.to_owned(), fxa_browserid_issuer: ISSUER.to_owned(), fxa_browserid_server_url: format!("{}/v2", mockito::server_url()), diff --git a/syncserver/src/tokenserver/auth/mod.rs b/tokenserver-auth/src/lib.rs similarity index 98% rename from syncserver/src/tokenserver/auth/mod.rs rename to tokenserver-auth/src/lib.rs index 1a5e4d54..d119c44d 100644 --- a/syncserver/src/tokenserver/auth/mod.rs +++ b/tokenserver-auth/src/lib.rs @@ -10,7 +10,7 @@ use pyo3::{ types::IntoPyDict, }; use serde::{Deserialize, Serialize}; -use tokenserver_common::error::TokenserverError; +use tokenserver_common::TokenserverError; /// Represents the origin of the token used by Sync clients to access their data. #[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)] diff --git a/syncserver/src/tokenserver/auth/oauth.rs b/tokenserver-auth/src/oauth.rs similarity index 98% rename from syncserver/src/tokenserver/auth/oauth.rs rename to tokenserver-auth/src/oauth.rs index 9853498b..fb61eb5f 100644 --- a/syncserver/src/tokenserver/auth/oauth.rs +++ b/tokenserver-auth/src/oauth.rs @@ -5,12 +5,12 @@ use pyo3::{ }; use serde::{Deserialize, Serialize}; use serde_json; -use tokenserver_common::error::TokenserverError; +use syncserver_common::BlockingThreadpool; +use tokenserver_common::TokenserverError; use tokenserver_settings::{Jwk, Settings}; use tokio::time; use super::VerifyToken; -use crate::server::BlockingThreadpool; use std::{sync::Arc, time::Duration}; @@ -156,7 +156,7 @@ impl VerifyToken for Verifier { .spawn(move || verify_inner(&verifier)); // The PyFxA OAuth client does not offer a way to set a request timeout, so we set one here - // by timing out the future if the verification process blocks its thread for longer + // by timing out the future if the verification process blocks this thread for longer // than the specified number of seconds. time::timeout(Duration::from_secs(self.timeout), fut) .await diff --git a/syncserver/src/tokenserver/auth/verify.py b/tokenserver-auth/src/verify.py similarity index 100% rename from syncserver/src/tokenserver/auth/verify.py rename to tokenserver-auth/src/verify.py diff --git a/tokenserver-common/Cargo.toml b/tokenserver-common/Cargo.toml index 6e924ac8..7cfd5d52 100644 --- a/tokenserver-common/Cargo.toml +++ b/tokenserver-common/Cargo.toml @@ -9,5 +9,4 @@ backtrace = "0.3.61" serde = "1.0" serde_json = { version = "1.0", features = ["arbitrary_precision"] } syncserver-common = { path = "../syncserver-common" } -syncserver-db-common = { path = "../syncserver-db-common" } thiserror = "1.0.26" diff --git a/tokenserver-common/src/error.rs b/tokenserver-common/src/error.rs index cab5ce00..1f7dcf31 100644 --- a/tokenserver-common/src/error.rs +++ b/tokenserver-common/src/error.rs @@ -7,8 +7,9 @@ use serde::{ Serialize, }; use syncserver_common::{InternalError, ReportableError}; -use syncserver_db_common::error::DbError; +/// An error type that represents application-specific errors to Tokenserver. This error is not +/// used to represent database-related errors; database-related errors have their own type. #[derive(Clone, Debug)] pub struct TokenserverError { pub status: &'static str, @@ -249,25 +250,6 @@ impl Serialize for TokenserverError { } } -impl From for TokenserverError { - fn from(db_error: DbError) -> Self { - TokenserverError { - description: db_error.to_string(), - context: db_error.to_string(), - backtrace: Box::new(db_error.backtrace), - http_status: if db_error.status.is_server_error() { - // Use the status code from the DbError if it already suggests an internal error; - // it might be more specific than `StatusCode::SERVICE_UNAVAILABLE` - db_error.status - } else { - StatusCode::SERVICE_UNAVAILABLE - }, - // An unhandled DbError in the Tokenserver code is an internal error - ..TokenserverError::internal_error() - } - } -} - impl From for HttpResponse { fn from(inner: TokenserverError) -> Self { ResponseError::error_response(&inner) diff --git a/tokenserver-common/src/lib.rs b/tokenserver-common/src/lib.rs index ab76d983..c03ea329 100644 --- a/tokenserver-common/src/lib.rs +++ b/tokenserver-common/src/lib.rs @@ -1,7 +1,9 @@ -pub mod error; +mod error; use serde::{Deserialize, Serialize}; +pub use error::{ErrorLocation, TokenType, TokenserverError}; + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] pub enum NodeType { #[serde(rename = "mysql")] diff --git a/tokenserver-db/Cargo.toml b/tokenserver-db/Cargo.toml new file mode 100644 index 00000000..d051e8cb --- /dev/null +++ b/tokenserver-db/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "tokenserver-db" +version = "0.13.6" +edition = "2021" + +[dependencies] +async-trait = "0.1.40" +backtrace = "0.3.61" +diesel = { version = "1.4", features = ["mysql", "r2d2"] } +diesel_logger = "0.1.1" +diesel_migrations = { version = "1.4.0", features = ["mysql"] } +futures = { version = "0.3", features = ["compat"] } +http = "0.2.5" +serde = "1.0" +serde_derive = "1.0" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } +slog-scope = "4.3" +syncserver-common = { path = "../syncserver-common" } +syncserver-db-common = { path = "../syncserver-db-common" } +thiserror = "1.0.26" +tokenserver-common = { path = "../tokenserver-common" } +tokenserver-settings = { path = "../tokenserver-settings" } +# pinning to 0.2.4 due to high number of dependencies (actix, bb8, deadpool, etc.) +tokio = { version = "0.2.4", features = ["macros", "sync"] } + +[dev-dependencies] +syncserver-settings = { path = "../syncserver-settings" } +env_logger = "0.9" diff --git a/syncserver/src/tokenserver/migrations/2021-07-16-001122_init/down.sql b/tokenserver-db/migrations/2021-07-16-001122_init/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-07-16-001122_init/down.sql rename to tokenserver-db/migrations/2021-07-16-001122_init/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-07-16-001122_init/up.sql b/tokenserver-db/migrations/2021-07-16-001122_init/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-07-16-001122_init/up.sql rename to tokenserver-db/migrations/2021-07-16-001122_init/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/down.sql b/tokenserver-db/migrations/2021-08-03-234845_populate_services/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/down.sql rename to tokenserver-db/migrations/2021-08-03-234845_populate_services/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/up.sql b/tokenserver-db/migrations/2021-08-03-234845_populate_services/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-08-03-234845_populate_services/up.sql rename to tokenserver-db/migrations/2021-08-03-234845_populate_services/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql b/tokenserver-db/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql rename to tokenserver-db/migrations/2021-09-30-142643_remove_foreign_key_constraints/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql b/tokenserver-db/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql rename to tokenserver-db/migrations/2021-09-30-142643_remove_foreign_key_constraints/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql b/tokenserver-db/migrations/2021-09-30-142654_remove_node_defaults/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/down.sql rename to tokenserver-db/migrations/2021-09-30-142654_remove_node_defaults/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql b/tokenserver-db/migrations/2021-09-30-142654_remove_node_defaults/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-142654_remove_node_defaults/up.sql rename to tokenserver-db/migrations/2021-09-30-142654_remove_node_defaults/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql b/tokenserver-db/migrations/2021-09-30-142746_add_indexes/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/down.sql rename to tokenserver-db/migrations/2021-09-30-142746_add_indexes/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql b/tokenserver-db/migrations/2021-09-30-142746_add_indexes/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-142746_add_indexes/up.sql rename to tokenserver-db/migrations/2021-09-30-142746_add_indexes/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql b/tokenserver-db/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql rename to tokenserver-db/migrations/2021-09-30-144043_remove_nodes_service_key/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql b/tokenserver-db/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql rename to tokenserver-db/migrations/2021-09-30-144043_remove_nodes_service_key/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql b/tokenserver-db/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql rename to tokenserver-db/migrations/2021-09-30-144225_remove_users_nodeid_key/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql b/tokenserver-db/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql rename to tokenserver-db/migrations/2021-09-30-144225_remove_users_nodeid_key/up.sql diff --git a/syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/down.sql b/tokenserver-db/migrations/2021-12-22-160451_remove_services/down.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/down.sql rename to tokenserver-db/migrations/2021-12-22-160451_remove_services/down.sql diff --git a/syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/up.sql b/tokenserver-db/migrations/2021-12-22-160451_remove_services/up.sql similarity index 100% rename from syncserver/src/tokenserver/migrations/2021-12-22-160451_remove_services/up.sql rename to tokenserver-db/migrations/2021-12-22-160451_remove_services/up.sql diff --git a/tokenserver-db/src/error.rs b/tokenserver-db/src/error.rs new file mode 100644 index 00000000..a7e30a12 --- /dev/null +++ b/tokenserver-db/src/error.rs @@ -0,0 +1,104 @@ +use std::fmt; + +use backtrace::Backtrace; +use http::StatusCode; +use syncserver_common::{from_error, impl_fmt_display, InternalError}; +use syncserver_db_common::error::MysqlError; +use thiserror::Error; +use tokenserver_common::TokenserverError; + +pub(crate) type DbFuture<'a, T> = syncserver_db_common::DbFuture<'a, T, DbError>; +pub(crate) type DbResult = Result; + +/// An error type that represents any database-related errors that may occur while processing a +/// tokenserver request. +#[derive(Debug)] +pub struct DbError { + kind: DbErrorKind, + pub status: StatusCode, + pub backtrace: Box, +} + +impl DbError { + pub(crate) fn internal(msg: String) -> Self { + DbErrorKind::Internal(msg).into() + } +} + +#[derive(Debug, Error)] +enum DbErrorKind { + #[error("{}", _0)] + Mysql(MysqlError), + + #[error("Unexpected error: {}", _0)] + Internal(String), +} + +impl From for DbError { + fn from(kind: DbErrorKind) -> Self { + match kind { + DbErrorKind::Mysql(ref mysql_error) => Self { + status: mysql_error.status, + backtrace: Box::new(mysql_error.backtrace.clone()), + kind, + }, + DbErrorKind::Internal(_) => Self { + kind, + status: StatusCode::INTERNAL_SERVER_ERROR, + backtrace: Box::new(Backtrace::new()), + }, + } + } +} + +impl From for TokenserverError { + fn from(db_error: DbError) -> Self { + TokenserverError { + description: db_error.to_string(), + context: db_error.to_string(), + backtrace: db_error.backtrace, + http_status: if db_error.status.is_server_error() { + // Use the status code from the DbError if it already suggests an internal error; + // it might be more specific than `StatusCode::SERVICE_UNAVAILABLE` + db_error.status + } else { + StatusCode::SERVICE_UNAVAILABLE + }, + // An unhandled DbError in the Tokenserver code is an internal error + ..TokenserverError::internal_error() + } + } +} + +impl InternalError for DbError { + fn internal_error(message: String) -> Self { + DbErrorKind::Internal(message).into() + } +} + +impl_fmt_display!(DbError, DbErrorKind); + +from_error!( + diesel::result::Error, + DbError, + |error: diesel::result::Error| DbError::from(DbErrorKind::Mysql(MysqlError::from(error))) +); +from_error!( + diesel::result::ConnectionError, + DbError, + |error: diesel::result::ConnectionError| DbError::from(DbErrorKind::Mysql(MysqlError::from( + error + ))) +); +from_error!( + diesel::r2d2::PoolError, + DbError, + |error: diesel::r2d2::PoolError| DbError::from(DbErrorKind::Mysql(MysqlError::from(error))) +); +from_error!( + diesel_migrations::RunMigrationsError, + DbError, + |error: diesel_migrations::RunMigrationsError| DbError::from(DbErrorKind::Mysql( + MysqlError::from(error) + )) +); diff --git a/tokenserver-db/src/lib.rs b/tokenserver-db/src/lib.rs new file mode 100644 index 00000000..1b9f86c6 --- /dev/null +++ b/tokenserver-db/src/lib.rs @@ -0,0 +1,13 @@ +extern crate diesel; +#[macro_use] +extern crate diesel_migrations; + +mod error; +pub mod mock; +mod models; +pub mod params; +mod pool; +pub mod results; + +pub use models::{Db, TokenserverDb}; +pub use pool::{DbPool, TokenserverPool}; diff --git a/syncserver/src/tokenserver/db/mock.rs b/tokenserver-db/src/mock.rs similarity index 97% rename from syncserver/src/tokenserver/db/mock.rs rename to tokenserver-db/src/mock.rs index 871e6a4e..29041091 100644 --- a/syncserver/src/tokenserver/db/mock.rs +++ b/tokenserver-db/src/mock.rs @@ -2,9 +2,10 @@ use async_trait::async_trait; use futures::future; -use syncserver_db_common::{error::DbError, GetPoolState, PoolState}; +use syncserver_db_common::{GetPoolState, PoolState}; -use super::models::{Db, DbFuture}; +use super::error::{DbError, DbFuture}; +use super::models::Db; use super::params; use super::pool::DbPool; use super::results; diff --git a/syncserver/src/tokenserver/db/models.rs b/tokenserver-db/src/models.rs similarity index 97% rename from syncserver/src/tokenserver/db/models.rs rename to tokenserver-db/src/models.rs index a94be926..f1cc51de 100644 --- a/syncserver/src/tokenserver/db/models.rs +++ b/tokenserver-db/src/models.rs @@ -1,4 +1,3 @@ -use actix_web::http::StatusCode; use diesel::{ mysql::MysqlConnection, r2d2::{ConnectionManager, PooledConnection}, @@ -7,25 +6,24 @@ use diesel::{ }; #[cfg(test)] use diesel_logger::LoggingConnection; -use futures::future::LocalBoxFuture; -use syncserver_db_common::error::DbError; +use http::StatusCode; +use syncserver_common::{BlockingThreadpool, Metrics}; +use syncserver_db_common::{sync_db_method, DbFuture}; use std::{ - result, sync::Arc, time::{SystemTime, UNIX_EPOCH}, }; -use super::{params, results}; -use crate::server::{metrics::Metrics, BlockingThreadpool}; -use crate::sync_db_method; +use super::{ + error::{DbError, DbResult}, + params, results, +}; /// The maximum possible generation number. Used as a tombstone to mark users that have been /// "retired" from the db. const MAX_GENERATION: i64 = i64::MAX; -pub type DbFuture<'a, T> = LocalBoxFuture<'a, Result>; -pub type DbResult = result::Result; type Conn = PooledConnection>; #[derive(Clone)] @@ -37,7 +35,7 @@ pub struct TokenserverDb { /// the thread pool but does not provide Send as the underlying db /// conn. structs are !Sync (Arc requires both for Send). See the Send impl /// below. - pub(super) inner: Arc, + inner: Arc, metrics: Metrics, service_id: Option, spanner_node_id: Option, @@ -49,7 +47,7 @@ pub struct TokenserverDb { /// queued to the thread pool via Futures, naturally serialized. unsafe impl Send for TokenserverDb {} -pub struct DbInner { +struct DbInner { #[cfg(not(test))] pub(super) conn: Conn, #[cfg(test)] @@ -255,7 +253,7 @@ impl TokenserverDb { .get_result::(&self.inner.conn) .map_err(|e| { let mut db_error = - DbError::internal(&format!("unable to get Spanner node: {}", e)); + DbError::internal(format!("unable to get Spanner node: {}", e)); db_error.status = StatusCode::SERVICE_UNAVAILABLE; db_error }) @@ -289,7 +287,7 @@ impl TokenserverDb { } } - let mut db_error = DbError::internal("unable to get a node"); + let mut db_error = DbError::internal("unable to get a node".to_owned()); db_error.status = StatusCode::SERVICE_UNAVAILABLE; Err(db_error) } @@ -463,7 +461,7 @@ impl TokenserverDb { // The most up-to-date user doesn't have a node and is retired. This is an internal // service error for compatibility reasons (the legacy Tokenserver returned an // internal service error in this situation). - (_, None) => Err(DbError::internal("Tokenserver user retired")), + (_, None) => Err(DbError::internal("Tokenserver user retired".to_owned())), } } } @@ -682,7 +680,7 @@ impl Db for TokenserverDb { #[cfg(test)] sync_db_method!(get_user, get_user_sync, GetUser); - fn check(&self) -> DbFuture<'_, results::Check> { + fn check(&self) -> DbFuture<'_, results::Check, DbError> { let db = self.clone(); Box::pin(self.blocking_threadpool.spawn(move || db.check_sync())) } @@ -718,63 +716,82 @@ impl Db for TokenserverDb { } pub trait Db { - fn replace_user(&self, params: params::ReplaceUser) -> DbFuture<'_, results::ReplaceUser>; + fn replace_user( + &self, + params: params::ReplaceUser, + ) -> DbFuture<'_, results::ReplaceUser, DbError>; - fn replace_users(&self, params: params::ReplaceUsers) -> DbFuture<'_, results::ReplaceUsers>; + fn replace_users( + &self, + params: params::ReplaceUsers, + ) -> DbFuture<'_, results::ReplaceUsers, DbError>; - fn post_user(&self, params: params::PostUser) -> DbFuture<'_, results::PostUser>; + fn post_user(&self, params: params::PostUser) -> DbFuture<'_, results::PostUser, DbError>; - fn put_user(&self, params: params::PutUser) -> DbFuture<'_, results::PutUser>; + fn put_user(&self, params: params::PutUser) -> DbFuture<'_, results::PutUser, DbError>; - fn check(&self) -> DbFuture<'_, results::Check>; + fn check(&self) -> DbFuture<'_, results::Check, DbError>; - fn get_node_id(&self, params: params::GetNodeId) -> DbFuture<'_, results::GetNodeId>; + fn get_node_id(&self, params: params::GetNodeId) -> DbFuture<'_, results::GetNodeId, DbError>; - fn get_best_node(&self, params: params::GetBestNode) -> DbFuture<'_, results::GetBestNode>; + fn get_best_node( + &self, + params: params::GetBestNode, + ) -> DbFuture<'_, results::GetBestNode, DbError>; fn add_user_to_node( &self, params: params::AddUserToNode, - ) -> DbFuture<'_, results::AddUserToNode>; + ) -> DbFuture<'_, results::AddUserToNode, DbError>; - fn get_users(&self, params: params::GetUsers) -> DbFuture<'_, results::GetUsers>; + fn get_users(&self, params: params::GetUsers) -> DbFuture<'_, results::GetUsers, DbError>; fn get_or_create_user( &self, params: params::GetOrCreateUser, - ) -> DbFuture<'_, results::GetOrCreateUser>; + ) -> DbFuture<'_, results::GetOrCreateUser, DbError>; - fn get_service_id(&self, params: params::GetServiceId) -> DbFuture<'_, results::GetServiceId>; + fn get_service_id( + &self, + params: params::GetServiceId, + ) -> DbFuture<'_, results::GetServiceId, DbError>; #[cfg(test)] fn set_user_created_at( &self, params: params::SetUserCreatedAt, - ) -> DbFuture<'_, results::SetUserCreatedAt>; + ) -> DbFuture<'_, results::SetUserCreatedAt, DbError>; #[cfg(test)] fn set_user_replaced_at( &self, params: params::SetUserReplacedAt, - ) -> DbFuture<'_, results::SetUserReplacedAt>; + ) -> DbFuture<'_, results::SetUserReplacedAt, DbError>; #[cfg(test)] - fn get_user(&self, params: params::GetUser) -> DbFuture<'_, results::GetUser>; + fn get_user(&self, params: params::GetUser) -> DbFuture<'_, results::GetUser, DbError>; #[cfg(test)] - fn post_node(&self, params: params::PostNode) -> DbFuture<'_, results::PostNode>; + fn post_node(&self, params: params::PostNode) -> DbFuture<'_, results::PostNode, DbError>; #[cfg(test)] - fn get_node(&self, params: params::GetNode) -> DbFuture<'_, results::GetNode>; + fn get_node(&self, params: params::GetNode) -> DbFuture<'_, results::GetNode, DbError>; #[cfg(test)] - fn unassign_node(&self, params: params::UnassignNode) -> DbFuture<'_, results::UnassignNode>; + fn unassign_node( + &self, + params: params::UnassignNode, + ) -> DbFuture<'_, results::UnassignNode, DbError>; #[cfg(test)] - fn remove_node(&self, params: params::RemoveNode) -> DbFuture<'_, results::RemoveNode>; + fn remove_node(&self, params: params::RemoveNode) + -> DbFuture<'_, results::RemoveNode, DbError>; #[cfg(test)] - fn post_service(&self, params: params::PostService) -> DbFuture<'_, results::PostService>; + fn post_service( + &self, + params: params::PostService, + ) -> DbFuture<'_, results::PostService, DbError>; } #[cfg(test)] @@ -786,7 +803,7 @@ mod tests { use syncserver_settings::Settings; - use crate::tokenserver::db::pool::{DbPool, TokenserverPool}; + use crate::pool::{DbPool, TokenserverPool}; #[tokio::test] async fn test_update_generation() -> DbResult<()> { diff --git a/syncserver/src/tokenserver/db/params.rs b/tokenserver-db/src/params.rs similarity index 100% rename from syncserver/src/tokenserver/db/params.rs rename to tokenserver-db/src/params.rs diff --git a/syncserver/src/tokenserver/db/pool.rs b/tokenserver-db/src/pool.rs similarity index 90% rename from syncserver/src/tokenserver/db/pool.rs rename to tokenserver-db/src/pool.rs index cf8f2c3f..dd100abb 100644 --- a/syncserver/src/tokenserver/db/pool.rs +++ b/tokenserver-db/src/pool.rs @@ -4,25 +4,27 @@ use async_trait::async_trait; use diesel::{ mysql::MysqlConnection, r2d2::{ConnectionManager, Pool}, + Connection, }; use diesel_logger::LoggingConnection; -use syncserver_db_common::{error::DbError, GetPoolState, PoolState}; +use syncserver_common::{BlockingThreadpool, Metrics}; +#[cfg(debug_assertions)] +use syncserver_db_common::test::TestTransactionCustomizer; +use syncserver_db_common::{GetPoolState, PoolState}; use tokenserver_settings::Settings; -use super::models::{Db, DbResult, TokenserverDb}; -use crate::diesel::Connection; -use crate::server::{metrics::Metrics, BlockingThreadpool}; +use super::{ + error::{DbError, DbResult}, + models::{Db, TokenserverDb}, +}; -#[cfg(test)] -use crate::db::mysql::TestTransactionCustomizer; - -embed_migrations!("src/tokenserver/migrations"); +embed_migrations!(); /// Run the diesel embedded migrations /// /// Mysql DDL statements implicitly commit which could disrupt MysqlPool's /// begin_test_transaction during tests. So this runs on its own separate conn. -pub fn run_embedded_migrations(database_url: &str) -> DbResult<()> { +fn run_embedded_migrations(database_url: &str) -> DbResult<()> { let conn = MysqlConnection::establish(database_url)?; embedded_migrations::run(&LoggingConnection::new(conn))?; @@ -60,7 +62,7 @@ impl TokenserverPool { )) .min_idle(settings.database_pool_min_idle); - #[cfg(test)] + #[cfg(debug_assertions)] let builder = if _use_test_transactions { builder.connection_customizer(Box::new(TestTransactionCustomizer)) } else { diff --git a/syncserver/src/tokenserver/db/results.rs b/tokenserver-db/src/results.rs similarity index 100% rename from syncserver/src/tokenserver/db/results.rs rename to tokenserver-db/src/results.rs