diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 307d9a6a..a409af97 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -1,3 +1,2 @@ [advisories] -ignore = [ -] +ignore = [] diff --git a/.dockerignore b/.dockerignore index 5ef407a2..b9eefd9f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,3 +10,4 @@ PULL_REQUEST_TEMPLATE.md README.md target tools/examples +db/ diff --git a/Cargo.lock b/Cargo.lock index 451d5bc0..331f1b74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,11 +4,11 @@ version = 3 [[package]] name = "actix-codec" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8" +checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "bytes", "futures-core", "futures-sink", @@ -36,9 +36,9 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.5.1" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "129d4c88e98860e1758c5de288d1632b07970a16d59bdf7b8d66053d582bb71f" +checksum = "d223b13fd481fc0d1f83bb12659ae774d9e3601814c68a0bc539731698cca743" dependencies = [ "actix-codec", "actix-rt", @@ -92,7 +92,7 @@ dependencies = [ "bytestring", "http", "regex", - "serde 1.0.195", + "serde 1.0.196", "tracing", ] @@ -119,7 +119,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2 0.5.5", + "socket2", "tokio", "tracing", ] @@ -147,9 +147,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.4.1" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e43428f3bf11dee6d166b00ec2df4e3aa8cc1606aaa0b7433c146852e2f4e03b" +checksum = "43a6556ddebb638c2358714d853257ed226ece6023ef9364f23f0c70737ea984" dependencies = [ "actix-codec", "actix-http", @@ -176,11 +176,11 @@ dependencies = [ "once_cell", "pin-project-lite", "regex", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.5.5", + "socket2", "time", "url 2.5.0", ] @@ -214,9 +214,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" dependencies = [ "cfg-if", "getrandom", @@ -297,7 +297,7 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" dependencies = [ - "serde 1.0.195", + "serde 1.0.196", "serde_json", ] @@ -487,16 +487,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", - "num-traits 0.2.17", + "num-traits 0.2.18", "wasm-bindgen", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -552,7 +552,7 @@ checksum = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" dependencies = [ "lazy_static", "nom 5.1.3", - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -564,7 +564,7 @@ dependencies = [ "lazy_static", "nom 5.1.3", "rust-ini", - "serde 1.0.195", + "serde 1.0.196", "serde-hjson", "serde_json", "toml", @@ -615,9 +615,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -658,24 +658,24 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.44" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509bd11746c7ac09ebd19f0b17782eae80aadee26237658a6b4808afb5c11a22" +checksum = "1e2161dd6eba090ff1594084e95fd67aeccf04382ffea77999ea94ed42ec67b6" dependencies = [ "curl-sys", "libc", "openssl-probe", "openssl-sys", "schannel", - "socket2 0.4.10", - "winapi", + "socket2", + "windows-sys 0.52.0", ] [[package]] name = "curl-sys" -version = "0.4.70+curl-8.5.0" +version = "0.4.72+curl-8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0333d8849afe78a4c8102a429a446bfdd055832af071945520e835ae2d841e" +checksum = "29cbdc8314c447d11e8fd156dcdd031d9e02a7a976163e396b548c03153bc9ea" dependencies = [ "cc", "libc", @@ -683,7 +683,7 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -695,7 +695,7 @@ dependencies = [ "config 0.10.1", "crossbeam-queue", "num_cpus", - "serde 1.0.195", + "serde 1.0.196", "tokio", ] @@ -705,7 +705,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" dependencies = [ - "serde 1.0.195", + "serde 1.0.196", "uuid", ] @@ -821,7 +821,7 @@ checksum = "7f3f119846c823f9eafcf953a8f6ffb6ed69bf6240883261a7f13b634579a51f" dependencies = [ "lazy_static", "regex", - "serde 1.0.195", + "serde 1.0.196", "strsim 0.10.0", ] @@ -833,9 +833,9 @@ checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "encoding_rs" @@ -884,7 +884,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" dependencies = [ - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -1164,9 +1164,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" [[package]] name = "hex" @@ -1269,7 +1269,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2", "tokio", "tower-service", "tracing", @@ -1292,9 +1292,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1352,9 +1352,9 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown", @@ -1374,12 +1374,12 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" dependencies = [ - "hermit-abi 0.3.4", - "rustix", + "hermit-abi 0.3.6", + "libc", "windows-sys 0.52.0", ] @@ -1391,18 +1391,18 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -1416,7 +1416,7 @@ dependencies = [ "base64", "js-sys", "ring", - "serde 1.0.195", + "serde 1.0.196", "serde_json", ] @@ -1453,9 +1453,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" @@ -1480,9 +1480,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.14" +version = "1.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" +checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" dependencies = [ "cc", "libc", @@ -1597,9 +1597,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -1665,20 +1665,26 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" dependencies = [ - "num-traits 0.2.17", + "num-traits 0.2.18", ] [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", ] @@ -1689,7 +1695,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.4", + "hermit-abi 0.3.6", "libc", ] @@ -1725,9 +1731,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.98" +version = "0.9.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" +checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" dependencies = [ "cc", "libc", @@ -1742,7 +1748,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006e42d5b888366f1880eda20371fedde764ed2213dc8496f49622fa0c99cd5e" dependencies = [ "log", - "serde 1.0.195", + "serde 1.0.196", "winapi", ] @@ -1849,9 +1855,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -1995,9 +2001,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.2" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -2007,9 +2013,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -2024,9 +2030,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64", "bytes", @@ -2047,9 +2053,10 @@ dependencies = [ "pin-project-lite", "rustls", "rustls-pemfile", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls", @@ -2105,9 +2112,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -2258,7 +2265,7 @@ dependencies = [ "once_cell", "rand", "sentry-types", - "serde 1.0.195", + "serde 1.0.196", "serde_json", ] @@ -2294,7 +2301,7 @@ dependencies = [ "debugid", "hex", "rand", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "thiserror", "time", @@ -2310,9 +2317,9 @@ checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -2331,9 +2338,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -2342,13 +2349,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -2360,7 +2367,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -2452,7 +2459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f400f1c5db96f1f52065e8931ca0c524cceb029f7537c9e6d5424488ca137ca0" dependencies = [ "chrono", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "slog", ] @@ -2494,19 +2501,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b187f0231d56fe41bfb12034819dd2bf336422a5866de41bc3fec4b2e3883e8" - -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" @@ -2570,6 +2567,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "syncserver" version = "0.14.4" @@ -2596,7 +2599,7 @@ dependencies = [ "regex", "sentry", "sentry-backtrace", - "serde 1.0.195", + "serde 1.0.196", "serde_derive", "serde_json", "sha2", @@ -2633,7 +2636,7 @@ dependencies = [ "cadence", "futures 0.3.30", "hkdf", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "sha2", "slog", @@ -2660,7 +2663,7 @@ version = "0.14.4" dependencies = [ "config 0.11.0", "num_cpus", - "serde 1.0.195", + "serde 1.0.196", "slog-scope", "syncserver-common", "syncstorage-settings", @@ -2703,7 +2706,7 @@ dependencies = [ "futures 0.3.30", "http", "lazy_static", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "syncserver-common", "syncserver-db-common", @@ -2738,7 +2741,7 @@ name = "syncstorage-settings" version = "0.14.4" dependencies = [ "rand", - "serde 1.0.195", + "serde 1.0.196", "syncserver-common", "time", ] @@ -2834,18 +2837,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", @@ -2864,16 +2867,17 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", - "serde 1.0.195", + "serde 1.0.196", "time-core", "time-macros", ] @@ -2886,10 +2890,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -2924,7 +2929,7 @@ dependencies = [ "pyo3", "reqwest", "ring", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "sha2", "slog-scope", @@ -2942,7 +2947,7 @@ dependencies = [ "actix-web", "backtrace", "jsonwebtoken", - "serde 1.0.195", + "serde 1.0.196", "serde_json", "syncserver-common", "thiserror", @@ -2960,7 +2965,7 @@ dependencies = [ "env_logger 0.10.2", "futures 0.3.30", "http", - "serde 1.0.195", + "serde 1.0.196", "serde_derive", "serde_json", "slog-scope", @@ -2978,15 +2983,15 @@ name = "tokenserver-settings" version = "0.14.4" dependencies = [ "jsonwebtoken", - "serde 1.0.195", + "serde 1.0.196", "tokenserver-common", ] [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -2995,7 +3000,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] @@ -3041,7 +3046,7 @@ version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" dependencies = [ - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -3160,7 +3165,7 @@ dependencies = [ "form_urlencoded", "idna 0.5.0", "percent-encoding 2.3.1", - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -3176,7 +3181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", - "serde 1.0.195", + "serde 1.0.196", ] [[package]] @@ -3188,7 +3193,7 @@ dependencies = [ "idna 0.4.0", "lazy_static", "regex", - "serde 1.0.195", + "serde 1.0.196", "serde_derive", "serde_json", "url 2.5.0", @@ -3271,9 +3276,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3281,9 +3286,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -3296,9 +3301,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -3308,9 +3313,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3318,9 +3323,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -3331,15 +3336,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -3347,9 +3352,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "which" diff --git a/tools/integration_tests/tokenserver/test_e2e.py b/tools/integration_tests/tokenserver/test_e2e.py index 85eab417..c0aa619d 100644 --- a/tools/integration_tests/tokenserver/test_e2e.py +++ b/tools/integration_tests/tokenserver/test_e2e.py @@ -16,6 +16,7 @@ from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.backends import default_backend from fxa.core import Client from fxa.oauth import Client as OAuthClient +from fxa.errors import ServerError from fxa.tests.utils import TestEmailAccount from hashlib import sha256 @@ -69,7 +70,15 @@ class TestE2e(TestCase, unittest.TestCase): @classmethod def tearDownClass(cls): cls.acct.clear() - cls.client.destroy_account(cls.acct.email, cls.fxa_password) + # A teardown of some of the tests can produce a 401 error because + # of a race condition, where the record had already been removed. + # This causes `destroy_account` to return an error if it attempts + # to parse the invalid JSON response. + # This traps for that event. + try: + cls.client.destroy_account(cls.acct.email, cls.fxa_password) + except ServerError as ex: + print(f"warning: Encountered error when cleaning up: {ex}") @staticmethod def _generate_password(): diff --git a/tools/tokenserver/purge_old_records.py b/tools/tokenserver/purge_old_records.py index b3d4df6a..81279c92 100644 --- a/tools/tokenserver/purge_old_records.py +++ b/tools/tokenserver/purge_old_records.py @@ -19,6 +19,7 @@ import binascii import hawkauthlib import logging import optparse +import os import random import requests import time @@ -28,14 +29,15 @@ import util from database import Database from util import format_key_id - logger = logging.getLogger("tokenserver.scripts.purge_old_records") +logger.setLevel(os.environ.get("PYTHON_LOG", "ERROR").upper()) PATTERN = "{node}/1.5/{uid}" def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, - request_timeout=60): + max_records=0, request_timeout=60, dryrun=False, + force=False): """Purge old records from the database. This function queries all of the old user records in the database, deletes @@ -51,6 +53,7 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, logger.info("Purging old user records") try: database = Database() + previous_list = [] # Process batches of items, until we run out. while True: offset = random.randint(0, max_offset) @@ -60,7 +63,14 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, "offset": offset, } rows = list(database.get_old_user_records(**kwds)) + if not rows: + logger.info("No more data") + break + if rows == previous_list: + raise Exception("Loop detected") + previous_list = rows logger.info("Fetched %d rows at offset %d", len(rows), offset) + counter = 0 for row in rows: # Don't attempt to purge data from downed nodes. # Instead wait for them to either come back up or to be @@ -68,11 +78,36 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, if row.node is None: logger.info("Deleting user record for uid %s on %s", row.uid, row.node) - database.delete_user_record(row.uid) + if not dryrun: + database.delete_user_record(row.uid) + # NOTE: only delete_user+service_data calls count + # against the counter elif not row.downed: logger.info("Purging uid %s on %s", row.uid, row.node) - delete_service_data(row, secret, timeout=request_timeout) - database.delete_user_record(row.uid) + if not dryrun: + delete_service_data( + row, + secret, + timeout=request_timeout, + dryrun=dryrun) + database.delete_user_record(row.uid) + counter += 1 + elif force: + logger.info( + "Forcing tokenserver record delete: " + f"{row.uid} on {row.node}" + ) + if not dryrun: + delete_service_data( + row, + secret, + timeout=request_timeout, + dryrun=dryrun) + database.delete_user_record(row.uid) + counter += 1 + if max_records and counter >= max_records: + logger.info("Reached max_records, exiting") + return True if len(rows) < max_per_loop: break except Exception: @@ -83,7 +118,7 @@ def purge_old_records(secret, grace_period=-1, max_per_loop=10, max_offset=0, return True -def delete_service_data(user, secret, timeout=60): +def delete_service_data(user, secret, timeout=60, dryrun=False): """Send a data-deletion request to the user's service node. This is a little bit of hackery to cause the user's service node to @@ -102,6 +137,11 @@ def delete_service_data(user, secret, timeout=60): secret = tokenlib.get_derived_secret(token, secret=secret) endpoint = PATTERN.format(uid=user.uid, node=user.node) auth = HawkAuth(token, secret) + if dryrun: + # NOTE: this function currently isn't called during dryrun + # (but we may want to add logging here and change that in the + # future) + return resp = requests.delete(endpoint, auth=auth, timeout=timeout) if resp.status_code >= 400 and resp.status_code != 404: resp.raise_for_status() @@ -137,12 +177,20 @@ def main(args=None): # selects will return zero rows. Choose this value accordingly. parser.add_option("", "--max-offset", type="int", default=0, help="Use random offset from 0 to max_offset") + parser.add_option("", "--max-records", type="int", default=0, + help="Max number of syncstorage data purges to " + "make") parser.add_option("", "--request-timeout", type="int", default=60, help="Timeout for service deletion requests") parser.add_option("", "--oneshot", action="store_true", help="Do a single purge run and then exit") parser.add_option("-v", "--verbose", action="count", dest="verbosity", help="Control verbosity of log messages") + parser.add_option("", "--dryrun", action="store_true", + help="Don't do destructive things") + parser.add_option("", "--force", action="store_true", + help="Force syncstorage data to be purged, even " + "if the user's node is marked as down") opts, args = parser.parse_args(args) if len(args) != 2: @@ -157,7 +205,10 @@ def main(args=None): grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, - request_timeout=opts.request_timeout) + max_records=opts.max_records, + request_timeout=opts.request_timeout, + dryrun=opts.dryrun, + force=opts.force) if not opts.oneshot: while True: # Randomize sleep interval +/- thirty percent to desynchronize @@ -169,7 +220,10 @@ def main(args=None): purge_old_records(grace_period=opts.grace_period, max_per_loop=opts.max_per_loop, max_offset=opts.max_offset, - request_timeout=opts.request_timeout) + max_records=opts.max_records, + request_timeout=opts.request_timeout, + dryrun=opts.dryrun, + force=opts.force) return 0 diff --git a/tools/tokenserver/test_purge_old_records.py b/tools/tokenserver/test_purge_old_records.py index 370e5dff..80d50693 100644 --- a/tools/tokenserver/test_purge_old_records.py +++ b/tools/tokenserver/test_purge_old_records.py @@ -138,3 +138,49 @@ class TestPurgeOldRecords(unittest.TestCase): user_records = list(self.database.get_user_records(email)) self.assertEqual(len(user_records), 1) self.assertEqual(len(self.service_requests), 1) + + def test_force(self): + # Make some old user records. + node_secret = "SECRET" + email = "test@mozilla.com" + user = self.database.allocate_user(email, client_state="aa") + self.database.update_user(user, client_state="bb") + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 2) + + # With the node down, we should be able to purge any records. + self.database.update_node(self.service_node, downed=1) + + self.assertTrue( + purge_old_records( + node_secret, + grace_period=0, + force=True) + ) + + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 1) + self.assertEqual(len(self.service_requests), 1) + + def test_dry_run(self): + # Make some old user records. + node_secret = "SECRET" + email = "test@mozilla.com" + user = self.database.allocate_user(email, client_state="aa") + self.database.update_user(user, client_state="bb") + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 2) + + self.database.update_node(self.service_node, downed=1) + + # Don't actually perform anything destructive. + self.assertTrue( + purge_old_records( + node_secret, + grace_period=0, + dryrun=True) + ) + + user_records = list(self.database.get_user_records(email)) + self.assertEqual(len(user_records), 2) + self.assertEqual(len(self.service_requests), 0)