diff --git a/CHANGELOG.md b/CHANGELOG.md index 67095acd..1f09ef36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ + +## 0.1.4 (2019-10-18) + + +#### Bug Fixes + +* switch sentry to its curl transport ([5cbd1974](https://github.com/mozilla-services/syncstorage-rs/commit/5cbd19744c13ef59f7fb0ba995231879c7a050d6), closes [#289](https://github.com/mozilla-services/syncstorage-rs/issues/289)) +* accept weighted content-type headers ([f3899695](https://github.com/mozilla-services/syncstorage-rs/commit/f389969517e60d41774ce71c4e7093a79c642ddd), closes [#287](https://github.com/mozilla-services/syncstorage-rs/issues/287)) + + + ## 0.1.2 (2019-10-12) diff --git a/Cargo.lock b/Cargo.lock index 086afcbb..45b013e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -632,6 +632,34 @@ dependencies = [ "subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "curl" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "curl-sys 0.4.23 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.49 (registry+https://github.com/rust-lang/crates.io-index)", + "schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "curl-sys" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.49 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "debugid" version = "0.4.0" @@ -1190,6 +1218,17 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "libz-sys" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -2000,6 +2039,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace 0.3.37 (registry+https://github.com/rust-lang/crates.io-index)", + "curl 0.4.25 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2013,6 +2053,7 @@ dependencies = [ "reqwest 0.9.20 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "sentry-types 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "uname 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2297,7 +2338,7 @@ dependencies = [ [[package]] name = "syncstorage" -version = "0.1.2" +version = "0.1.4" dependencies = [ "actix-cors 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-http 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2967,6 +3008,8 @@ dependencies = [ "checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b" "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" "checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +"checksum curl 0.4.25 (registry+https://github.com/rust-lang/crates.io-index)" = "06aa71e9208a54def20792d877bc663d6aae0732b9852e612c4a933177c31283" +"checksum curl-sys 0.4.23 (registry+https://github.com/rust-lang/crates.io-index)" = "f71cd2dbddb49c744c1c9e0b96106f50a634e8759ec51bcd5399a578700a3ab3" "checksum debugid 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "088c9627adec1e494ff9dea77377f1e69893023d631254a0ec68b16ee20be3e9" "checksum derive_more 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6d944ac6003ed268757ef1ee686753b57efc5fcf0ebe7b64c9fc81e7e32ff839" "checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe" @@ -3029,6 +3072,7 @@ dependencies = [ "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" "checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" diff --git a/Cargo.toml b/Cargo.toml index 72d3222a..c99e11bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "syncstorage" -version = "0.1.2" +version = "0.1.4" license = "MPL-2.0" authors = [ "Ben Bangert ", @@ -43,7 +43,7 @@ num_cpus = "1.10" protobuf = "2.7.0" rand = "0.7" regex = "1.3" -sentry = "0.17.0" +sentry = { version = "0.17.0", features = ["with_curl_transport"] } serde = "1.0" serde_derive = "1.0" serde_json = { version = "1.0", features = ["arbitrary_precision"] } diff --git a/Dockerfile b/Dockerfile index fd4ac633..b19fb032 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ RUN \ groupadd --gid 10001 app && \ useradd --uid 10001 --gid 10001 --home /app --create-home app && \ apt-get -q update && \ - apt-get -q install -y default-libmysqlclient-dev libssl-dev ca-certificates && \ + apt-get -q install -y default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 && \ rm -rf /var/lib/apt/lists COPY --from=builder /app/bin /app/bin diff --git a/db-tests/Cargo.lock b/db-tests/Cargo.lock index 17b09aec..1bb18192 100644 --- a/db-tests/Cargo.lock +++ b/db-tests/Cargo.lock @@ -650,7 +650,7 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syncstorage 0.1.0", + "syncstorage 0.1.2", ] [[package]] @@ -2405,7 +2405,7 @@ dependencies = [ [[package]] name = "syncstorage" -version = "0.1.0" +version = "0.1.2" dependencies = [ "actix-cors 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-http 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/spanner-2019-10-01.ddl b/spanner-2019-10-01.ddl index 2a10e263..24a5a439 100644 --- a/spanner-2019-10-01.ddl +++ b/spanner-2019-10-01.ddl @@ -2,7 +2,7 @@ -- usually a UUID, so presuming a formatted form. -- fxa_kid: <`mono_num`>-<`client_state`> -- --- - mono_num: a monotonically increasing timestamp or generation number +-- - mono_num: a monotonically increasing timestamp or generation number -- in hex and padded to 13 digits, provided by the fxa server -- - client_state: the first 16 bytes of a SHA256 hash of the user's sync -- encryption key. @@ -11,17 +11,17 @@ -- ALSO, CONSOLE WANTS ONE SPACE BETWEEN DDL COMMANDS CREATE TABLE user_collections ( - fxa_uid STRING(36) NOT NULL, - fxa_kid STRING(48) NOT NULL, + fxa_uid STRING(MAX) NOT NULL, + fxa_kid STRING(MAX) NOT NULL, collection_id INT64 NOT NULL, modified TIMESTAMP NOT NULL, ) PRIMARY KEY(fxa_uid, fxa_kid, collection_id); CREATE TABLE bso ( - fxa_uid STRING(36) NOT NULL, - fxa_kid STRING(48) NOT NULL, + fxa_uid STRING(MAX) NOT NULL, + fxa_kid STRING(MAX) NOT NULL, collection_id INT64 NOT NULL, - id STRING(64) NOT NULL, + id STRING(64) NOT NULL, sortindex INT64, payload STRING(MAX) NOT NULL, modified TIMESTAMP NOT NULL, @@ -38,15 +38,30 @@ INTERLEAVE IN user_collections; CREATE TABLE collections ( id INT64 NOT NULL, - name STRING(32) NOT NULL, + name STRING(32) NOT NULL, ) PRIMARY KEY(id); CREATE UNIQUE INDEX CollectionName ON collections(name); +INSERT INTO collections (id, name) VALUES + ( 1, "clients"), + ( 2, "crypto"), + ( 3, "forms"), + ( 4, "history"), + ( 5, "keys"), + ( 6, "meta"), + ( 7, "bookmarks"), + ( 8, "prefs"), + ( 9, "tabs"), + (10, "passwords"), + (11, "addons"), + (12, "addresses"), + (13, "creditcards"); + CREATE TABLE batches ( - fxa_uid STRING(36) NOT NULL, - fxa_kid STRING(48) NOT NULL, + fxa_uid STRING(MAX) NOT NULL, + fxa_kid STRING(MAX) NOT NULL, id TIMESTAMP NOT NULL, collection_id INT64 NOT NULL, bsos STRING(MAX) NOT NULL, diff --git a/src/main.rs b/src/main.rs index 84ee4e61..7d6b5e25 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,8 +23,17 @@ struct Args { fn main() -> Result<(), Box> { env_logger::init(); debug!("Starting up..."); - // Set SENTRY_DSN environment variable to enable Sentry - let sentry = sentry::init(sentry::ClientOptions::default()); + // Set SENTRY_DSN environment variable to enable Sentry. + // Avoid its default reqwest transport for now due to issues w/ + // likely grpcio's boringssl + let curl_transport_factory = |options: &sentry::ClientOptions| { + Box::new(sentry::transports::CurlHttpTransport::new(options)) + as Box + }; + let sentry = sentry::init(sentry::ClientOptions { + transport: Box::new(curl_transport_factory), + ..sentry::ClientOptions::default() + }); if sentry.is_enabled() { sentry::integrations::panic::register_panic_handler(); } diff --git a/src/web/extractors.rs b/src/web/extractors.rs index 3436048e..f451506a 100644 --- a/src/web/extractors.rs +++ b/src/web/extractors.rs @@ -42,6 +42,9 @@ const BSO_MAX_TTL: u32 = 31_536_000; const BSO_MAX_SORTINDEX_VALUE: i32 = 999_999_999; const BSO_MIN_SORTINDEX_VALUE: i32 = -999_999_999; +const ACCEPTED_CONTENT_TYPES: [&str; 3] = + ["application/json", "text/plain", "application/newlines"]; + lazy_static! { static ref KNOWN_BAD_PAYLOAD_REGEX: Regex = Regex::new(r#"IV":\s*"AAAAAAAAAAAAAAAAAAAAAA=="#).unwrap(); @@ -88,11 +91,46 @@ impl BatchBsoBody { } } -fn get_trimmed_header(headers: &HeaderMap, key: HeaderName, default: &HeaderValue) -> String { - let ct_raw = std::str::from_utf8(headers.get(key).unwrap_or(&default).as_bytes()) +// This tries to do the right thing to get the Accepted header according to +// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept, but some corners can absolutely be cut. +// This will pull the first accepted content type listed, or the highest rated non-accepted type. +fn get_weighted_header( + headers: &HeaderMap, + key: HeaderName, + accepted: &[&str], + default: &'static str, +) -> String { + let def_hv = HeaderValue::from_static(default); + let hv_raw = std::str::from_utf8(headers.get(key).unwrap_or(&def_hv).as_bytes()) .unwrap_or_else(|_| "invalid"); - let ct_parts: Vec<&str> = ct_raw.split(';').collect(); - ct_parts[0].trim_end().to_owned() + let hv_parts = hv_raw.split(','); + let mut choice_weight = 0.0; + let mut pick: String = default.to_owned(); + for choice in hv_parts { + let opt_weight: Vec<&str> = choice.split(';').collect(); + let mut opt = opt_weight[0].trim_end().to_lowercase(); + if opt == "*/*" { + opt = default.to_owned() + }; + let weight = if opt_weight.len() > 1 { + let weights: Vec<&str> = opt_weight[1].split('=').collect(); + // if the weight is malformed, ignore this choice. + if weights.len() < 2 { + continue; + } + f32::from_str(weights[1]).unwrap_or(0.0) + } else { + 1.0 + }; + if weight.abs().trunc() > 0.0 && accepted.contains(&&opt.as_ref()) { + return opt; + } + if weight > choice_weight { + pick = opt; + choice_weight = weight; + } + } + pick } #[derive(Default, Deserialize)] @@ -118,23 +156,24 @@ impl FromRequest for BsoBodies { fn from_request(req: &HttpRequest, payload: &mut Payload) -> Self::Future { // Only try and parse the body if its a valid content-type let headers = req.headers(); - let default = HeaderValue::from_static(""); - let content_type = get_trimmed_header(headers, CONTENT_TYPE, &default); + let content_type = get_weighted_header( + headers, + CONTENT_TYPE, + &ACCEPTED_CONTENT_TYPES, + "application/json", + ); debug!("content_type: {:?}", &content_type); - match content_type.as_str() { - "application/json" | "text/plain" | "application/newlines" | "" => (), - _ => { - return Box::new(future::err( - ValidationErrorKind::FromDetails( - format!("Invalid Content-Type {:?}", content_type), - RequestErrorLocation::Header, - Some("Content-Type".to_owned()), - ) - .into(), - )); - } + if !ACCEPTED_CONTENT_TYPES.contains(&content_type.as_str()) { + return Box::new(future::err( + ValidationErrorKind::FromDetails( + format!("Invalid Content-Type {:?}", content_type), + RequestErrorLocation::Header, + Some("Content-Type".to_owned()), + ) + .into(), + )); } // Load the entire request into a String @@ -181,7 +220,7 @@ impl FromRequest for BsoBodies { let max_post_bytes = state.limits.max_post_bytes as usize; let fut = fut.and_then(move |body| { - // Get all the raw JSON values + // Get all the raw / values let bsos: Vec = if newlines { let mut bsos = Vec::new(); for item in body.lines() { @@ -298,20 +337,21 @@ impl FromRequest for BsoBody { // Only try and parse the body if its a valid content-type let headers = req.headers(); - let default = HeaderValue::from_static(""); - let content_type = get_trimmed_header(&headers, CONTENT_TYPE, &default); - match content_type.as_str() { - "application/json" | "text/plain" | "" => (), - _ => { - return Box::new(future::err( - ValidationErrorKind::FromDetails( - "Invalid Content-Type".to_owned(), - RequestErrorLocation::Header, - Some("Content-Type".to_owned()), - ) - .into(), - )); - } + let content_type = get_weighted_header( + &headers, + CONTENT_TYPE, + &ACCEPTED_CONTENT_TYPES, + "application/json", + ); + if !ACCEPTED_CONTENT_TYPES.contains(&content_type.as_str()) { + return Box::new(future::err( + ValidationErrorKind::FromDetails( + "Invalid Content-Type".to_owned(), + RequestErrorLocation::Header, + Some("Content-Type".to_owned()), + ) + .into(), + )); } let state = match req.app_data::() { Some(s) => s, @@ -561,8 +601,12 @@ impl FromRequest for CollectionRequest { let db = >::from_request(req, payload)?; let query = BsoQueryParams::from_request(req, payload)?; let collection = CollectionParam::from_request(req, payload)?.collection; - let content_type = - get_trimmed_header(&req.headers(), ACCEPT, &HeaderValue::from_static("")); + let content_type = get_weighted_header( + &req.headers(), + ACCEPT, + &ACCEPTED_CONTENT_TYPES, + "application/json", + ); let reply = match content_type.as_str() { "application/newlines" => ReplyFormat::Newlines, "application/json" | "" => ReplyFormat::Json, @@ -1532,6 +1576,62 @@ mod tests { */ } + #[test] + fn test_weighted_header() { + // test non-priority, full weight selection + let mut header_map = HeaderMap::new(); + header_map.insert( + CONTENT_TYPE, + HeaderValue::from_static("application/json;q=0.9,text/plain"), + ); + let selected = get_weighted_header( + &header_map, + CONTENT_TYPE, + &ACCEPTED_CONTENT_TYPES, + "application/json", + ); + assert_eq!(selected, "text/plain".to_owned()); + + // test default for */* + let mut header_map = HeaderMap::new(); + header_map.insert(CONTENT_TYPE, HeaderValue::from_static("*/*,text/plain")); + let selected = get_weighted_header( + &header_map, + CONTENT_TYPE, + &ACCEPTED_CONTENT_TYPES, + "application/json", + ); + assert_eq!(selected, "application/json".to_owned()); + + // test default for selected weighted. + let mut header_map = HeaderMap::new(); + header_map.insert( + CONTENT_TYPE, + HeaderValue::from_static("foo/bar;q=0.1,application/json;q=0.2,text/plain;q=0.3"), + ); + let selected = get_weighted_header( + &header_map, + CONTENT_TYPE, + &ACCEPTED_CONTENT_TYPES, + "application/json", + ); + assert_eq!(selected, "text/plain".to_owned()); + + // test default for selected weighted. + let mut header_map = HeaderMap::new(); + header_map.insert( + CONTENT_TYPE, + HeaderValue::from_static("foo/bar;0.1,text/plain;q=0.1"), + ); + let selected = get_weighted_header( + &header_map, + CONTENT_TYPE, + &ACCEPTED_CONTENT_TYPES, + "text/plain", + ); + assert_eq!(selected, "text/plain".to_owned()); + } + #[test] fn test_valid_query_args() { let req = TestRequest::with_uri("/?ids=1,2&full=&sort=index&older=2.43") @@ -1675,7 +1775,7 @@ mod tests { let req = TestRequest::with_uri(&uri) .data(state) .header("authorization", header) - .header("accept", "application/json;a=0.9,/;q=0.2") + .header("accept", "application/json;a=0.9,*/*;q=0.2") .method(Method::GET) .param("uid", &USER_ID_STR) .param("collection", "tabs")