Merge branch 'master' of github.com:mozilla-services/syncstorage-rs into tools

This commit is contained in:
jrconlin 2019-10-18 08:34:27 -07:00
commit 0deaade01f
No known key found for this signature in database
GPG Key ID: 91B7F708D9FC4D84
8 changed files with 232 additions and 53 deletions

View File

@ -1,3 +1,14 @@
<a name="0.1.4"></a>
## 0.1.4 (2019-10-18)
#### Bug Fixes
* switch sentry to its curl transport ([5cbd1974](https://github.com/mozilla-services/syncstorage-rs/commit/5cbd19744c13ef59f7fb0ba995231879c7a050d6), closes [#289](https://github.com/mozilla-services/syncstorage-rs/issues/289))
* accept weighted content-type headers ([f3899695](https://github.com/mozilla-services/syncstorage-rs/commit/f389969517e60d41774ce71c4e7093a79c642ddd), closes [#287](https://github.com/mozilla-services/syncstorage-rs/issues/287))
<a name="0.1.2"></a>
## 0.1.2 (2019-10-12)

46
Cargo.lock generated
View File

@ -632,6 +632,34 @@ dependencies = [
"subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "curl"
version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"curl-sys 0.4.23 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-sys 0.9.49 (registry+https://github.com/rust-lang/crates.io-index)",
"schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"socket2 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "curl-sys"
version = "0.4.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl-sys 0.9.49 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "debugid"
version = "0.4.0"
@ -1190,6 +1218,17 @@ dependencies = [
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libz-sys"
version = "1.0.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "linked-hash-map"
version = "0.3.0"
@ -2000,6 +2039,7 @@ version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"backtrace 0.3.37 (registry+https://github.com/rust-lang/crates.io-index)",
"curl 0.4.25 (registry+https://github.com/rust-lang/crates.io-index)",
"env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2013,6 +2053,7 @@ dependencies = [
"reqwest 0.9.20 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"sentry-types 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
"uname 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -2297,7 +2338,7 @@ dependencies = [
[[package]]
name = "syncstorage"
version = "0.1.2"
version = "0.1.4"
dependencies = [
"actix-cors 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"actix-http 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2967,6 +3008,8 @@ dependencies = [
"checksum crossbeam-queue 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7c979cd6cfe72335896575c6b5688da489e420d36a27a0b9eb0c73db574b4a4b"
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
"checksum curl 0.4.25 (registry+https://github.com/rust-lang/crates.io-index)" = "06aa71e9208a54def20792d877bc663d6aae0732b9852e612c4a933177c31283"
"checksum curl-sys 0.4.23 (registry+https://github.com/rust-lang/crates.io-index)" = "f71cd2dbddb49c744c1c9e0b96106f50a634e8759ec51bcd5399a578700a3ab3"
"checksum debugid 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "088c9627adec1e494ff9dea77377f1e69893023d631254a0ec68b16ee20be3e9"
"checksum derive_more 0.14.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6d944ac6003ed268757ef1ee686753b57efc5fcf0ebe7b64c9fc81e7e32ff839"
"checksum derive_more 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a141330240c921ec6d074a3e188a7c7ef95668bb95e7d44fa0e5778ec2a7afe"
@ -3029,6 +3072,7 @@ dependencies = [
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba"
"checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c"

View File

@ -1,6 +1,6 @@
[package]
name = "syncstorage"
version = "0.1.2"
version = "0.1.4"
license = "MPL-2.0"
authors = [
"Ben Bangert <ben@groovie.org>",
@ -43,7 +43,7 @@ num_cpus = "1.10"
protobuf = "2.7.0"
rand = "0.7"
regex = "1.3"
sentry = "0.17.0"
sentry = { version = "0.17.0", features = ["with_curl_transport"] }
serde = "1.0"
serde_derive = "1.0"
serde_json = { version = "1.0", features = ["arbitrary_precision"] }

View File

@ -19,7 +19,7 @@ RUN \
groupadd --gid 10001 app && \
useradd --uid 10001 --gid 10001 --home /app --create-home app && \
apt-get -q update && \
apt-get -q install -y default-libmysqlclient-dev libssl-dev ca-certificates && \
apt-get -q install -y default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 && \
rm -rf /var/lib/apt/lists
COPY --from=builder /app/bin /app/bin

4
db-tests/Cargo.lock generated
View File

@ -650,7 +650,7 @@ dependencies = [
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
"syncstorage 0.1.0",
"syncstorage 0.1.2",
]
[[package]]
@ -2405,7 +2405,7 @@ dependencies = [
[[package]]
name = "syncstorage"
version = "0.1.0"
version = "0.1.2"
dependencies = [
"actix-cors 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"actix-http 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",

View File

@ -2,7 +2,7 @@
-- usually a UUID, so presuming a formatted form.
-- fxa_kid: <`mono_num`>-<`client_state`>
--
-- - mono_num: a monotonically increasing timestamp or generation number
-- - mono_num: a monotonically increasing timestamp or generation number
-- in hex and padded to 13 digits, provided by the fxa server
-- - client_state: the first 16 bytes of a SHA256 hash of the user's sync
-- encryption key.
@ -11,17 +11,17 @@
-- ALSO, CONSOLE WANTS ONE SPACE BETWEEN DDL COMMANDS
CREATE TABLE user_collections (
fxa_uid STRING(36) NOT NULL,
fxa_kid STRING(48) NOT NULL,
fxa_uid STRING(MAX) NOT NULL,
fxa_kid STRING(MAX) NOT NULL,
collection_id INT64 NOT NULL,
modified TIMESTAMP NOT NULL,
) PRIMARY KEY(fxa_uid, fxa_kid, collection_id);
CREATE TABLE bso (
fxa_uid STRING(36) NOT NULL,
fxa_kid STRING(48) NOT NULL,
fxa_uid STRING(MAX) NOT NULL,
fxa_kid STRING(MAX) NOT NULL,
collection_id INT64 NOT NULL,
id STRING(64) NOT NULL,
id STRING(64) NOT NULL,
sortindex INT64,
payload STRING(MAX) NOT NULL,
modified TIMESTAMP NOT NULL,
@ -38,15 +38,30 @@ INTERLEAVE IN user_collections;
CREATE TABLE collections (
id INT64 NOT NULL,
name STRING(32) NOT NULL,
name STRING(32) NOT NULL,
) PRIMARY KEY(id);
CREATE UNIQUE INDEX CollectionName
ON collections(name);
INSERT INTO collections (id, name) VALUES
( 1, "clients"),
( 2, "crypto"),
( 3, "forms"),
( 4, "history"),
( 5, "keys"),
( 6, "meta"),
( 7, "bookmarks"),
( 8, "prefs"),
( 9, "tabs"),
(10, "passwords"),
(11, "addons"),
(12, "addresses"),
(13, "creditcards");
CREATE TABLE batches (
fxa_uid STRING(36) NOT NULL,
fxa_kid STRING(48) NOT NULL,
fxa_uid STRING(MAX) NOT NULL,
fxa_kid STRING(MAX) NOT NULL,
id TIMESTAMP NOT NULL,
collection_id INT64 NOT NULL,
bsos STRING(MAX) NOT NULL,

View File

@ -23,8 +23,17 @@ struct Args {
fn main() -> Result<(), Box<dyn Error>> {
env_logger::init();
debug!("Starting up...");
// Set SENTRY_DSN environment variable to enable Sentry
let sentry = sentry::init(sentry::ClientOptions::default());
// Set SENTRY_DSN environment variable to enable Sentry.
// Avoid its default reqwest transport for now due to issues w/
// likely grpcio's boringssl
let curl_transport_factory = |options: &sentry::ClientOptions| {
Box::new(sentry::transports::CurlHttpTransport::new(options))
as Box<dyn sentry::internals::Transport>
};
let sentry = sentry::init(sentry::ClientOptions {
transport: Box::new(curl_transport_factory),
..sentry::ClientOptions::default()
});
if sentry.is_enabled() {
sentry::integrations::panic::register_panic_handler();
}

View File

@ -42,6 +42,9 @@ const BSO_MAX_TTL: u32 = 31_536_000;
const BSO_MAX_SORTINDEX_VALUE: i32 = 999_999_999;
const BSO_MIN_SORTINDEX_VALUE: i32 = -999_999_999;
const ACCEPTED_CONTENT_TYPES: [&str; 3] =
["application/json", "text/plain", "application/newlines"];
lazy_static! {
static ref KNOWN_BAD_PAYLOAD_REGEX: Regex =
Regex::new(r#"IV":\s*"AAAAAAAAAAAAAAAAAAAAAA=="#).unwrap();
@ -88,11 +91,46 @@ impl BatchBsoBody {
}
}
fn get_trimmed_header(headers: &HeaderMap, key: HeaderName, default: &HeaderValue) -> String {
let ct_raw = std::str::from_utf8(headers.get(key).unwrap_or(&default).as_bytes())
// This tries to do the right thing to get the Accepted header according to
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept, but some corners can absolutely be cut.
// This will pull the first accepted content type listed, or the highest rated non-accepted type.
fn get_weighted_header(
headers: &HeaderMap,
key: HeaderName,
accepted: &[&str],
default: &'static str,
) -> String {
let def_hv = HeaderValue::from_static(default);
let hv_raw = std::str::from_utf8(headers.get(key).unwrap_or(&def_hv).as_bytes())
.unwrap_or_else(|_| "invalid");
let ct_parts: Vec<&str> = ct_raw.split(';').collect();
ct_parts[0].trim_end().to_owned()
let hv_parts = hv_raw.split(',');
let mut choice_weight = 0.0;
let mut pick: String = default.to_owned();
for choice in hv_parts {
let opt_weight: Vec<&str> = choice.split(';').collect();
let mut opt = opt_weight[0].trim_end().to_lowercase();
if opt == "*/*" {
opt = default.to_owned()
};
let weight = if opt_weight.len() > 1 {
let weights: Vec<&str> = opt_weight[1].split('=').collect();
// if the weight is malformed, ignore this choice.
if weights.len() < 2 {
continue;
}
f32::from_str(weights[1]).unwrap_or(0.0)
} else {
1.0
};
if weight.abs().trunc() > 0.0 && accepted.contains(&&opt.as_ref()) {
return opt;
}
if weight > choice_weight {
pick = opt;
choice_weight = weight;
}
}
pick
}
#[derive(Default, Deserialize)]
@ -118,23 +156,24 @@ impl FromRequest for BsoBodies {
fn from_request(req: &HttpRequest, payload: &mut Payload) -> Self::Future {
// Only try and parse the body if its a valid content-type
let headers = req.headers();
let default = HeaderValue::from_static("");
let content_type = get_trimmed_header(headers, CONTENT_TYPE, &default);
let content_type = get_weighted_header(
headers,
CONTENT_TYPE,
&ACCEPTED_CONTENT_TYPES,
"application/json",
);
debug!("content_type: {:?}", &content_type);
match content_type.as_str() {
"application/json" | "text/plain" | "application/newlines" | "" => (),
_ => {
return Box::new(future::err(
ValidationErrorKind::FromDetails(
format!("Invalid Content-Type {:?}", content_type),
RequestErrorLocation::Header,
Some("Content-Type".to_owned()),
)
.into(),
));
}
if !ACCEPTED_CONTENT_TYPES.contains(&content_type.as_str()) {
return Box::new(future::err(
ValidationErrorKind::FromDetails(
format!("Invalid Content-Type {:?}", content_type),
RequestErrorLocation::Header,
Some("Content-Type".to_owned()),
)
.into(),
));
}
// Load the entire request into a String
@ -181,7 +220,7 @@ impl FromRequest for BsoBodies {
let max_post_bytes = state.limits.max_post_bytes as usize;
let fut = fut.and_then(move |body| {
// Get all the raw JSON values
// Get all the raw / values
let bsos: Vec<Value> = if newlines {
let mut bsos = Vec::new();
for item in body.lines() {
@ -298,20 +337,21 @@ impl FromRequest for BsoBody {
// Only try and parse the body if its a valid content-type
let headers = req.headers();
let default = HeaderValue::from_static("");
let content_type = get_trimmed_header(&headers, CONTENT_TYPE, &default);
match content_type.as_str() {
"application/json" | "text/plain" | "" => (),
_ => {
return Box::new(future::err(
ValidationErrorKind::FromDetails(
"Invalid Content-Type".to_owned(),
RequestErrorLocation::Header,
Some("Content-Type".to_owned()),
)
.into(),
));
}
let content_type = get_weighted_header(
&headers,
CONTENT_TYPE,
&ACCEPTED_CONTENT_TYPES,
"application/json",
);
if !ACCEPTED_CONTENT_TYPES.contains(&content_type.as_str()) {
return Box::new(future::err(
ValidationErrorKind::FromDetails(
"Invalid Content-Type".to_owned(),
RequestErrorLocation::Header,
Some("Content-Type".to_owned()),
)
.into(),
));
}
let state = match req.app_data::<ServerState>() {
Some(s) => s,
@ -561,8 +601,12 @@ impl FromRequest for CollectionRequest {
let db = <Box<dyn Db>>::from_request(req, payload)?;
let query = BsoQueryParams::from_request(req, payload)?;
let collection = CollectionParam::from_request(req, payload)?.collection;
let content_type =
get_trimmed_header(&req.headers(), ACCEPT, &HeaderValue::from_static(""));
let content_type = get_weighted_header(
&req.headers(),
ACCEPT,
&ACCEPTED_CONTENT_TYPES,
"application/json",
);
let reply = match content_type.as_str() {
"application/newlines" => ReplyFormat::Newlines,
"application/json" | "" => ReplyFormat::Json,
@ -1532,6 +1576,62 @@ mod tests {
*/
}
#[test]
fn test_weighted_header() {
// test non-priority, full weight selection
let mut header_map = HeaderMap::new();
header_map.insert(
CONTENT_TYPE,
HeaderValue::from_static("application/json;q=0.9,text/plain"),
);
let selected = get_weighted_header(
&header_map,
CONTENT_TYPE,
&ACCEPTED_CONTENT_TYPES,
"application/json",
);
assert_eq!(selected, "text/plain".to_owned());
// test default for */*
let mut header_map = HeaderMap::new();
header_map.insert(CONTENT_TYPE, HeaderValue::from_static("*/*,text/plain"));
let selected = get_weighted_header(
&header_map,
CONTENT_TYPE,
&ACCEPTED_CONTENT_TYPES,
"application/json",
);
assert_eq!(selected, "application/json".to_owned());
// test default for selected weighted.
let mut header_map = HeaderMap::new();
header_map.insert(
CONTENT_TYPE,
HeaderValue::from_static("foo/bar;q=0.1,application/json;q=0.2,text/plain;q=0.3"),
);
let selected = get_weighted_header(
&header_map,
CONTENT_TYPE,
&ACCEPTED_CONTENT_TYPES,
"application/json",
);
assert_eq!(selected, "text/plain".to_owned());
// test default for selected weighted.
let mut header_map = HeaderMap::new();
header_map.insert(
CONTENT_TYPE,
HeaderValue::from_static("foo/bar;0.1,text/plain;q=0.1"),
);
let selected = get_weighted_header(
&header_map,
CONTENT_TYPE,
&ACCEPTED_CONTENT_TYPES,
"text/plain",
);
assert_eq!(selected, "text/plain".to_owned());
}
#[test]
fn test_valid_query_args() {
let req = TestRequest::with_uri("/?ids=1,2&full=&sort=index&older=2.43")
@ -1675,7 +1775,7 @@ mod tests {
let req = TestRequest::with_uri(&uri)
.data(state)
.header("authorization", header)
.header("accept", "application/json;a=0.9,/;q=0.2")
.header("accept", "application/json;a=0.9,*/*;q=0.2")
.method(Method::GET)
.param("uid", &USER_ID_STR)
.param("collection", "tabs")