mirrors/syncstorage-rs
1
0
Fork 0
mirror of https://github.com/mozilla-services/syncstorage-rs.git synced 2026-05-05 20:26:26 +02:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

feat: Integrate Spanner emulator with CI (#1079)

Browse Source 
Closes #566
This commit is contained in:
Ethan Donowitz 2021-05-19 18:20:25 -04:00 committed by GitHub
parent f25e4e0fae
commit e6ec1acd87
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 284 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.circleci/config.yml
View File

@ -79,14 +79,28 @@ commands:
environment:
SYNC_ENFORCE_QUOTA: 1
run-e2e-tests:
run-e2e-mysql-tests:
steps:
- run:
name: e2e tests
command: >
/usr/local/bin/docker-compose
-f docker-compose.yaml
-f docker-compose.e2e.yaml
-f docker-compose.mysql.yaml
-f docker-compose.e2e.mysql.yaml
up
--exit-code-from e2e-tests
--abort-on-container-exit
environment:
SYNCSTORAGE_RS_IMAGE: app:build
run-e2e-spanner-tests:
steps:
- run:
name: e2e tests
command: >
/usr/local/bin/docker-compose
-f docker-compose.spanner.yaml
-f docker-compose.e2e.spanner.yaml
up
--exit-code-from e2e-tests
--abort-on-container-exit
@ -207,7 +221,8 @@ jobs:
- run:
name: Restore docker-compose config
command: cp /home/circleci/cache/docker-compose*.yaml .
- run-e2e-tests
- run-e2e-mysql-tests
- run-e2e-spanner-tests
deploy:
docker:

8
Dockerfile
View File

@ -1,4 +1,4 @@
FROM rust:1.51-buster as builder
FROM rust:1.52.1-buster as builder
WORKDIR /app
ADD . /app
ENV PATH=$PATH:/root/.cargo/bin
@ -20,7 +20,7 @@ RUN \
groupadd --gid 10001 app && \
useradd --uid 10001 --gid 10001 --home /app --create-home app && \
apt-get -q update && \
apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-dev python3-pip && \
apt-get -q install -y build-essential default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-dev python3-pip curl jq && \
pip3 install tokenlib && \
rm -rf /var/lib/apt/lists/*
@ -29,6 +29,10 @@ COPY --from=builder /app/version.json /app
COPY --from=builder /app/spanner_config.ini /app
COPY --from=builder /app/tools/spanner /app/tools/spanner
COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
COPY --from=builder /app/src/db/spanner/schema.ddl /app/schema.ddl
RUN chmod +x /app/scripts/prepare-spanner.sh
USER app:app

21
Makefile
View File

@ -16,14 +16,23 @@ clippy:
# Matches what's run in circleci
cargo clippy --all --all-targets -- -D warnings
docker_start:
docker-compose up -d
docker_start_mysql:
docker-compose -f docker-compose.mysql.yaml up -d
docker_start_rebuild:
docker-compose up --build -d
docker_start_mysql_rebuild:
docker-compose -f docker-compose.mysql.yaml up --build -d
docker_stop:
docker-compose down
docker_stop_mysql:
docker-compose -f docker-compose.mysql.yaml down
docker_start_spanner:
docker-compose -f docker-compose.spanner.yaml up -d
docker_start_spanner_rebuild:
docker-compose -f docker-compose.spanner.yaml up --build -d
docker_stop_spanner:
docker-compose -f docker-compose.spanner.yaml down
run:
RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml

40
README.md
View File

@ -106,19 +106,53 @@ To point to a GCP hosted Spanner instance from your local machine, follow these
4. `make run_spanner`.
5. Visit `http://localhost:8000/__heartbeat__` to make sure the server is running.
#### Emulator
Google supports an in-memory Spanner emulator, which can run on your local machine for development purposes. You can install the emulator via the gcloud CLI or Docker by following the instructions [here](https://cloud.google.com/spanner/docs/emulator#installing_and_running_the_emulator). Once the emulator is running, you'll need to create a new instance and a new database. To create an instance using the REST API (exposed via port 9020 on the emulator), we can use `curl`:
```sh
curl --request POST \
"localhost:9020/v1/projects/$PROJECT_ID/instances" \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--data "{\"instance\":{\"config\":\"emulator-test-config\",\"nodeCount\":1,\"displayName\":\"Test Instance\"},\"instanceId\":\"$INSTANCE_ID\"}"
```
Note that you may set `PROJECT_ID` and `INSTANCE_ID` to your liking. To create a new database on this instance, we'll use a similar HTTP request, but we'll need to include information about the database schema. Since we don't have migrations for Spanner, we keep an up-to-date schema in `src/db/spanner/schema.ddl`. The `jq` utility allows us to parse this file for use in the JSON body of an HTTP POST request:
```sh
DDL_STATEMENTS=$(
grep -v ^-- schema.ddl \
| sed -n 's/ \+/ /gp' \
| tr -d '\n' \
| sed 's/\(.*\);/\1/' \
| jq -R -s -c 'split(";")'
)
```
Finally, to create the database:
```sh
curl -sS --request POST \
"localhost:9020/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--data "{\"createStatement\":\"CREATE DATABASE \`$DATABASE_ID\`\",\"extraStatements\":$DDL_STATEMENTS}"
```
Note that, again, you may set `DATABASE_ID` to your liking. Make sure that the `database_url` config variable reflects your choice of project name, instance name, and database name (i.e. it should be of the format `spanner://projects/<your project ID here>/instances/<your instance ID here>/databases/<your database ID here>`).
To run an application server that points to the local Spanner emulator:
```sh
SYNC_SPANNER_EMULATOR_HOST=localhost:9010 make run_spanner
```
### Running via Docker
This requires access to the mozilla-rust-sdk which is now available at `/vendor/mozilla-rust-adk`.
1. Make sure you have [Docker installed](https://docs.docker.com/install/) locally.
2. Copy the contents of mozilla-rust-sdk into top level root dir here.
3. Change cargo.toml mozilla-rust-sdk entry to point to `"path = "mozilla-rust-sdk/googleapis-raw"` instead of the parent dir.
4. Comment out the `image` value under `syncstorage-rs` in docker-compose.yml, and add this instead:
4. Comment out the `image` value under `syncstorage-rs` in either docker-compose.mysql.yml or docker-compose.spanner.yml (depending on which database backend you want to run), and add this instead:
```yml
build:
context: .
```
5. Adjust the MySQL db credentials in docker-compose.yml to match your local setup.
6. `make docker_start` - You can verify it's working by visiting [localhost:8000/\_\_heartbeat\_\_](http://localhost:8000/__heartbeat__)
5. If you are using MySQL, adjust the MySQL db credentials in docker-compose.mysql.yml to match your local setup.
6. `make docker_start_mysql` or `make docker_start_spanner` - You can verify it's working by visiting [localhost:8000/\_\_heartbeat\_\_](http://localhost:8000/__heartbeat__)
### Connecting to Firefox

0
docker-compose.e2e.yaml → docker-compose.e2e.mysql.yaml
View File

35
docker-compose.e2e.spanner.yaml Normal file
View File

@ -0,0 +1,35 @@
version: '3'
services:
db:
db-setup:
syncstorage-rs:
depends_on:
- db-setup
# TODO: either syncstorage-rs should retry the db connection
# itself a few times or should include a wait-for-it.sh script
# inside its docker that would do this for us. Same (probably
# the latter solution) for server-syncstorage below
entrypoint: >
/bin/sh -c "
sleep 15;
/app/bin/syncstorage;
"
e2e-tests:
depends_on:
- syncstorage-rs
image: app:build
privileged: true
user: root
environment:
SYNC_HOST: 0.0.0.0
SYNC_MASTER_SECRET: secret0
SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
SYNC_SPANNER_EMULATOR_HOST: db:9010
SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
SYNC_FXA_METRICS_HASH_SECRET: insecure
entrypoint: >
/bin/sh -c "
sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
"

0
docker-compose.yaml → docker-compose.mysql.yaml
View File

40
docker-compose.spanner.yaml Normal file
View File

@ -0,0 +1,40 @@
version: '3'
services:
db:
image: gcr.io/cloud-spanner-emulator/emulator
ports:
- "9010:9010"
- "9020:9020"
environment:
PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
db-setup:
image: app:build
depends_on:
- db
restart: "no"
entrypoint: "/app/scripts/prepare-spanner.sh"
environment:
SYNC_SPANNER_EMULATOR_HOST: db:9020
syncstorage-rs:
image: ${SYNCSTORAGE_RS_IMAGE:-syncstorage-rs:latest}
restart: always
ports:
- "8000:8000"
depends_on:
- db-setup
environment:
SYNC_HOST: 0.0.0.0
SYNC_MASTER_SECRET: secret0
SYNC_DATABASE_URL: spanner://projects/test-project/instances/test-instance/databases/test-database
SYNC_SPANNER_EMULATOR_HOST: db:9010
SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver
SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ
SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB
SYNC_FXA_METRICS_HASH_SECRET: insecure
volumes:
db_data:
# Application runs off of port 8000.
# you can test if it's available with
# curl "http://localhost:8000/__heartbeat__"

31
scripts/prepare-spanner.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/sh
sleep 5
set -e
PROJECT_ID=test-project
INSTANCE_ID=test-instance
DATABASE_ID=test-database
DDL_STATEMENTS=$(
grep -v ^-- schema.ddl \
| sed -n 's/ \+/ /gp' \
| tr -d '\n' \
| sed 's/\(.*\);/\1/' \
| jq -R -s -c 'split(";")'
)
curl -sS --request POST \
"$SYNC_SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances" \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--data "{\"instance\":{\"config\":\"emulator-test-config\",\"nodeCount\":1,\"displayName\":\"Test Instance\"},\"instanceId\":\"$INSTANCE_ID\"}"
curl -sS --request POST \
"$SYNC_SPANNER_EMULATOR_HOST/v1/projects/$PROJECT_ID/instances/$INSTANCE_ID/databases" \
--header 'Accept: application/json' \
--header 'Content-Type: application/json' \
--data "{\"createStatement\":\"CREATE DATABASE \`$DATABASE_ID\`\",\"extraStatements\":$DDL_STATEMENTS}"
sleep infinity

17
src/db/spanner/insert_standard_collections.sql Normal file
View File

@ -0,0 +1,17 @@
-- These are the 13 standard collections that are expected to exist by clients.
-- The IDs are fixed. The below statement can be used to add these collections
-- to a Spanner instance.
INSERT INTO collections (collection_id, name) VALUES
( 1, "clients"),
( 2, "crypto"),
( 3, "forms"),
( 4, "history"),
( 5, "keys"),
( 6, "meta"),
( 7, "bookmarks"),
( 8, "prefs"),
( 9, "tabs"),
(10, "passwords"),
(11, "addons"),
(12, "addresses"),
(13, "creditcards");

11
src/db/spanner/manager/session.rs
View File

@ -29,6 +29,8 @@ pub struct SpannerSession {
/// Session has a similar `create_time` value that is managed by protobuf,
/// but some clock skew issues are possible.
pub(in crate::db::spanner) create_time: i64,
/// Whether we are using the Spanner emulator
pub using_spanner_emulator: bool,
}
/// Create a Session (and the underlying gRPC Channel)
@ -39,10 +41,8 @@ pub async fn create_spanner_session(
use_test_transactions: bool,
emulator_host: Option<String>,
) -> Result<SpannerSession, DbError> {
// XXX: issue732: Could google_default_credentials (or
// ChannelBuilder::secure_connect) block?!
let using_spanner_emulator = emulator_host.is_some();
let chan = block(move || -> Result<grpcio::Channel, grpcio::Error> {
metrics.start_timer("storage.pool.grpc_auth", None);
if let Some(spanner_emulator_address) = emulator_host {
Ok(ChannelBuilder::new(env)
.max_send_message_len(100 << 20)
@ -51,6 +51,10 @@ pub async fn create_spanner_session(
} else {
// Requires
// GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
metrics.start_timer("storage.pool.grpc_auth", None);
// XXX: issue732: Could google_default_credentials (or
// ChannelBuilder::secure_connect) block?!
let creds = ChannelCredentials::google_default_credentials()?;
Ok(ChannelBuilder::new(env)
.max_send_message_len(100 << 20)
@ -75,6 +79,7 @@ pub async fn create_spanner_session(
client,
use_test_transactions,
create_time: now(),
using_spanner_emulator,
})
}

50
src/db/spanner/models.rs
View File

@ -1247,13 +1247,19 @@ impl SpannerDb {
if let Some(timestamp) = offset.clone().unwrap_or_default().timestamp {
query = match sort {
Sorting::Newest => {
sqlparams.insert("older_eq".to_string(), as_value(timestamp.as_rfc3339()?));
sqltypes.insert("older_eq".to_string(), as_type(TypeCode::TIMESTAMP));
sqlparams.insert(
"older_eq".to_string(),
timestamp.as_rfc3339()?.to_spanner_value(),
);
sqlparam_types.insert("older_eq".to_string(), as_type(TypeCode::TIMESTAMP));
format!("{} AND modified <= @older_eq", query)
}
Sorting::Oldest => {
sqlparams.insert("newer_eq".to_string(), as_value(timestamp.as_rfc3339()?));
sqltypes.insert("newer_eq".to_string(), as_type(TypeCode::TIMESTAMP));
sqlparams.insert(
"newer_eq".to_string(),
timestamp.as_rfc3339()?.to_spanner_value(),
);
sqlparam_types.insert("newer_eq".to_string(), as_type(TypeCode::TIMESTAMP));
format!("{} AND modified >= @newer_eq", query)
}
_ => query,
@ -1270,20 +1276,23 @@ impl SpannerDb {
sqlparams.insert("newer".to_string(), newer.as_rfc3339()?.to_spanner_value());
sqlparam_types.insert("newer".to_string(), as_type(TypeCode::TIMESTAMP));
}
query = match sort {
// issue559: Revert to previous sorting
/*
Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query),
Sorting::Newest | Sorting::None => {
format!("{} ORDER BY modified DESC, bso_id DESC", query)
}
Sorting::Oldest => format!("{} ORDER BY modified ASC, bso_id ASC", query),
*/
Sorting::Index => format!("{} ORDER BY sortindex DESC", query),
Sorting::Newest => format!("{} ORDER BY modified DESC", query),
Sorting::Oldest => format!("{} ORDER BY modified ASC", query),
_ => query,
};
if self.stabilize_bsos_sort_order() {
query = match sort {
Sorting::Index => format!("{} ORDER BY sortindex DESC, bso_id DESC", query),
Sorting::Newest | Sorting::None => {
format!("{} ORDER BY modified DESC, bso_id DESC", query)
}
Sorting::Oldest => format!("{} ORDER BY modified ASC, bso_id ASC", query),
};
} else {
query = match sort {
Sorting::Index => format!("{} ORDER BY sortindex DESC", query),
Sorting::Newest => format!("{} ORDER BY modified DESC", query),
Sorting::Oldest => format!("{} ORDER BY modified ASC", query),
_ => query,
};
}
if let Some(limit) = limit {
// fetch an extra row to detect if there are more rows that match
@ -1311,6 +1320,11 @@ impl SpannerDb {
.execute_async(&self.conn)
}
/// Whether to stabilize the sort order for get_bsos_async
fn stabilize_bsos_sort_order(&self) -> bool {
self.inner.conn.using_spanner_emulator
}
pub fn encode_next_offset(
&self,
_sort: Sorting,

29
spanner-2019-10-01.ddl → src/db/spanner/schema.ddl
View File

@ -15,6 +15,9 @@ CREATE TABLE user_collections (
fxa_kid STRING(MAX) NOT NULL,
collection_id INT64 NOT NULL,
modified TIMESTAMP NOT NULL,
count INT64,
total_bytes INT64,
) PRIMARY KEY(fxa_uid, fxa_kid, collection_id);
CREATE TABLE bsos (
@ -57,8 +60,9 @@ CREATE TABLE batches (
) PRIMARY KEY(fxa_uid, fxa_kid, collection_id, batch_id),
INTERLEAVE IN PARENT user_collections ON DELETE CASCADE;
CREATE INDEX BatchExpiry
ON batches(expiry);
CREATE INDEX BatchExpireId
ON batches(fxa_uid, fxa_kid, collection_id, expiry),
INTERLEAVE IN user_collections;
CREATE TABLE batch_bsos (
fxa_uid STRING(MAX) NOT NULL,
@ -78,21 +82,6 @@ CREATE TABLE batch_bsos (
-- no "modified" column because the modification timestamp gets set on
-- batch commit.
-- 8< Cut Here >8 --
-- Inserting values into table(s) should happen only
-- after table creation.
INSERT INTO collections (collection_id, name) VALUES
( 1, "clients"),
( 2, "crypto"),
( 3, "forms"),
( 4, "history"),
( 5, "keys"),
( 6, "meta"),
( 7, "bookmarks"),
( 8, "prefs"),
( 9, "tabs"),
(10, "passwords"),
(11, "addons"),
(12, "addresses"),
(13, "creditcards");
-- *NOTE*:
-- Newly created Spanner instances should pre-populate the `collections` table by
-- running the content of `insert_standard_collections.sql `

55
src/web/extractors.rs
View File

@ -1245,33 +1245,33 @@ impl FromRequest for BsoQueryParams {
})?;
// issue559: Dead code (timestamp always None)
/*
if params.sort != Sorting::Index {
if let Some(timestamp) = params.offset.as_ref().and_then(|offset| offset.timestamp)
{
let bound = timestamp.as_i64();
if let Some(newer) = params.newer {
if bound < newer.as_i64() {
return Err(ValidationErrorKind::FromDetails(
format!("Invalid Offset {} {}", bound, newer.as_i64()),
RequestErrorLocation::QueryString,
Some("newer".to_owned()),
None,
)
.into());
}
} else if let Some(older) = params.older {
if bound > older.as_i64() {
return Err(ValidationErrorKind::FromDetails(
"Invalid Offset".to_owned(),
RequestErrorLocation::QueryString,
Some("older".to_owned()),
None,
)
.into());
}
}
}
}
if params.sort != Sorting::Index {
if let Some(timestamp) = params.offset.as_ref().and_then(|offset| offset.timestamp)
{
let bound = timestamp.as_i64();
if let Some(newer) = params.newer {
if bound < newer.as_i64() {
return Err(ValidationErrorKind::FromDetails(
format!("Invalid Offset {} {}", bound, newer.as_i64()),
RequestErrorLocation::QueryString,
Some("newer".to_owned()),
None,
)
.into());
}
} else if let Some(older) = params.older {
if bound > older.as_i64() {
return Err(ValidationErrorKind::FromDetails(
"Invalid Offset".to_owned(),
RequestErrorLocation::QueryString,
Some("older".to_owned()),
None,
)
.into());
}
}
}
}
*/
Ok(params)
})
@ -2313,7 +2313,6 @@ mod tests {
offset: 1234,
};
//Issue559: only use offset, don't use timestamp, even if set.
let test_offset = Offset {
timestamp: None,
offset: sample_offset.offset,

9
tools/integration_tests/test_storage.py
View File

@ -1397,7 +1397,14 @@ class TestStorage(StorageFunctionalTestCase):
secret = auth_policy._get_token_secrets(self.host_url)[-1]
tm = tokenlib.TokenManager(secret=secret)
exp = time.time() - 60
data = {"uid": self.user_id, "node": self.host_url, "expires": exp}
data = {
"uid": self.user_id,
"node": self.host_url,
"expires": exp,
"hashed_fxa_uid": self.hashed_fxa_uid,
"fxa_uid": self.fxa_uid,
"fxa_kid": self.fxa_kid
}
self.auth_token = tm.make_token(data)
self.auth_secret = tm.get_derived_secret(self.auth_token)