From e1f48b48c8d876ba64c8d8e7dfbf7b7962662741 Mon Sep 17 00:00:00 2001 From: Rachel Tublitz Date: Fri, 27 Dec 2019 10:36:48 -0500 Subject: [PATCH] chore: add python to docker image also upgrading to rust 1.40.0 and adding a few details to docs for running via docker closes: #384 --- .dockerignore | 12 ++++++++++++ .gitignore | 1 + Dockerfile | 16 +++++++++++----- Makefile | 26 ++++++++++++++++++++++++-- README.md | 22 ++++++++++++++-------- docker-compose.e2e.yaml | 2 +- docker-compose.yaml | 2 +- src/db/spanner/models.rs | 3 +++ tools/spanner/README.md | 4 ++-- 9 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5ef407a2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +CHANGELOG.md +CODE_OF_CONDUCT.md +config/*.example.toml +CONTRIBUTING.md +Dockerfile +docs/ +Makefile +PULL_REQUEST_TEMPLATE.md +README.md +target +tools/examples diff --git a/.gitignore b/.gitignore index 8668c15e..889e7e6b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ service-account.json config/local.toml venv +mozilla-rust-sdk diff --git a/Dockerfile b/Dockerfile index 23928e54..1ecc4e5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ -FROM rust:1.39.0-buster as builder +FROM rust:1.40.0-buster as builder WORKDIR /app ADD . /app ENV PATH=$PATH:/root/.cargo/bin RUN apt-get -q update && \ - apt-get -q install -y default-libmysqlclient-dev cmake golang-go && \ + apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go && \ + rm -rf /var/lib/apt/lists/* && \ cd /app && \ mkdir -m 755 bin @@ -18,11 +19,16 @@ RUN \ groupadd --gid 10001 app && \ useradd --uid 10001 --gid 10001 --home /app --create-home app && \ apt-get -q update && \ - apt-get -q install -y default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 && \ - rm -rf /var/lib/apt/lists + apt-get -q install -y --no-install-recommends default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-venv python3-pip && \ + python3 -m pip install setuptools wheel && \ + python3 -m pip install google-cloud-spanner && \ + rm -rf /var/lib/apt/lists/* COPY --from=builder /app/bin /app/bin COPY --from=builder /app/version.json /app COPY --from=builder /app/spanner_config.ini /app +COPY --from=builder /app/tools/spanner /app/tools/spanner -CMD ["/app/bin/syncstorage", "--config=spanner_config.ini"] +USER app:app + +ENTRYPOINT ["/app/bin/syncstorage", "--config=spanner_config.ini"] diff --git a/Makefile b/Makefile index 9bd8bac8..811ae84a 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,24 @@ -run_local: - RUST_LOG=debug RUST_BACKTRACE=1 cargo run -- --config config/local.toml +## +# Collection of helper scripts used for local dev. +## + +SYNC_DATABASE_URL = 'mysql://sample_user:sample_password@localhost/syncstorage_rs' + +clippy: + # Matches what's run in circleci + cargo clippy --all --all-targets -- -D warnings + +docker_start: + docker-compose up -d + +docker_start_rebuild: + docker-compose up --build -d + +docker_stop: + docker-compose down + +run: + RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml + +test: + cd db-tests && SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) RUST_TEST_THREADS=1 cargo test diff --git a/README.md b/README.md index d7a11454..5f492597 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,19 @@ RUST_LOG=warn GOOGLE_APPLICATION_CREDENTIALS=`pwd`/keys/sync-spanner.json` cargo Note, that unlike MySQL, there is no automatic migrations facility. Currently Spanner schema must be hand edited and modified. +### Running via Docker +This currently requires access to the [mozilla-rust-sdk](https://github.com/mozilla-services/mozilla-rust-sdk) repo. If you don't have it, this will be made public soon; we'll update the README here when that happens. +1. Make sure you have [Docker installed](https://docs.docker.com/install/) locally. +2. Copy the contents of mozilla-rust-sdk into top level root dir here. +3. Change cargo.toml mozilla-rust-sdk entry to point to `"path = "mozilla-rust-sdk/googleapis-raw"` instead of the parent dir. +4. Comment out the `image` value under `syncstorage-rs` in docker-compose.yml, and add this instead: + ``` + build: + context: . + ``` +5. Adjust the MySQL db creds in docker-compose.yml to match your local setup. +6. `make docker_start` - You can verify it's working by visiting [localhost:8000/__heartbeat__](http://localhost:8000/__heartbeat__) + ### Connecting to Firefox This will walk you through the steps to connect this project to your local copy of Firefox. @@ -147,14 +160,7 @@ We use [env_logger](https://crates.io/crates/env_logger): set the `RUST_LOG` env ### Unit tests -1. `cd db-tests`. -2. Pass along your `SYNC_DATABASE_URL` to the test runner. Ie: - -``` -SYNC_DATABASE_URL="mysql://sample_user:sample_password@localhost/syncstorage_rs" && / -RUST_TEST_THREADS=1 && / -cargo test -``` +`make test` - open the Makefile to adjust your `SYNC_DATABASE_URL` as needed. ### End-to-End tests diff --git a/docker-compose.e2e.yaml b/docker-compose.e2e.yaml index 0ea52f33..cd525471 100644 --- a/docker-compose.e2e.yaml +++ b/docker-compose.e2e.yaml @@ -6,7 +6,7 @@ services: # itself a few times or should include a wait-for-it.sh script # inside its docker that would do this for us. Same (probably # the latter solution) for server-syncstorage below - command: > + entrypoint: > /bin/sh -c " sleep 15; /app/bin/syncstorage; diff --git a/docker-compose.yaml b/docker-compose.yaml index cd6fb600..6b500fa3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -29,6 +29,6 @@ services: volumes: db_data: -# Applicaiton runs off of port 8000. +# Application runs off of port 8000. # you can test if it's available with # curl "http://localhost:8000/__heartbeat__" diff --git a/src/db/spanner/models.rs b/src/db/spanner/models.rs index 50373b7c..f3a48755 100644 --- a/src/db/spanner/models.rs +++ b/src/db/spanner/models.rs @@ -1267,6 +1267,7 @@ impl SpannerDb { "".to_string() } ); + q = format!( "{}{}", q, @@ -1279,6 +1280,7 @@ impl SpannerDb { "".to_string() } ); + q = format!( "{}{}", q, @@ -1290,6 +1292,7 @@ impl SpannerDb { "".to_string() } ); + q = format!( "{}{}", q, diff --git a/tools/spanner/README.md b/tools/spanner/README.md index d58418fd..c0f920bb 100644 --- a/tools/spanner/README.md +++ b/tools/spanner/README.md @@ -1,6 +1,6 @@ # Spanner Tools and Scripts -These tools are supplimental scripts for working with the Google Cloud Platform. Follow [the general installation instructions](https://cloud.google.com/spanner/docs/getting-started/python/), as well as fetch the proper service account credentials file. +These tools are supplimental scripts for working with the Google Cloud Platform. Follow [the general installation instructions](https://cloud.google.com/spanner/docs/getting-started/python/), as well as fetch the proper service account credentials file. Remember, the `GOOGLE_APPLICATION_CREDENTIALS` environment variable should point to the absolute path location of your service account credential file. @@ -8,4 +8,4 @@ e.g. ```bash GOOGLE_APPLICATION_CREDENTIALS=`pwd`/keys/project-id-service-cred.json venv/bin/python purge_ttl.py ``` -See each script for details about funciton and use. \ No newline at end of file +See each script for details about function and use.