chore: add python to docker image

also upgrading to rust 1.40.0 and adding a few details to docs for running via docker

closes: #384

.dockerignore

@@ -0,0 +1,12 @@
+.git
+CHANGELOG.md
+CODE_OF_CONDUCT.md
+config/*.example.toml
+CONTRIBUTING.md
+Dockerfile
+docs/
+Makefile
+PULL_REQUEST_TEMPLATE.md
+README.md
+target
+tools/examples

.gitignore

@@ -35,3 +35,4 @@ service-account.json
 
 config/local.toml
 venv
+mozilla-rust-sdk

Dockerfile

@@ -1,9 +1,10 @@
-FROM rust:1.39.0-buster as builder
+FROM rust:1.40.0-buster as builder
 WORKDIR /app
 ADD . /app
 ENV PATH=$PATH:/root/.cargo/bin
 RUN apt-get -q update && \
-    apt-get -q install -y default-libmysqlclient-dev cmake golang-go && \
+    apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go && \
+    rm -rf /var/lib/apt/lists/* && \
     cd /app && \
     mkdir -m 755 bin
 
@@ -18,11 +19,16 @@ RUN \
     groupadd --gid 10001 app && \
     useradd --uid 10001 --gid 10001 --home /app --create-home app && \
     apt-get -q update && \
-    apt-get -q install -y default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 && \
-    rm -rf /var/lib/apt/lists
+    apt-get -q install -y --no-install-recommends default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-venv python3-pip && \
+    python3 -m pip install setuptools wheel && \
+    python3 -m pip install google-cloud-spanner && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /app/bin /app/bin
 COPY --from=builder /app/version.json /app
 COPY --from=builder /app/spanner_config.ini /app
+COPY --from=builder /app/tools/spanner /app/tools/spanner
 
-CMD ["/app/bin/syncstorage", "--config=spanner_config.ini"]
+USER app:app
+
+ENTRYPOINT ["/app/bin/syncstorage", "--config=spanner_config.ini"]

Makefile

@@ -1,2 +1,24 @@
-run_local:
-	RUST_LOG=debug RUST_BACKTRACE=1 cargo run -- --config config/local.toml
+##
+# Collection of helper scripts used for local dev.
+##
+
+SYNC_DATABASE_URL = 'mysql://sample_user:sample_password@localhost/syncstorage_rs'
+
+clippy:
+	# Matches what's run in circleci
+	cargo clippy --all --all-targets -- -D warnings
+
+docker_start:
+	docker-compose up -d
+
+docker_start_rebuild:
+	docker-compose up --build -d
+
+docker_stop:
+	docker-compose down
+
+run:
+	RUST_LOG=debug RUST_BACKTRACE=full cargo run -- --config config/local.toml
+
+test:
+	cd db-tests && SYNC_DATABASE_URL=$(SYNC_DATABASE_URL) RUST_TEST_THREADS=1 cargo test

README.md

@@ -101,6 +101,19 @@ RUST_LOG=warn GOOGLE_APPLICATION_CREDENTIALS=`pwd`/keys/sync-spanner.json` cargo
 
 Note, that unlike MySQL, there is no automatic migrations facility. Currently Spanner schema must be hand edited and modified.
 
+### Running via Docker
+This currently requires access to the [mozilla-rust-sdk](https://github.com/mozilla-services/mozilla-rust-sdk) repo. If you don't have it, this will be made public soon; we'll update the README here when that happens.
+1. Make sure you have [Docker installed](https://docs.docker.com/install/) locally.
+2. Copy the contents of mozilla-rust-sdk into top level root dir here.
+3. Change cargo.toml mozilla-rust-sdk entry to point to `"path = "mozilla-rust-sdk/googleapis-raw"` instead of the parent dir.
+4. Comment out the `image` value under `syncstorage-rs` in docker-compose.yml, and add this instead:
+    ```
+      build:
+        context: .
+    ```
+5. Adjust the MySQL db creds in docker-compose.yml to match your local setup.
+6. `make docker_start` - You can verify it's working by visiting [localhost:8000/__heartbeat__](http://localhost:8000/__heartbeat__)
+
 ### Connecting to Firefox
 
 This will walk you through the steps to connect this project to your local copy of Firefox. 
@@ -147,14 +160,7 @@ We use [env_logger](https://crates.io/crates/env_logger): set the `RUST_LOG` env
 
 ### Unit tests
 
-1. `cd db-tests`.
-2. Pass along your `SYNC_DATABASE_URL` to the test runner. Ie:
-
-```
-SYNC_DATABASE_URL="mysql://sample_user:sample_password@localhost/syncstorage_rs" && /
-RUST_TEST_THREADS=1 && /
-cargo test
-```
+`make test` - open the Makefile to adjust your `SYNC_DATABASE_URL` as needed.
 
 ### End-to-End tests
 

docker-compose.e2e.yaml

@@ -6,7 +6,7 @@ services:
         # itself a few times or should include a wait-for-it.sh script
         # inside its docker that would do this for us. Same (probably
         # the latter solution) for server-syncstorage below
-        command: >
+        entrypoint: >
           /bin/sh -c "
             sleep 15;
             /app/bin/syncstorage;

docker-compose.yaml

@@ -29,6 +29,6 @@ services:
 volumes:
     db_data:
 
-# Applicaiton runs off of port 8000.
+# Application runs off of port 8000.
 # you can test if it's available with
 # curl "http://localhost:8000/__heartbeat__"

src/db/spanner/models.rs

@@ -1267,6 +1267,7 @@ impl SpannerDb {
                     "".to_string()
                 }
             );
+
             q = format!(
                 "{}{}",
                 q,
@@ -1279,6 +1280,7 @@ impl SpannerDb {
                     "".to_string()
                 }
             );
+
             q = format!(
                 "{}{}",
                 q,
@@ -1290,6 +1292,7 @@ impl SpannerDb {
                     "".to_string()
                 }
             );
+
             q = format!(
                 "{}{}",
                 q,

tools/spanner/README.md

@@ -1,6 +1,6 @@
 # Spanner Tools and Scripts
 
-These tools are supplimental scripts for working with the Google Cloud Platform. Follow [the general installation instructions](https://cloud.google.com/spanner/docs/getting-started/python/), as well as fetch the proper service account credentials file. 
+These tools are supplimental scripts for working with the Google Cloud Platform. Follow [the general installation instructions](https://cloud.google.com/spanner/docs/getting-started/python/), as well as fetch the proper service account credentials file.
 
 Remember, the `GOOGLE_APPLICATION_CREDENTIALS` environment variable should point to the absolute path location of your service account credential file.
 
@@ -8,4 +8,4 @@ e.g.
 ```bash
 GOOGLE_APPLICATION_CREDENTIALS=`pwd`/keys/project-id-service-cred.json venv/bin/python purge_ttl.py
 ```
-See each script for details about funciton and use.
+See each script for details about function and use.