mirror of
https://github.com/mozilla-services/syncstorage-rs.git
synced 2026-01-20 16:01:52 +01:00
30478687d7
This commit enables Postgres builds and tests in CircleCI. Only Tokenserver supports Postgres at the moment, and the Python based integration tests have an implicit dependency on MySQL, so the CI jobs only use Postgres where possible. The jobs have been split up and named more explicitly. The hope is to simply delete the mysql jobs in the future.
185 lines
8.0 KiB
Docker
185 lines
8.0 KiB
Docker
ARG SYNCSTORAGE_DATABASE_BACKEND=spanner
|
|
ARG TOKENSERVER_DATABASE_BACKEND=mysql
|
|
# Alternatively MYSQLCLIENT_PKG=libmysqlclient-dev for the Oracle/MySQL official client
|
|
ARG MYSQLCLIENT_PKG=libmariadb-dev-compat
|
|
|
|
# NOTE: Ensure builder's Rust version matches CI's in .circleci/config.yml
|
|
# RUST_VER
|
|
FROM docker.io/lukemathwalker/cargo-chef:0.1.72-rust-1.89-bookworm AS chef
|
|
WORKDIR /app
|
|
|
|
FROM chef AS planner
|
|
COPY . .
|
|
RUN cargo chef prepare --recipe-path recipe.json
|
|
|
|
FROM chef AS cacher
|
|
ARG SYNCSTORAGE_DATABASE_BACKEND
|
|
ARG TOKENSERVER_DATABASE_BACKEND
|
|
ARG MYSQLCLIENT_PKG
|
|
|
|
# cmake is required to build grpcio-sys for Spanner builds
|
|
RUN apt-get -q update && \
|
|
MYSQL_PKG="" && \
|
|
POSTGRES_DEV_PKG="" && \
|
|
if [ "$SYNCSTORAGE_DATABASE_BACKEND" = "mysql" ] || [ "$TOKENSERVER_DATABASE_BACKEND" = "mysql" ]; then \
|
|
MYSQL_PKG="$MYSQLCLIENT_PKG"; \
|
|
if [ "$MYSQLCLIENT_PKG" = libmysqlclient-dev ] ; then \
|
|
# First install gnupg and setup MySQL repo
|
|
apt-get -q install -y --no-install-recommends gnupg ca-certificates && \
|
|
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
|
|
# Fetch and install the MySQL public key
|
|
gpg --batch --keyserver hkp://keyserver.ubuntu.com --recv-keys A8D3785C && \
|
|
gpg --batch --armor --export A8D3785C | tee /etc/apt/trusted.gpg.d/mysql.asc && \
|
|
apt-get -q update ; \
|
|
fi; \
|
|
fi && \
|
|
if [ "$TOKENSERVER_DATABASE_BACKEND" = "postgres" ]; then \
|
|
POSTGRES_DEV_PKG="libpq-dev"; \
|
|
fi && \
|
|
apt-get -q install -y --no-install-recommends $MYSQL_PKG $POSTGRES_DEV_PKG cmake
|
|
|
|
COPY --from=planner /app/recipe.json recipe.json
|
|
RUN set -x && \
|
|
TOKENSERVER_FEATURES="" && \
|
|
if [ "$TOKENSERVER_DATABASE_BACKEND" = "postgres" ]; then \
|
|
TOKENSERVER_FEATURES="--features=tokenserver-db/postgres"; \
|
|
fi && \
|
|
cargo chef cook --release --no-default-features --features=syncstorage-db/$SYNCSTORAGE_DATABASE_BACKEND $TOKENSERVER_FEATURES --features=py_verifier --recipe-path recipe.json
|
|
|
|
FROM chef AS builder
|
|
ARG SYNCSTORAGE_DATABASE_BACKEND
|
|
ARG TOKENSERVER_DATABASE_BACKEND
|
|
ARG MYSQLCLIENT_PKG
|
|
|
|
ENV POETRY_HOME="/opt/poetry" \
|
|
POETRY_VIRTUALENVS_IN_PROJECT=1 \
|
|
POETRY_NO_INTERACTION=1
|
|
|
|
ENV PATH="$POETRY_HOME/bin:$PATH"
|
|
|
|
COPY . /app
|
|
COPY --from=cacher /app/target /app/target
|
|
COPY --from=cacher $CARGO_HOME /app/$CARGO_HOME
|
|
|
|
RUN apt-get -q update && \
|
|
MYSQL_PKG="" && \
|
|
POSTGRES_DEV_PKG="" && \
|
|
if [ "$SYNCSTORAGE_DATABASE_BACKEND" = "mysql" ] || [ "$TOKENSERVER_DATABASE_BACKEND" = "mysql" ]; then \
|
|
MYSQL_PKG="$MYSQLCLIENT_PKG"; \
|
|
if [ "$MYSQLCLIENT_PKG" = libmysqlclient-dev ] ; then \
|
|
# First install gnupg and setup MySQL repo
|
|
# Key ID A8D3785C from https://dev.mysql.com/doc/refman/8.0/en/checking-gpg-signature.html
|
|
apt-get -q install -y --no-install-recommends gnupg ca-certificates && \
|
|
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
|
|
# Fetch and install the MySQL public key
|
|
gpg --batch --keyserver hkp://keyserver.ubuntu.com --recv-keys A8D3785C && \
|
|
gpg --batch --armor --export A8D3785C | tee /etc/apt/trusted.gpg.d/mysql.asc && \
|
|
apt-get -q update ; \
|
|
fi; \
|
|
fi && \
|
|
if [ "$TOKENSERVER_DATABASE_BACKEND" = "postgres" ]; then \
|
|
POSTGRES_DEV_PKG="libpq-dev"; \
|
|
fi && \
|
|
apt-get -q update && \
|
|
apt-get -q install -y --no-install-recommends $MYSQL_PKG $POSTGRES_DEV_PKG cmake golang-go python3-dev python3-pip python3-setuptools python3-wheel python3-venv pkg-config && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN curl -sSL https://install.python-poetry.org | python3 - && \
|
|
ln -s $POETRY_HOME/bin/poetry /usr/local/bin/poetry && \
|
|
poetry --version && \
|
|
poetry config virtualenvs.create false && \
|
|
poetry self add poetry-plugin-export
|
|
|
|
# Generating a requirements.txt from Poetry dependencies.
|
|
# [tool.poetry.dependencies]
|
|
RUN poetry export --no-interaction --without dev --output requirements.txt --without-hashes && \
|
|
pip3 install --break-system-packages -r requirements.txt
|
|
|
|
|
|
ENV PATH=$PATH:/root/.cargo/bin
|
|
|
|
RUN set -x && \
|
|
TOKENSERVER_FEATURES="" && \
|
|
if [ "$TOKENSERVER_DATABASE_BACKEND" = "postgres" ]; then \
|
|
TOKENSERVER_FEATURES="--features=tokenserver-db/postgres"; \
|
|
fi && \
|
|
cargo --version && \
|
|
rustc --version && \
|
|
cargo install --path ./syncserver --no-default-features --features=syncstorage-db/$SYNCSTORAGE_DATABASE_BACKEND $TOKENSERVER_FEATURES --features=py_verifier --locked --root /app
|
|
|
|
FROM docker.io/library/debian:bookworm-slim
|
|
ARG SYNCSTORAGE_DATABASE_BACKEND
|
|
ARG TOKENSERVER_DATABASE_BACKEND
|
|
ARG MYSQLCLIENT_PKG
|
|
|
|
ENV POETRY_HOME="/opt/poetry" \
|
|
POETRY_VIRTUALENVS_IN_PROJECT=1 \
|
|
POETRY_NO_INTERACTION=1
|
|
|
|
ENV PATH="$POETRY_HOME/bin:$PATH"
|
|
|
|
WORKDIR /app
|
|
COPY --from=builder /app/requirements.txt /app
|
|
COPY --from=builder /app/pyproject.toml /app/poetry.lock /app/
|
|
|
|
RUN apt-get -q update && apt-get -qy install wget
|
|
RUN groupadd --gid 10001 app && \
|
|
useradd --uid 10001 --gid 10001 --home /app --create-home app
|
|
RUN apt-get -q update && \
|
|
MYSQL_PKG="" && \
|
|
POSTGRES_PKG="" && \
|
|
# Always install MySQL libs because Python integration tests depend on mysqlclient
|
|
MYSQL_PKG="$MYSQLCLIENT_PKG" && \
|
|
if [ "$MYSQLCLIENT_PKG" = libmysqlclient-dev ] ; then \
|
|
# First install gnupg and setup MySQL repo
|
|
apt-get install -y gnupg ca-certificates wget && \
|
|
echo "deb https://repo.mysql.com/apt/debian/ bookworm mysql-8.0" >> /etc/apt/sources.list && \
|
|
# Fetch and install the MySQL public key
|
|
gpg --batch --keyserver hkp://keyserver.ubuntu.com --recv-keys A8D3785C && \
|
|
gpg --batch --armor --export A8D3785C | tee /etc/apt/trusted.gpg.d/mysql.asc && \
|
|
apt-get -q update ; \
|
|
fi && \
|
|
POSTGRES_PKG="libpq5" && \
|
|
if [ "$TOKENSERVER_DATABASE_BACKEND" = "postgres" ]; then \
|
|
POSTGRES_PKG="$POSTGRES_PKG libpq-dev"; \
|
|
fi && \
|
|
apt-get -q install -y build-essential $MYSQL_PKG $POSTGRES_PKG libssl-dev libffi-dev libcurl4 python3-dev python3-pip python3-setuptools python3-wheel python3-venv cargo curl jq pkg-config && \
|
|
# The python3-cryptography debian package installs version 2.6.1, but we
|
|
# we want to use the version specified in requirements.txt. To do this,
|
|
# we have to remove the python3-cryptography package here.
|
|
apt-get -q remove -y python3-cryptography && \
|
|
rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN curl -sSL https://install.python-poetry.org | python3 - && \
|
|
ln -s $POETRY_HOME/bin/poetry /usr/local/bin/poetry && \
|
|
poetry --version && \
|
|
poetry config virtualenvs.create false && \
|
|
poetry self add poetry-plugin-export
|
|
# Generating a requirements.txt from Poetry dependencies.
|
|
# [tool.poetry.dependencies]
|
|
RUN poetry export --no-interaction --without dev --output requirements.txt --without-hashes && \
|
|
pip3 install --break-system-packages -r requirements.txt
|
|
|
|
COPY --from=builder /app/bin /app/bin
|
|
COPY --from=builder /app/syncserver/version.json /app
|
|
COPY --from=builder /app/tools/spanner /app/tools/spanner
|
|
COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
|
|
COPY --from=builder /app/tools/tokenserver /app/tools/tokenserver
|
|
COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
|
|
COPY --from=builder /app/scripts/start_mock_fxa_server.sh /app/scripts/start_mock_fxa_server.sh
|
|
COPY --from=builder /app/syncstorage-spanner/src/schema.ddl /app/schema.ddl
|
|
|
|
RUN chmod +x /app/scripts/prepare-spanner.sh
|
|
|
|
WORKDIR /app/tools/integration_tests/
|
|
RUN poetry export --no-interaction --without dev --output requirements.txt --without-hashes
|
|
WORKDIR /app/tools/tokenserver/
|
|
RUN poetry export --no-interaction --without dev --output requirements.txt --without-hashes
|
|
WORKDIR /app
|
|
RUN pip3 install --break-system-packages -r /app/tools/integration_tests/requirements.txt
|
|
RUN pip3 install --break-system-packages -r /app/tools/tokenserver/requirements.txt
|
|
|
|
USER app:app
|
|
|
|
ENTRYPOINT ["/app/bin/syncserver"]
|