mirrors/syncstorage-rs
1
0
Fork 0
mirror of https://github.com/mozilla-services/syncstorage-rs.git synced 2026-05-05 12:16:21 +02:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

feat: add basic logging to stdout and statsd metrics for purge_ttl.py

Browse Source
This commit is contained in:
Erik Olson 2020-01-07 13:40:06 -05:00
parent 5b0fb643e6
commit 92a57e653d
No known key found for this signature in database
GPG Key ID: BB933F02985A125E
2 changed files with 34 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -21,7 +21,7 @@ RUN \
apt-get -q update && \
apt-get -q install -y --no-install-recommends default-libmysqlclient-dev libssl-dev ca-certificates libcurl4 python3-venv python3-pip && \
python3 -m pip install setuptools wheel && \
python3 -m pip install google-cloud-spanner && \
python3 -m pip install google-cloud-spanner statsd && \
rm -rf /var/lib/apt/lists/*
COPY --from=builder /app/bin /app/bin

40
tools/spanner/purge_ttl.py
View File

@ -5,12 +5,23 @@
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import os
import sys
import logging
from datetime import datetime
from statsd.defaults.env import statsd
from urllib import parse
from google.cloud import spanner
# Change these to match your install.
# set up logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('{"datetime": "%(asctime)s", "message": "%(message)s"}')
handler.setFormatter(formatter)
logger.addHandler(handler)
# Change these to match your install.
client = spanner.Client()
@ -36,21 +47,36 @@ def spanner_read_data(request=None):
(instance_id, database_id) = from_env()
instance = client.instance(instance_id)
database = instance.database(database_id)
outputs = []
outputs.append("For {}:{}".format(instance_id, database_id))
logger.info("For {}:{}".format(instance_id, database_id))
# Delete Batches. Also deletes child batch_bsos rows (INTERLEAVE
# IN PARENT batches ON DELETE CASCADE)
batches_start = datetime.now()
query = 'DELETE FROM batches WHERE expiry < CURRENT_TIMESTAMP()'
result = database.execute_partitioned_dml(query)
outputs.append("batches: removed {} rows".format(result))
batches_end = datetime.now()
batches_duration = batches_end - batches_start
logger.info("batches: removed {} rows, batches_duration: {}".format(result, batches_duration))
statsd.timing("sync.purge_ttl.batches_duration", batches_duration)
# Delete BSOs
bso_start = datetime.now()
query = 'DELETE FROM bsos WHERE expiry < CURRENT_TIMESTAMP()'
result = database.execute_partitioned_dml(query)
outputs.append("bso: removed {} rows".format(result))
return '\n'.join(outputs)
bso_end = datetime.now()
bso_duration = bso_end - bso_start
logger.info("bso: removed {} rows, bso_duration: {}".format(result, bso_duration))
statsd.timing("sync.purge_ttl.bso_duration", bso_duration)
if __name__ == "__main__":
print(spanner_read_data())
start_time = datetime.now()
logger.info('Starting purge_ttl.py')
spanner_read_data()
end_time = datetime.now()
duration = end_time - start_time
logger.info('Completed purge_ttl.py, total_duration: {}'.format(duration))
statsd.timing("sync.purge_ttl.total_duration", duration)