mirrors/syncstorage-rs
1
0
Fork 0
mirror of https://github.com/mozilla-services/syncstorage-rs.git synced 2026-05-05 12:16:21 +02:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

purge_ttl.py updates for PR comments

Browse Source
This commit is contained in:
Erik Olson 2020-01-07 15:57:46 -05:00
parent 92a57e653d
commit 74b95544b0
No known key found for this signature in database
GPG Key ID: BB933F02985A125E
1 changed files with 22 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
tools/spanner/purge_ttl.py
View File

@ -14,12 +14,8 @@ from urllib import parse
from google.cloud import spanner
# set up logger
logger = logging.getLogger()
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter('{"datetime": "%(asctime)s", "message": "%(message)s"}')
handler.setFormatter(formatter)
logger.addHandler(handler)
logging.basicConfig(format='{"datetime": "%(asctime)s", "message": "%(message)s"}',
stream=sys.stdout,level=logging.INFO)
# Change these to match your install.
client = spanner.Client()
@ -48,35 +44,33 @@ def spanner_read_data(request=None):
instance = client.instance(instance_id)
database = instance.database(database_id)
logger.info("For {}:{}".format(instance_id, database_id))
logging.info("For {}:{}".format(instance_id, database_id))
# Delete Batches. Also deletes child batch_bsos rows (INTERLEAVE
# IN PARENT batches ON DELETE CASCADE)
batches_start = datetime.now()
query = 'DELETE FROM batches WHERE expiry < CURRENT_TIMESTAMP()'
result = database.execute_partitioned_dml(query)
batches_end = datetime.now()
batches_duration = batches_end - batches_start
logger.info("batches: removed {} rows, batches_duration: {}".format(result, batches_duration))
statsd.timing("sync.purge_ttl.batches_duration", batches_duration)
with statsd.timer("syncstorage.purge_ttl.batches_duration"):
batches_start = datetime.now()
query = 'DELETE FROM batches WHERE expiry < CURRENT_TIMESTAMP()'
result = database.execute_partitioned_dml(query)
batches_end = datetime.now()
logging.info("batches: removed {} rows, batches_duration: {}".format(result, batches_end - batches_start))
# Delete BSOs
bso_start = datetime.now()
query = 'DELETE FROM bsos WHERE expiry < CURRENT_TIMESTAMP()'
result = database.execute_partitioned_dml(query)
bso_end = datetime.now()
bso_duration = bso_end - bso_start
logger.info("bso: removed {} rows, bso_duration: {}".format(result, bso_duration))
statsd.timing("sync.purge_ttl.bso_duration", bso_duration)
with statsd.timer("syncstorage.purge_ttl.bso_duration"):
bso_start = datetime.now()
query = 'DELETE FROM bsos WHERE expiry < CURRENT_TIMESTAMP()'
result = database.execute_partitioned_dml(query)
bso_end = datetime.now()
logging.info("bso: removed {} rows, bso_duration: {}".format(result, bso_end - bso_start))
if __name__ == "__main__":
start_time = datetime.now()
logger.info('Starting purge_ttl.py')
with statsd.timer("syncstorage.purge_ttl.total_duration"):
start_time = datetime.now()
logging.info('Starting purge_ttl.py')
spanner_read_data()
spanner_read_data()
end_time = datetime.now()
duration = end_time - start_time
logger.info('Completed purge_ttl.py, total_duration: {}'.format(duration))
statsd.timing("sync.purge_ttl.total_duration", duration)
end_time = datetime.now()
duration = end_time - start_time
logging.info('Completed purge_ttl.py, total_duration: {}'.format(duration))