mirrors/syncstorage-rs
1
0
Fork 0
mirror of https://github.com/mozilla-services/syncstorage-rs.git synced 2026-05-05 12:16:21 +02:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

bug: fix purge_ttl advanced features

Browse Source 
* fix `fxa_uaid` typo
* fix `collection_ids` handler to deal with default empty ids.

Closes #799
This commit is contained in:
jrconlin 2020-08-24 15:13:26 +00:00
parent 0a2fecccd7
commit 714168d107
No known key found for this signature in database
GPG Key ID: D2B33CD056ABD330
1 changed files with 22 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
tools/spanner/purge_ttl.py
View File

@ -42,11 +42,13 @@ def use_dsn(args):
return args
def deleter(database: Database, name: str, query: str, prefix: Optional[str]):
def deleter(database: Database, name: str, query: str, prefix: Optional[str]=None, dryrun: Optional[bool]=False):
with statsd.timer("syncstorage.purge_ttl.{}_duration".format(name)):
logging.info("Running: {}".format(query))
start = datetime.now()
result = database.execute_partitioned_dml(query)
result = 0
if not dryrun:
result = database.execute_partitioned_dml(query)
end = datetime.now()
logging.info(
"{name}: removed {result} rows, {name}_duration: {time}, prefix: {prefix}".format(
@ -62,14 +64,16 @@ def add_conditions(args, query: str, prefix: Optional[str]):
:return: The updated SQL query
"""
if args.collection_ids:
query += " AND collection_id"
if len(args.collection_ids) == 1:
query += " = {:d}".format(args.collection_ids[0])
else:
query += " in ({})".format(
', '.join(map(str, args.collection_ids)))
ids = list(map(str, filter(len, args.collection_ids)))
if len(ids):
query += " AND collection_id"
if len(ids) == 1:
query += " = {:d}".format(ids[0])
else:
query += " in ({})".format(
', '.join(ids))
if prefix:
query += ' AND REGEXP_CONTAINS(fxa_uaid, r"{}")'.format(prefix)
query += ' AND REGEXP_CONTAINS(fxa_uid, r"{}")'.format(prefix)
return query
@ -104,7 +108,8 @@ def spanner_purge(args):
database,
name="batches",
query=batch_query,
prefix=prefix
prefix=prefix,
dryrun=args.dryrun,
)
if args.mode in ["bsos", "both"]:
@ -118,7 +123,8 @@ def spanner_purge(args):
database,
name="bso",
query=bso_query,
prefix=prefix
prefix=prefix,
dryrun=args.dryrun,
)
@ -171,6 +177,11 @@ def get_args():
default=os.environ.get("PURGE_EXPIRY_MODE", "midnight"),
help="Choose the timestamp used to check if an entry is expired"
)
parser.add_argument(
'--dryrun',
action="store_true",
help="Do not purge user records from spanner"
)
args = parser.parse_args()
# override using the DSN URL: