feat: Add debugging tools

This adds a few useful inline testing tools for syncstorage-rs

tools/hawk/README.md

@@ -0,0 +1,12 @@
+# Make a Hawk compatible Auth header
+
+The best way to install this is probably to set up a python virtual
+env.
+
+`python3 -m venv venv && venv/bin/pip install -r requirements.txt`
+
+this will create a python virtual environment in the `/venv` directory.
+
+*Note* You may need to install `python3-venv` for the above to work.
+
+Use `-h` for help.

tools/hawk/make_hawk_token.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Create a Hawk token for tests
+
+requires hawkauthlib, tokenlib, webob
+
+Creates the hawk headers for auth::tests, in particular valid_header and
+valid_header_with_querystring.
+
+The latter modifies the query string which changes the mac/nonce and
+potentially ts values (in the Hawk header).
+
+"""
+import argparse
+import hmac
+import os
+import time
+from binascii import hexlify
+from datetime import timedelta
+from hashlib import sha256
+
+import hawkauthlib
+import tokenlib
+from webob.request import Request
+
+LEGACY_UID = 1
+COL = "col2"
+URI = "/1.5/{uid}/storage/{col}/".format(uid=LEGACY_UID, col=COL)
+FXA_UID = "DEADBEEF00004be4ae957006c0ceb620"
+FXA_KID = "DEADBEEF00004be4ae957006c0ceb620"
+DEVICE_ID = "device1"
+NODE = "http://localhost:8000"
+SECRET = "Ted Koppel is a robot"
+HMAC_KEY = b"foo"
+
+# 10 years
+DURATION = timedelta(days=10 * 365).total_seconds()
+
+SALT = hexlify(os.urandom(3)).decode('ascii')
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description="Create a hawk header for use in testing"
+        )
+    parser.add_argument(
+        '--uid', type=int, default=LEGACY_UID,
+        help="Legacy UID ({})".format(LEGACY_UID))
+    parser.add_argument(
+        '--uri', default=URI,
+        help="URI path ({})".format(URI))
+    parser.add_argument(
+        '--fxa_uid', default=FXA_UID,
+        help="FxA User ID ({})".format(FXA_UID))
+    parser.add_argument(
+        '--fxa_kid', default=FXA_KID,
+        help="FxA K ID ({})".format(FXA_KID))
+    parser.add_argument(
+        '--device_id', default=DEVICE_ID,
+        help="FxA Device ID ({})".format(DEVICE_ID))
+    parser.add_argument(
+        '--node', default=NODE,
+        help="HTTP Host URI for node ({})".format(NODE))
+    parser.add_argument(
+        '--duration', type=int, default=DURATION,
+        help="Hawk TTL ({})".format(DURATION))
+    parser.add_argument(
+        '--secret', default=SECRET,
+        help="Shared HAWK secret ({})".format(SECRET))
+    parser.add_argument(
+        '--hmac_key', default=HMAC_KEY,
+        help="HAWK HMAC key ({})".format(HMAC_KEY))
+    parser.add_argument(
+        '--as_header', action="store_true", default=False,
+        help="return only header (False)")
+    return parser.parse_args()
+
+
+def create_token(args):
+    expires = int(time.time()) + args.duration
+    token_data = {
+        'uid': args.uid,
+        'node': args.node,
+        'expires': expires,
+        'fxa_uid': args.fxa_uid,
+        'fxa_kid': args.fxa_kid,
+        'hashed_fxa_uid': metrics_hash(args, args.fxa_uid),
+        'hashed_device_id': metrics_hash(args, args.device_id),
+        'salt': SALT,
+    }
+    token = tokenlib.make_token(token_data, secret=args.secret)
+    key = tokenlib.get_derived_secret(token, secret=args.secret)
+    return token, key, expires, SALT
+
+
+def metrics_hash(args, value):
+    if isinstance(args.hmac_key, str):
+        args.hmac_key = args.hmac_key.encode()
+    hasher = hmac.new(args.hmac_key, b'', sha256)
+    # value may be an email address, in which case we only want the first part
+    hasher.update(value.encode('utf-8').split(b"@", 1)[0])
+    return hasher.hexdigest()
+
+
+def main():
+    args = get_args()
+    token, key, expires, salt = create_token(args)
+    path = "{node}{uri}".format(
+            node=args.node,
+            uri=args.uri)
+    req = Request.blank(path)
+    header = hawkauthlib.sign_request(req, token, key)
+    if not args.as_header:
+        print("Expires: ", expires)
+        print("Salt: ", salt)
+        print("\nPath: ", path)
+        print("Hawk Authorization Header: ", header)
+    else:
+        print(header)
+
+
+if __name__ == '__main__':
+    main()

tools/hawk/requirements.txt

@@ -0,0 +1,2 @@
+hawkauthlib
+tokenlib

tools/poster/README.md

@@ -0,0 +1,4 @@
+# Post a fake BSO to the local server
+
+`post.bash` is a crappy bash script that attempts to post a fake BSO to a local instance of syncserver-rs. It's useful for line testing operations.
+

tools/poster/post.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+NODE="http://localhost:8000"
+URI="/1.5/1/storage/col2/DEADBEEF"
+AUTH=`../hawk/venv/bin/python ../hawk/make_hawk_token.py --node $NODE --uri $URI --as_header`
+curl -vv -X PUT "$NODE$URI" \
+    -H "Authorization: $AUTH" \
+    -H 'Content-Type: application/json' \
+    -H 'Accept: application/json' \
+    -d '{"id": "womble", "payload": "mary had a little lamb with a nice mint jelly", "sortindex": 0, "ttl": 86400}'

tools/spanner/write_batch.py

@@ -52,7 +52,7 @@ BATCH_SIZE = 2000
 # Total number of threads to use
 THREAD_COUNT = 16
 # Number of batches per thread
-BATCHES = 186412
+BATCHES = 330
 
 # `100` is the bottom limit for reserved collections.
 COLL_ID = 100
@@ -76,7 +76,10 @@ PAYLOAD = ''.join(
     for _ in range(PAYLOAD_SIZE))
 
 
-def load(instance, db, fxa_uid, fxa_kid, coll_id):
+def load(instance, db, coll_id, name):
+    fxa_uid = "DEADBEEF" + uuid.uuid4().hex[8:]
+    fxa_kid = "{:013d}-{}".format(22, fxa_uid)
+    print("{} -> Loading {} {}".format(name, fxa_uid, fxa_kid))
     name = threading.current_thread().getName()
     spanner_client = spanner.Client()
     instance = spanner_client.instance(instance)
@@ -196,11 +199,9 @@ def loader():
     # Each loader thread gets it's own fake user to prevent some hotspot
     # issues.
     (instance_id, database_id) = from_env()
-    fxa_uid = "DEADBEEF" + uuid.uuid4().hex[8:]
-    fxa_kid = "{:013d}-{}".format(22, fxa_uid)
+    # switching uid/kid to per load because of weird google trimming
     name = threading.current_thread().getName()
-    print("{} -> Loading {} {}".format(name, fxa_uid, fxa_kid))
-    load(instance_id, database_id, fxa_uid, fxa_kid, COLL_ID)
+    load(instance_id, database_id, COLL_ID, name)
 
 
 def main():