diff --git a/tools/hawk/README.md b/tools/hawk/README.md new file mode 100644 index 00000000..caaa6807 --- /dev/null +++ b/tools/hawk/README.md @@ -0,0 +1,12 @@ +# Make a Hawk compatible Auth header + +The best way to install this is probably to set up a python virtual +env. + +`python3 -m venv venv && venv/bin/pip install -r requirements.txt` + +this will create a python virtual environment in the `/venv` directory. + +*Note* You may need to install `python3-venv` for the above to work. + +Use `-h` for help. \ No newline at end of file diff --git a/tools/hawk/make_hawk_token.py b/tools/hawk/make_hawk_token.py new file mode 100644 index 00000000..ee6417dd --- /dev/null +++ b/tools/hawk/make_hawk_token.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Create a Hawk token for tests + +requires hawkauthlib, tokenlib, webob + +Creates the hawk headers for auth::tests, in particular valid_header and +valid_header_with_querystring. + +The latter modifies the query string which changes the mac/nonce and +potentially ts values (in the Hawk header). + +""" +import argparse +import hmac +import os +import time +from binascii import hexlify +from datetime import timedelta +from hashlib import sha256 + +import hawkauthlib +import tokenlib +from webob.request import Request + +LEGACY_UID = 1 +COL = "col2" +URI = "/1.5/{uid}/storage/{col}/".format(uid=LEGACY_UID, col=COL) +FXA_UID = "DEADBEEF00004be4ae957006c0ceb620" +FXA_KID = "DEADBEEF00004be4ae957006c0ceb620" +DEVICE_ID = "device1" +NODE = "http://localhost:8000" +SECRET = "Ted Koppel is a robot" +HMAC_KEY = b"foo" + +# 10 years +DURATION = timedelta(days=10 * 365).total_seconds() + +SALT = hexlify(os.urandom(3)).decode('ascii') + + +def get_args(): + parser = argparse.ArgumentParser( + description="Create a hawk header for use in testing" + ) + parser.add_argument( + '--uid', type=int, default=LEGACY_UID, + help="Legacy UID ({})".format(LEGACY_UID)) + parser.add_argument( + '--uri', default=URI, + help="URI path ({})".format(URI)) + parser.add_argument( + '--fxa_uid', default=FXA_UID, + help="FxA User ID ({})".format(FXA_UID)) + parser.add_argument( + '--fxa_kid', default=FXA_KID, + help="FxA K ID ({})".format(FXA_KID)) + parser.add_argument( + '--device_id', default=DEVICE_ID, + help="FxA Device ID ({})".format(DEVICE_ID)) + parser.add_argument( + '--node', default=NODE, + help="HTTP Host URI for node ({})".format(NODE)) + parser.add_argument( + '--duration', type=int, default=DURATION, + help="Hawk TTL ({})".format(DURATION)) + parser.add_argument( + '--secret', default=SECRET, + help="Shared HAWK secret ({})".format(SECRET)) + parser.add_argument( + '--hmac_key', default=HMAC_KEY, + help="HAWK HMAC key ({})".format(HMAC_KEY)) + parser.add_argument( + '--as_header', action="store_true", default=False, + help="return only header (False)") + return parser.parse_args() + + +def create_token(args): + expires = int(time.time()) + args.duration + token_data = { + 'uid': args.uid, + 'node': args.node, + 'expires': expires, + 'fxa_uid': args.fxa_uid, + 'fxa_kid': args.fxa_kid, + 'hashed_fxa_uid': metrics_hash(args, args.fxa_uid), + 'hashed_device_id': metrics_hash(args, args.device_id), + 'salt': SALT, + } + token = tokenlib.make_token(token_data, secret=args.secret) + key = tokenlib.get_derived_secret(token, secret=args.secret) + return token, key, expires, SALT + + +def metrics_hash(args, value): + if isinstance(args.hmac_key, str): + args.hmac_key = args.hmac_key.encode() + hasher = hmac.new(args.hmac_key, b'', sha256) + # value may be an email address, in which case we only want the first part + hasher.update(value.encode('utf-8').split(b"@", 1)[0]) + return hasher.hexdigest() + + +def main(): + args = get_args() + token, key, expires, salt = create_token(args) + path = "{node}{uri}".format( + node=args.node, + uri=args.uri) + req = Request.blank(path) + header = hawkauthlib.sign_request(req, token, key) + if not args.as_header: + print("Expires: ", expires) + print("Salt: ", salt) + print("\nPath: ", path) + print("Hawk Authorization Header: ", header) + else: + print(header) + + +if __name__ == '__main__': + main() diff --git a/tools/hawk/requirements.txt b/tools/hawk/requirements.txt new file mode 100644 index 00000000..1433bcc3 --- /dev/null +++ b/tools/hawk/requirements.txt @@ -0,0 +1,2 @@ +hawkauthlib +tokenlib diff --git a/tools/poster/README.md b/tools/poster/README.md new file mode 100644 index 00000000..716042da --- /dev/null +++ b/tools/poster/README.md @@ -0,0 +1,4 @@ +# Post a fake BSO to the local server + +`post.bash` is a crappy bash script that attempts to post a fake BSO to a local instance of syncserver-rs. It's useful for line testing operations. + diff --git a/tools/poster/post.bash b/tools/poster/post.bash new file mode 100644 index 00000000..fa4a56f4 --- /dev/null +++ b/tools/poster/post.bash @@ -0,0 +1,9 @@ +#!/bin/bash +NODE="http://localhost:8000" +URI="/1.5/1/storage/col2/DEADBEEF" +AUTH=`../hawk/venv/bin/python ../hawk/make_hawk_token.py --node $NODE --uri $URI --as_header` +curl -vv -X PUT "$NODE$URI" \ + -H "Authorization: $AUTH" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json' \ + -d '{"id": "womble", "payload": "mary had a little lamb with a nice mint jelly", "sortindex": 0, "ttl": 86400}' diff --git a/tools/spanner/write_batch.py b/tools/spanner/write_batch.py index f3bf196e..f71fb18a 100644 --- a/tools/spanner/write_batch.py +++ b/tools/spanner/write_batch.py @@ -52,7 +52,7 @@ BATCH_SIZE = 2000 # Total number of threads to use THREAD_COUNT = 16 # Number of batches per thread -BATCHES = 186412 +BATCHES = 330 # `100` is the bottom limit for reserved collections. COLL_ID = 100 @@ -76,7 +76,10 @@ PAYLOAD = ''.join( for _ in range(PAYLOAD_SIZE)) -def load(instance, db, fxa_uid, fxa_kid, coll_id): +def load(instance, db, coll_id, name): + fxa_uid = "DEADBEEF" + uuid.uuid4().hex[8:] + fxa_kid = "{:013d}-{}".format(22, fxa_uid) + print("{} -> Loading {} {}".format(name, fxa_uid, fxa_kid)) name = threading.current_thread().getName() spanner_client = spanner.Client() instance = spanner_client.instance(instance) @@ -196,11 +199,9 @@ def loader(): # Each loader thread gets it's own fake user to prevent some hotspot # issues. (instance_id, database_id) = from_env() - fxa_uid = "DEADBEEF" + uuid.uuid4().hex[8:] - fxa_kid = "{:013d}-{}".format(22, fxa_uid) + # switching uid/kid to per load because of weird google trimming name = threading.current_thread().getName() - print("{} -> Loading {} {}".format(name, fxa_uid, fxa_kid)) - load(instance_id, database_id, fxa_uid, fxa_kid, COLL_ID) + load(instance_id, database_id, COLL_ID, name) def main():