diff --git a/.circleci/config.yml b/.circleci/config.yml index 0feb26d1..14627dc2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,7 +29,7 @@ commands: name: Setup python command: | sudo apt-get update && sudo apt-get install -y python3-dev python3-pip - pip3 install tokenlib + pip3 install hawkauthlib konfig pyramid pyramid_hawkauth requests simplejson tokenlib unittest2 WebTest WSGIProxy2 rust-check: steps: - run: @@ -88,7 +88,7 @@ commands: -f docker-compose.yaml -f docker-compose.e2e.yaml up - --exit-code-from server-syncstorage + --exit-code-from e2e-tests --abort-on-container-exit environment: SYNCSTORAGE_RS_IMAGE: app:build diff --git a/Dockerfile b/Dockerfile index 313dc684..812deb09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,9 +6,7 @@ ENV PATH=$PATH:/root/.cargo/bin RUN apt-get -q update && \ apt-get -q install -y --no-install-recommends default-libmysqlclient-dev cmake golang-go python3-dev python3-pip && \ pip3 install tokenlib && \ - rm -rf /var/lib/apt/lists/* && \ - cd /app && \ - mkdir -m 755 bin + rm -rf /var/lib/apt/lists/* RUN \ cargo --version && \ @@ -30,6 +28,7 @@ COPY --from=builder /app/bin /app/bin COPY --from=builder /app/version.json /app COPY --from=builder /app/spanner_config.ini /app COPY --from=builder /app/tools/spanner /app/tools/spanner +COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests USER app:app diff --git a/docker-compose.e2e.yaml b/docker-compose.e2e.yaml index cd525471..437761a2 100644 --- a/docker-compose.e2e.yaml +++ b/docker-compose.e2e.yaml @@ -2,6 +2,8 @@ version: '3' services: db: syncstorage-rs: + depends_on: + - db # TODO: either syncstorage-rs should retry the db connection # itself a few times or should include a wait-for-it.sh script # inside its docker that would do this for us. Same (probably @@ -11,12 +13,21 @@ services: sleep 15; /app/bin/syncstorage; " - server-syncstorage: - image: mozilla/server-syncstorage:latest + e2e-tests: depends_on: - syncstorage-rs + image: app:build + privileged: true + user: root + environment: + SYNC_HOST: 0.0.0.0 + SYNC_MASTER_SECRET: secret0 + SYNC_DATABASE_URL: mysql://test:test@db:3306/syncstorage + SYNC_TOKENSERVER_DATABASE_URL: mysql://username:pw@localhost/tokenserver + SYNC_TOKENSERVER_JWKS_RSA_MODULUS: 2lDphW0lNZ4w1m9CfmIhC1AxYG9iwihxBdQZo7_6e0TBAi8_TNaoHHI90G9n5d8BQQnNcF4j2vOs006zlXcqGrP27b49KkN3FmbcOMovvfesMseghaqXqqFLALL9us3Wstt_fV_qV7ceRcJq5Hd_Mq85qUgYSfb9qp0vyePb26KEGy4cwO7c9nCna1a_i5rzUEJu6bAtcLS5obSvmsOOpTLHXojKKOnC4LRC3osdR6AU6v3UObKgJlkk_-8LmPhQZqOXiI_TdBpNiw6G_-eishg8V_poPlAnLNd8mfZBam-_7CdUS4-YoOvJZfYjIoboOuVmUrBjogFyDo72EPTReQ + SYNC_TOKENSERVER_JWKS_RSA_EXPONENT: AQAB + SYNC_FXA_METRICS_HASH_SECRET: insecure entrypoint: > - /bin/ash -c " - sleep 18; - /app/docker-entrypoint.sh test_endpoint http://syncstorage-rs:8000#secret0; + /bin/sh -c " + sleep 28; pip3 install -r /app/tools/integration_tests/requirements.txt && python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0' " diff --git a/tests.ini b/tests.ini new file mode 100644 index 00000000..80ebf170 --- /dev/null +++ b/tests.ini @@ -0,0 +1,23 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:SyncStorage + +[storage] +backend = syncstorage.storage.sql.SQLStorage +sqluri = ${MOZSVC_SQLURI} +standard_collections = true +quota_size = 5242880 +pool_size = 100 +pool_recycle = 3600 +reset_on_return = true +create_tables = true +max_post_records = 4000 +batch_upload_enabled = true +force_consistent_sort_order = true + +[hawkauth] +secret = "TED KOPPEL IS A ROBOT" diff --git a/tools/integration_tests/requirements.txt b/tools/integration_tests/requirements.txt new file mode 100644 index 00000000..7f0036e5 --- /dev/null +++ b/tools/integration_tests/requirements.txt @@ -0,0 +1,10 @@ +hawkauthlib +konfig +pyramid +pyramid_hawkauth +requests +simplejson +tokenlib +unittest2 +webtest +wsgiproxy2 diff --git a/tools/integration_tests/run.py b/tools/integration_tests/run.py new file mode 100644 index 00000000..4f6c8ab2 --- /dev/null +++ b/tools/integration_tests/run.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import atexit +import os.path +import subprocess +import sys +from test_storage import TestStorage +from test_support import run_live_functional_tests +import time + + +DEBUG_BUILD = 'target/debug/syncstorage' +RELEASE_BUILD = '/app/bin/syncstorage' + +if __name__ == "__main__": + # When run as a script, this file will execute the + # functional tests against a live webserver. + target_binary = None + if os.path.exists(DEBUG_BUILD): + target_binary = DEBUG_BUILD + elif os.path.exists(RELEASE_BUILD): + target_binary = RELEASE_BUILD + else: + raise RuntimeError("Neither target/debug/syncstorage nor /app/bin/syncstorage were found.") + the_server_subprocess = subprocess.Popen('SYNC_MASTER_SECRET=secret0 ' + target_binary, shell=True) + ## TODO we should change this to watch for a log message on startup to know when to continue instead of sleeping for a fixed amount + time.sleep(20) + + def stop_subprocess(): + the_server_subprocess.terminate() + the_server_subprocess.wait() + + atexit.register(stop_subprocess) + + res = run_live_functional_tests(TestStorage, sys.argv) + sys.exit(res) diff --git a/tools/integration_tests/test_storage.py b/tools/integration_tests/test_storage.py new file mode 100644 index 00000000..f6e17340 --- /dev/null +++ b/tools/integration_tests/test_storage.py @@ -0,0 +1,2085 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Functional tests for the SyncStorage server protocol. + +This file runs tests to ensure the correct operation of the server +as specified in: + + http://docs.services.mozilla.com/storage/apis-1.5.html + +If there's an aspect of that spec that's not covered by a test in this file, +consider it a bug. + +""" + +import unittest2 + +import re +import json +import time +import random +import string +import urllib +import webtest +import contextlib +# import math +import simplejson + +from pyramid.interfaces import IAuthenticationPolicy +from test_support import StorageFunctionalTestCase + +import tokenlib + + +class ConflictError(Exception): + pass + + +class BackendError(Exception): + pass + + +WEAVE_INVALID_WBO = 8 # Invalid Weave Basic Object +WEAVE_SIZE_LIMIT_EXCEEDED = 17 # Size limit exceeded + +BATCH_MAX_IDS = 100 + + +def get_limit_config(request, limit): + """Get the configured value for the named size limit.""" + return request.registry.settings["storage." + limit] + + +def json_dumps(value): + """Decimal-aware version of json.dumps().""" + return simplejson.dumps(value, use_decimal=True) + + +def json_loads(value): + """Decimal-aware version of json.loads().""" + return simplejson.loads(value, use_decimal=True) + + +_PLD = '*' * 500 +_ASCII = string.ascii_letters + string.digits + + +def randtext(size=10): + return ''.join([random.choice(_ASCII) for i in range(size)]) + + +class TestStorage(StorageFunctionalTestCase): + """Storage testcases that only use the web API. + + These tests are suitable for running against both in-process and live + external web servers. + """ + + def setUp(self): + super(TestStorage, self).setUp() + self.root = '/1.5/%d' % (self.user_id,) + # Reset the storage to a known state, aka "empty". + self.retry_delete(self.root) + + @contextlib.contextmanager + def _switch_user(self): + orig_root = self.root + try: + with super(TestStorage, self)._switch_user(): + self.root = '/1.5/%d' % (self.user_id,) + yield + finally: + self.root = orig_root + + def retry_post_json(self, *args, **kwargs): + return self._retry_send(self.app.post_json, *args, **kwargs) + + def retry_put_json(self, *args, **kwargs): + return self._retry_send(self.app.put_json, *args, **kwargs) + + def retry_delete(self, *args, **kwargs): + return self._retry_send(self.app.delete, *args, **kwargs) + + def _retry_send(self, func, *args, **kwargs): + try: + return func(*args, **kwargs) + except webtest.AppError as ex: + if "409 " not in ex.args[0] and "503 " not in ex.args[0]: + raise ex + time.sleep(0.01) + return func(*args, **kwargs) + + def test_get_info_collections(self): + # xxx_col1 gets 3 items, xxx_col2 gets 5 items. + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)] + resp = self.retry_post_json(self.root + "/storage/xxx_col1", bsos) + ts1 = resp.json["modified"] + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + resp = self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + ts2 = resp.json["modified"] + # only those collections should appear in the query. + resp = self.app.get(self.root + '/info/collections') + res = resp.json + keys = sorted(list(res.keys())) + self.assertEquals(keys, ["xxx_col1", "xxx_col2"]) + self.assertEquals(res["xxx_col1"], ts1) + self.assertEquals(res["xxx_col2"], ts2) + # Updating items in xxx_col2, check timestamps. + bsos = [{"id": str(i).zfill(2), "payload": "yyy"} for i in range(2)] + resp = self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + self.assertTrue(ts2 < resp.json["modified"]) + ts2 = resp.json["modified"] + resp = self.app.get(self.root + '/info/collections') + res = resp.json + keys = sorted(list(res.keys())) + self.assertEquals(keys, ["xxx_col1", "xxx_col2"]) + self.assertEquals(res["xxx_col1"], ts1) + self.assertEquals(res["xxx_col2"], ts2) + + def test_get_collection_count(self): + # xxx_col1 gets 3 items, xxx_col2 gets 5 items. + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)] + self.retry_post_json(self.root + "/storage/xxx_col1", bsos) + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + # those counts should be reflected back in query. + resp = self.app.get(self.root + '/info/collection_counts') + res = resp.json + self.assertEquals(len(res), 2) + self.assertEquals(res["xxx_col1"], 3) + self.assertEquals(res["xxx_col2"], 5) + + def test_bad_cache(self): + # fixes #637332 + # the collection name <-> id mapper is temporarely cached to + # save a few requests. + # but should get purged when new collections are added + + # 1. get collection info + resp = self.app.get(self.root + '/info/collections') + numcols = len(resp.json) + + # 2. add a new collection + stuff + bso = {'id': '125', 'payload': _PLD} + self.retry_put_json(self.root + '/storage/xxxx/125', bso) + + # 3. get collection info again, should find the new ones + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(len(resp.json), numcols + 1) + + def test_get_collection_only(self): + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + + # non-existent collections appear as empty + resp = self.app.get(self.root + '/storage/nonexistent') + res = resp.json + self.assertEquals(res, []) + + # try just getting all items at once. + resp = self.app.get(self.root + '/storage/xxx_col2') + res = resp.json + res.sort() + self.assertEquals(res, ['00', '01', '02', '03', '04']) + self.assertEquals(int(resp.headers['X-Weave-Records']), 5) + + # trying various filters + + # "ids" + # Returns the ids for objects in the collection that are in the + # provided comma-separated list. + res = self.app.get(self.root + '/storage/xxx_col2?ids=01,03,17') + res = res.json + res.sort() + self.assertEquals(res, ['01', '03']) + + # "newer" + # Returns only ids for objects in the collection that have been last + # modified after the timestamp given. + + self.retry_delete(self.root + '/storage/xxx_col2') + + bso = {'id': '128', 'payload': 'x'} + res = self.retry_put_json(self.root + '/storage/xxx_col2/128', bso) + ts1 = float(res.headers["X-Last-Modified"]) + + bso = {'id': '129', 'payload': 'x'} + res = self.retry_put_json(self.root + '/storage/xxx_col2/129', bso) + ts2 = float(res.headers["X-Last-Modified"]) + + self.assertTrue(ts1 < ts2) + + res = self.app.get(self.root + '/storage/xxx_col2?newer=%s' % ts1) + self.assertEquals(res.json, ['129']) + + res = self.app.get(self.root + '/storage/xxx_col2?newer=%s' % ts2) + self.assertEquals(res.json, []) + + res = self.app.get( + self.root + '/storage/xxx_col2?newer=%s' % (ts1 - 1)) + self.assertEquals(sorted(res.json), ['128', '129']) + + # "older" + # Returns only ids for objects in the collection that have been last + # modified before the timestamp given. + + self.retry_delete(self.root + '/storage/xxx_col2') + + bso = {'id': '128', 'payload': 'x'} + res = self.retry_put_json(self.root + '/storage/xxx_col2/128', bso) + ts1 = float(res.headers["X-Last-Modified"]) + + bso = {'id': '129', 'payload': 'x'} + res = self.retry_put_json(self.root + '/storage/xxx_col2/129', bso) + ts2 = float(res.headers["X-Last-Modified"]) + + self.assertTrue(ts1 < ts2) + + res = self.app.get(self.root + '/storage/xxx_col2?older=%s' % ts1) + self.assertEquals(res.json, []) + + res = self.app.get(self.root + '/storage/xxx_col2?older=%s' % ts2) + self.assertEquals(res.json, ['128']) + + res = self.app.get( + self.root + '/storage/xxx_col2?older=%s' % (ts2 + 1)) + self.assertEquals(sorted(res.json), ['128', '129']) + + qs = '?older=%s&newer=%s' % (ts2 + 1, ts1) + res = self.app.get(self.root + '/storage/xxx_col2' + qs) + self.assertEquals(sorted(res.json), ['129']) + + # "full" + # If defined, returns the full BSO, rather than just the id. + res = self.app.get(self.root + '/storage/xxx_col2?full=1') + keys = list(res.json[0].keys()) + keys.sort() + wanted = ['id', 'modified', 'payload'] + self.assertEquals(keys, wanted) + + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertTrue(isinstance(res.json, list)) + + # "limit" + # Sets the maximum number of ids that will be returned + self.retry_delete(self.root + '/storage/xxx_col2') + + bsos = [] + for i in range(10): + bso = {'id': str(i).zfill(2), 'payload': 'x', 'sortindex': i} + bsos.append(bso) + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + + query_url = self.root + '/storage/xxx_col2?sort=index' + res = self.app.get(query_url) + all_items = res.json + self.assertEquals(len(all_items), 10) + + res = self.app.get(query_url + '&limit=2') + self.assertEquals(res.json, all_items[:2]) + + # "offset" + # Skips over items that have already been returned. + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&limit=3&offset=' + next_offset) + self.assertEquals(res.json, all_items[2:5]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + self.assertTrue("X-Weave-Next-Offset" not in res.headers) + + res = self.app.get( + query_url + '&limit=10000&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + self.assertTrue("X-Weave-Next-Offset" not in res.headers) + + # "offset" again, this time ordering by descending timestamp. + query_url = self.root + '/storage/xxx_col2?sort=newest' + res = self.app.get(query_url) + all_items = res.json + self.assertEquals(len(all_items), 10) + + res = self.app.get(query_url + '&limit=2') + self.assertEquals(res.json, all_items[:2]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&limit=3&offset=' + next_offset) + self.assertEquals(res.json, all_items[2:5]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + self.assertTrue("X-Weave-Next-Offset" not in res.headers) + + res = self.app.get( + query_url + '&limit=10000&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + + # "offset" again, this time ordering by ascending timestamp. + query_url = self.root + '/storage/xxx_col2?sort=oldest' + res = self.app.get(query_url) + all_items = res.json + self.assertEquals(len(all_items), 10) + + res = self.app.get(query_url + '&limit=2') + self.assertEquals(res.json, all_items[:2]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&limit=3&offset=' + next_offset) + self.assertEquals(res.json, all_items[2:5]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + self.assertTrue("X-Weave-Next-Offset" not in res.headers) + + res = self.app.get( + query_url + '&limit=10000&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + + # "offset" once more, this time with no explicit ordering + query_url = self.root + '/storage/xxx_col2?' + res = self.app.get(query_url) + all_items = res.json + self.assertEquals(len(all_items), 10) + + res = self.app.get(query_url + '&limit=2') + self.assertEquals(res.json, all_items[:2]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&limit=3&offset=' + next_offset) + self.assertEquals(res.json, all_items[2:5]) + + next_offset = res.headers["X-Weave-Next-Offset"] + res = self.app.get(query_url + '&offset=' + next_offset) + self.assertEquals(res.json, all_items[5:]) + self.assertTrue("X-Weave-Next-Offset" not in res.headers) + + res = self.app.get( + query_url + '&limit=10000&offset=' + next_offset) + + # "sort" + # 'newest': Orders by timestamp number (newest first) + # 'oldest': Orders by timestamp number (oldest first) + # 'index': Orders by the sortindex descending (highest weight first) + self.retry_delete(self.root + '/storage/xxx_col2') + + for index, sortindex in (('00', -1), ('01', 34), ('02', 12)): + bso = {'id': index, 'payload': 'x', 'sortindex': sortindex} + self.retry_post_json(self.root + '/storage/xxx_col2', [bso]) + + res = self.app.get(self.root + '/storage/xxx_col2?sort=newest') + res = res.json + self.assertEquals(res, ['02', '01', '00']) + + res = self.app.get(self.root + '/storage/xxx_col2?sort=oldest') + res = res.json + self.assertEquals(res, ['00', '01', '02']) + + res = self.app.get(self.root + '/storage/xxx_col2?sort=index') + res = res.json + self.assertEquals(res, ['01', '02', '00']) + + def test_alternative_formats(self): + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + + # application/json + res = self.app.get(self.root + '/storage/xxx_col2', + headers=[('Accept', 'application/json')]) + self.assertEquals(res.content_type.split(";")[0], 'application/json') + + res = res.json + res.sort() + self.assertEquals(res, ['00', '01', '02', '03', '04']) + + # application/newlines + res = self.app.get(self.root + '/storage/xxx_col2', + headers=[('Accept', 'application/newlines')]) + self.assertEquals(res.content_type, 'application/newlines') + + self.assertTrue(res.body.endswith(b'\n')) + res = [json_loads(line) for line in res.body.decode( + 'utf-8').strip().split('\n')] + res.sort() + self.assertEquals(res, ['00', '01', '02', '03', '04']) + + # unspecified format defaults to json + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(res.content_type.split(";")[0], 'application/json') + + # unkown format gets a 406 + self.app.get(self.root + '/storage/xxx_col2', + headers=[('Accept', 'x/yy')], status=406) + + def test_set_collection_with_if_modified_since(self): + # Create five items with different timestamps. + for i in range(5): + bsos = [{"id": str(i).zfill(2), "payload": "xxx"}] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + # Get them all, along with their timestamps. + res = self.app.get(self.root + '/storage/xxx_col2?full=true').json + self.assertEquals(len(res), 5) + timestamps = sorted([r["modified"] for r in res]) + # The timestamp of the collection should be the max of all those. + self.app.get(self.root + "/storage/xxx_col2", headers={ + "X-If-Modified-Since": str(timestamps[0]) + }, status=200) + res = self.app.get(self.root + "/storage/xxx_col2", headers={ + "X-If-Modified-Since": str(timestamps[-1]) + }, status=304) + self.assertTrue("X-Last-Modified" in res.headers) + + def test_get_item(self): + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + # grabbing object 1 from xxx_col2 + res = self.app.get(self.root + '/storage/xxx_col2/01') + res = res.json + keys = list(res.keys()) + keys.sort() + self.assertEquals(keys, ['id', 'modified', 'payload']) + self.assertEquals(res['id'], '01') + + # unexisting object + self.app.get(self.root + '/storage/xxx_col2/99', status=404) + + # using x-if-modified-since header. + self.app.get(self.root + '/storage/xxx_col2/01', headers={ + "X-If-Modified-Since": str(res["modified"]) + }, status=304) + self.app.get(self.root + '/storage/xxx_col2/01', headers={ + "X-If-Modified-Since": str(res["modified"] + 1) + }, status=304) + res = self.app.get(self.root + '/storage/xxx_col2/01', headers={ + "X-If-Modified-Since": str(res["modified"] - 1) + }) + self.assertEquals(res.json['id'], '01') + + def test_set_item(self): + # let's create an object + bso = {'payload': _PLD} + self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + res = self.app.get(self.root + '/storage/xxx_col2/12345') + res = res.json + self.assertEquals(res['payload'], _PLD) + + # now let's update it + bso = {'payload': 'YYY'} + self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + res = self.app.get(self.root + '/storage/xxx_col2/12345') + res = res.json + self.assertEquals(res['payload'], 'YYY') + + def test_set_collection(self): + # sending two bsos + bso1 = {'id': '12', 'payload': _PLD} + bso2 = {'id': '13', 'payload': _PLD} + bsos = [bso1, bso2] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + + # checking what we did + res = self.app.get(self.root + '/storage/xxx_col2/12') + res = res.json + self.assertEquals(res['payload'], _PLD) + res = self.app.get(self.root + '/storage/xxx_col2/13') + res = res.json + self.assertEquals(res['payload'], _PLD) + + # one more time, with changes + bso1 = {'id': '13', 'payload': 'XyX'} + bso2 = {'id': '14', 'payload': _PLD} + bsos = [bso1, bso2] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + + # checking what we did + res = self.app.get(self.root + '/storage/xxx_col2/14') + res = res.json + self.assertEquals(res['payload'], _PLD) + res = self.app.get(self.root + '/storage/xxx_col2/13') + res = res.json + self.assertEquals(res['payload'], 'XyX') + + # sending two bsos with one bad sortindex + bso1 = {'id': 'one', 'payload': _PLD} + bso2 = {'id': 'two', 'payload': _PLD, + 'sortindex': 'FAIL'} + bsos = [bso1, bso2] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + self.app.get(self.root + '/storage/xxx_col2/two', status=404) + + def test_set_collection_input_formats(self): + # If we send with application/newlines it should work. + bso1 = {'id': '12', 'payload': _PLD} + bso2 = {'id': '13', 'payload': _PLD} + bsos = [bso1, bso2] + body = "\n".join(json_dumps(bso) for bso in bsos) + self.app.post(self.root + '/storage/xxx_col2', body, headers={ + "Content-Type": "application/newlines" + }) + items = self.app.get(self.root + "/storage/xxx_col2").json + self.assertEquals(len(items), 2) + # If we send an unknown content type, we get an error. + self.retry_delete(self.root + "/storage/xxx_col2") + body = json_dumps(bsos) + self.app.post(self.root + '/storage/xxx_col2', body, headers={ + "Content-Type": "application/octet-stream" + }, status=415) + items = self.app.get(self.root + "/storage/xxx_col2").json + self.assertEquals(len(items), 0) + + def test_set_item_input_formats(self): + # If we send with application/json it should work. + body = json_dumps({'payload': _PLD}) + self.app.put(self.root + '/storage/xxx_col2/TEST', body, headers={ + "Content-Type": "application/json" + }) + item = self.app.get(self.root + "/storage/xxx_col2/TEST").json + self.assertEquals(item["payload"], _PLD) + # If we send json with some other content type, it should fail + self.retry_delete(self.root + "/storage/xxx_col2") + self.app.put(self.root + '/storage/xxx_col2/TEST', body, headers={ + "Content-Type": "application/octet-stream" + }, status=415) + self.app.get(self.root + "/storage/xxx_col2/TEST", status=404) + # Unless we use text/plain, which is a special bw-compat case. + self.app.put(self.root + '/storage/xxx_col2/TEST', body, headers={ + "Content-Type": "text/plain" + }) + item = self.app.get(self.root + "/storage/xxx_col2/TEST").json + self.assertEquals(item["payload"], _PLD) + + def test_app_newlines_when_payloads_contain_newlines(self): + # Send some application/newlines with embedded newline chars. + bsos = [ + {'id': '01', 'payload': 'hello\nworld'}, + {'id': '02', 'payload': '\nmarco\npolo\n'}, + ] + body = "\n".join(json_dumps(bso) for bso in bsos) + self.assertEquals(len(body.split("\n")), 2) + self.app.post(self.root + '/storage/xxx_col2', body, headers={ + "Content-Type": "application/newlines" + }) + # Read them back as JSON list, check payloads. + items = self.app.get(self.root + "/storage/xxx_col2?full=1").json + self.assertEquals(len(items), 2) + items.sort(key=lambda bso: bso["id"]) + self.assertEquals(items[0]["payload"], bsos[0]["payload"]) + self.assertEquals(items[1]["payload"], bsos[1]["payload"]) + # Read them back as application/newlines, check payloads. + res = self.app.get(self.root + "/storage/xxx_col2?full=1", headers={ + "Accept": "application/newlines", + }) + items = [json_loads(line) for line in res.body.decode( + 'utf-8').strip().split('\n')] + self.assertEquals(len(items), 2) + items.sort(key=lambda bso: bso["id"]) + self.assertEquals(items[0]["payload"], bsos[0]["payload"]) + self.assertEquals(items[1]["payload"], bsos[1]["payload"]) + + def test_collection_usage(self): + self.retry_delete(self.root + "/storage") + + bso1 = {'id': '13', 'payload': 'XyX'} + bso2 = {'id': '14', 'payload': _PLD} + bsos = [bso1, bso2] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + + res = self.app.get(self.root + '/info/collection_usage') + usage = res.json + xxx_col2_size = usage['xxx_col2'] + wanted = (len(bso1['payload']) + len(bso2['payload'])) / 1024.0 + self.assertEqual(round(xxx_col2_size, 2), round(wanted, 2)) + + def test_delete_collection_items(self): + # creating a collection of three + bso1 = {'id': '12', 'payload': _PLD} + bso2 = {'id': '13', 'payload': _PLD} + bso3 = {'id': '14', 'payload': _PLD} + bsos = [bso1, bso2, bso3] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 3) + + # deleting all items + self.retry_delete(self.root + '/storage/xxx_col2') + items = self.app.get(self.root + '/storage/xxx_col2').json + self.assertEquals(len(items), 0) + + # Deletes the ids for objects in the collection that are in the + # provided comma-separated list. + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 3) + self.retry_delete(self.root + '/storage/xxx_col2?ids=12,14') + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 1) + self.retry_delete(self.root + '/storage/xxx_col2?ids=13') + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 0) + + def test_delete_item(self): + # creating a collection of three + bso1 = {'id': '12', 'payload': _PLD} + bso2 = {'id': '13', 'payload': _PLD} + bso3 = {'id': '14', 'payload': _PLD} + bsos = [bso1, bso2, bso3] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 3) + ts = float(res.headers['X-Last-Modified']) + + # deleting item 13 + self.retry_delete(self.root + '/storage/xxx_col2/13') + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 2) + + # unexisting item should return a 404 + self.retry_delete(self.root + '/storage/xxx_col2/12982', status=404) + + # The collection should get an updated timestsamp. + res = self.app.get(self.root + '/info/collections') + self.assertTrue(ts < float(res.headers['X-Last-Modified'])) + + def test_delete_storage(self): + # creating a collection of three + bso1 = {'id': '12', 'payload': _PLD} + bso2 = {'id': '13', 'payload': _PLD} + bso3 = {'id': '14', 'payload': _PLD} + bsos = [bso1, bso2, bso3] + self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 3) + + # deleting all + self.retry_delete(self.root + '/storage') + items = self.app.get(self.root + '/storage/xxx_col2').json + self.assertEquals(len(items), 0) + self.retry_delete(self.root + '/storage/xxx_col2', status=200) + self.assertEquals(len(items), 0) + + def test_x_timestamp_header(self): + # This can't be run against a live server. + if self.distant: + raise unittest2.SkipTest + + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + + now = round(time.time(), 2) + time.sleep(0.01) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertTrue(now <= float(res.headers['X-Weave-Timestamp'])) + + # getting the timestamp with a PUT + now = round(time.time(), 2) + time.sleep(0.01) + bso = {'payload': _PLD} + res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + self.assertTrue(now <= float(res.headers['X-Weave-Timestamp'])) + self.assertTrue(abs(now - + float(res.headers['X-Weave-Timestamp'])) <= 200) + + # getting the timestamp with a POST + now = round(time.time(), 2) + time.sleep(0.01) + bso1 = {'id': '12', 'payload': _PLD} + bso2 = {'id': '13', 'payload': _PLD} + bsos = [bso1, bso2] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + self.assertTrue(now <= float(res.headers['X-Weave-Timestamp'])) + + def test_ifunmodifiedsince(self): + bso = {'id': '12345', 'payload': _PLD} + res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + # Using an X-If-Unmodified-Since in the past should cause 412s. + ts = str(float(res.headers['X-Last-Modified']) - 1) + bso = {'id': '12345', 'payload': _PLD + "XXX"} + res = self.retry_put_json( + self.root + '/storage/xxx_col2/12345', bso, + headers=[('X-If-Unmodified-Since', ts)], + status=412) + self.assertTrue("X-Last-Modified" in res.headers) + res = self.retry_delete( + self.root + '/storage/xxx_col2/12345', + headers=[('X-If-Unmodified-Since', ts)], + status=412) + self.assertTrue("X-Last-Modified" in res.headers) + self.retry_post_json( + self.root + '/storage/xxx_col2', [bso], + headers=[('X-If-Unmodified-Since', ts)], + status=412) + self.retry_delete( + self.root + '/storage/xxx_col2?ids=12345', + headers=[('X-If-Unmodified-Since', ts)], + status=412) + self.app.get( + self.root + '/storage/xxx_col2/12345', + headers=[('X-If-Unmodified-Since', ts)], + status=412) + self.app.get( + self.root + '/storage/xxx_col2', + headers=[('X-If-Unmodified-Since', ts)], + status=412) + # Deleting items from a collection should give 412 even if some + # other, unrelated item in the collection has been modified. + ts = res.headers['X-Last-Modified'] + res2 = self.retry_put_json(self.root + '/storage/xxx_col2/54321', { + 'payload': _PLD, + }) + self.retry_delete( + self.root + '/storage/xxx_col2?ids=12345', + headers=[('X-If-Unmodified-Since', ts)], + status=412) + ts = res2.headers['X-Last-Modified'] + # All of those should have left the BSO unchanged + res2 = self.app.get(self.root + '/storage/xxx_col2/12345') + self.assertEquals(res2.json['payload'], _PLD) + self.assertEquals(res2.headers['X-Last-Modified'], + res.headers['X-Last-Modified']) + # Using an X-If-Unmodified-Since equal to + # X-Last-Modified should allow the request to succeed. + res = self.retry_post_json( + self.root + '/storage/xxx_col2', [bso], + headers=[('X-If-Unmodified-Since', ts)], + status=200) + ts = res.headers['X-Last-Modified'] + self.app.get( + self.root + '/storage/xxx_col2/12345', + headers=[('X-If-Unmodified-Since', ts)], + status=200) + self.retry_delete( + self.root + '/storage/xxx_col2/12345', + headers=[('X-If-Unmodified-Since', ts)], + status=200) + res = self.retry_put_json( + self.root + '/storage/xxx_col2/12345', bso, + headers=[('X-If-Unmodified-Since', '0')], + status=200) + ts = res.headers['X-Last-Modified'] + self.app.get( + self.root + '/storage/xxx_col2', + headers=[('X-If-Unmodified-Since', ts)], + status=200) + self.retry_delete( + self.root + '/storage/xxx_col2?ids=12345', + headers=[('X-If-Unmodified-Since', ts)], + status=200) + + def test_quota(self): + res = self.app.get(self.root + '/info/quota') + old_used = res.json[0] + bso = {'payload': _PLD} + self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + res = self.app.get(self.root + '/info/quota') + used = res.json[0] + self.assertEquals(used - old_used, len(_PLD) / 1024.0) + + def test_overquota(self): + # This can't be run against a live server. + raise unittest2.SkipTest + if self.distant: + raise unittest2.SkipTest + + # Clear out any data that's already in the store. + self.retry_delete(self.root + "/storage") + + # Set a low quota for the storage. + self.config.registry.settings["storage.quota_size"] = 700 + + # Check the the remaining quota is correctly reported. + bso = {'payload': _PLD} + res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + wanted = str(round(200 / 1024.0, 2)) + self.assertEquals(res.headers['X-Weave-Quota-Remaining'], wanted) + + # Set the quota so that they're over their limit. + self.config.registry.settings["storage.quota_size"] = 10 + bso = {'payload': _PLD} + res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso, + status=403) + self.assertEquals(res.content_type.split(";")[0], 'application/json') + self.assertEquals(res.json["status"], "quota-exceeded") + + def test_get_collection_ttl(self): + bso = {'payload': _PLD, 'ttl': 0} + res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso) + time.sleep(1.1) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(res.json, []) + + bso = {'payload': _PLD, 'ttl': 2} + res = self.retry_put_json(self.root + '/storage/xxx_col2/123456', bso) + + # it should exists now + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 1) + + # trying a second put again + self.retry_put_json(self.root + '/storage/xxx_col2/123456', bso) + + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 1) + time.sleep(2.1) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(len(res.json), 0) + + def test_multi_item_post_limits(self): + res = self.app.get(self.root + '/info/configuration') + try: + max_bytes = res.json['max_post_bytes'] + max_count = res.json['max_post_records'] + max_req_bytes = res.json['max_request_bytes'] + except KeyError: + # Can't run against live server if it doesn't + # report the right config options. + if self.distant: + raise unittest2.SkipTest + max_bytes = get_limit_config(self.config, 'max_post_bytes') + max_count = get_limit_config(self.config, 'max_post_records') + max_req_bytes = get_limit_config(self.config, 'max_request_bytes') + + # Uploading max_count-5 small objects should succeed. + bsos = [{'id': str(i).zfill(2), + 'payload': 'X'} for i in range(max_count - 5)] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = res.json + self.assertEquals(len(res['success']), max_count - 5) + self.assertEquals(len(res['failed']), 0) + + # Uploading max_count+5 items should produce five failures. + bsos = [{'id': str(i).zfill(2), + 'payload': 'X'} for i in range(max_count + 5)] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = res.json + self.assertEquals(len(res['success']), max_count) + self.assertEquals(len(res['failed']), 5) + + # Uploading items such that the last item puts us over the + # cumulative limit on payload size, should produce 1 failure. + # The item_size here is arbitrary, so I made it a prime in kB. + item_size = (227 * 1024) + max_items, leftover = divmod(max_bytes, item_size) + bsos = [{'id': str(i).zfill(2), 'payload': 'X' * item_size} + for i in range(max_items)] + bsos.append({'id': str(max_items), 'payload': 'X' * (leftover + 1)}) + + # Check that we don't go over the limit on raw request bytes, + # which would get us rejected in production with a 413. + self.assertTrue(len(json.dumps(bsos)) < max_req_bytes) + + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = res.json + self.assertEquals(len(res['success']), max_items) + self.assertEquals(len(res['failed']), 1) + + def test_weird_args(self): + # pushing some data in xxx_col2 + bsos = [{'id': str(i).zfill(2), 'payload': _PLD} for i in range(10)] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = res.json + + # trying weird args and make sure the server returns 400s + # Note: "Offset" is a string since the bsoid could be anything. + # skipping that for now. + args = ('newer', 'older', 'limit', 'offset') + for arg in args: + value = randtext() + self.app.get(self.root + '/storage/xxx_col2?%s=%s' % (arg, value), + status=400) + + # what about a crazy ids= string ? + ids = ','.join([randtext(10) for i in range(100)]) + res = self.app.get(self.root + '/storage/xxx_col2?ids=%s' % ids) + self.assertEquals(res.json, []) + + # trying unexpected args - they should not break + self.app.get(self.root + '/storage/xxx_col2?blabla=1', + status=200) + + def test_guid_deletion(self): + # pushing some data in xxx_col2 + bsos = [{'id': '6820f3ca-6e8a-4ff4-8af7-8b3625d7d65%d' % i, + 'payload': _PLD} for i in range(5)] + res = self.retry_post_json(self.root + '/storage/passwords', bsos) + res = res.json + self.assertEquals(len(res["success"]), 5) + + # now deleting some of them + ids = ','.join(['6820f3ca-6e8a-4ff4-8af7-8b3625d7d65%d' % i + for i in range(2)]) + + self.retry_delete(self.root + '/storage/passwords?ids=%s' % ids) + + res = self.app.get(self.root + '/storage/passwords?ids=%s' % ids) + self.assertEqual(len(res.json), 0) + res = self.app.get(self.root + '/storage/passwords') + self.assertEqual(len(res.json), 3) + + def test_specifying_ids_with_percent_encoded_query_string(self): + # create some items + bsos = [{'id': 'test-%d' % i, 'payload': _PLD} for i in range(5)] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = res.json + self.assertEquals(len(res["success"]), 5) + # now delete some of them + ids = ','.join(['test-%d' % i for i in range(2)]) + ids = urllib.request.quote(ids) + self.retry_delete(self.root + '/storage/xxx_col2?ids=%s' % ids) + # check that the correct items were deleted + res = self.app.get(self.root + '/storage/xxx_col2?ids=%s' % ids) + self.assertEqual(len(res.json), 0) + res = self.app.get(self.root + '/storage/xxx_col2') + self.assertEqual(len(res.json), 3) + + def test_timestamp_numbers_are_decimals(self): + # Create five items with different timestamps. + for i in range(5): + bsos = [{"id": str(i).zfill(2), "payload": "xxx"}] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + + # make sure the server returns only proper precision timestamps. + resp = self.app.get(self.root + '/storage/xxx_col2?full=1') + bsos = json_loads(resp.body) + timestamps = [] + for bso in bsos: + ts = bso['modified'] + # timestamps could be on the hundred seconds (.10) or on the + # second (.0) and the zero could be dropped. We just don't want + # anything beyond milisecond. + self.assertLessEqual(len(str(ts).split(".")[-1]), 2) + timestamps.append(ts) + + timestamps.sort() + + # try a newer filter now, to get the last two objects + ts = float(timestamps[-3]) + + # Returns only ids for objects in the collection that have been + # last modified since the timestamp given. + res = self.app.get(self.root + '/storage/xxx_col2?newer=%s' % ts) + res = res.json + try: + self.assertEquals(sorted(res), ['03', '04']) + except AssertionError: + # need to display the whole collection to understand the issue + msg = 'Timestamp used: %s' % ts + msg += ' ' + self.app.get(self.root + + '/storage/xxx_col2?full=1').body + msg += ' Timestamps received: %s' % str(timestamps) + msg += ' Result of newer query: %s' % res + raise AssertionError(msg) + + def test_strict_newer(self): + # send two bsos in the 'meh' collection + bso1 = {'id': '01', 'payload': _PLD} + bso2 = {'id': '02', 'payload': _PLD} + bsos = [bso1, bso2] + res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos) + ts = float(res.headers["X-Last-Modified"]) + + # send two more bsos + bso3 = {'id': '03', 'payload': _PLD} + bso4 = {'id': '04', 'payload': _PLD} + bsos = [bso3, bso4] + res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos) + + # asking for bsos using newer=ts where newer is the timestamp + # of bso 1 and 2, should not return them + res = self.app.get(self.root + '/storage/xxx_meh?newer=%s' % ts) + res = res.json + self.assertEquals(sorted(res), ['03', '04']) + + def test_strict_older(self): + # send two bsos in the 'xxx_meh' collection + bso1 = {'id': '01', 'payload': _PLD} + bso2 = {'id': '02', 'payload': _PLD} + bsos = [bso1, bso2] + res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos) + + # send two more bsos + bso3 = {'id': '03', 'payload': _PLD} + bso4 = {'id': '04', 'payload': _PLD} + bsos = [bso3, bso4] + res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos) + ts = float(res.headers["X-Last-Modified"]) + + # asking for bsos using older=ts where older is the timestamp + # of bso 3 and 4, should not return them + res = self.app.get(self.root + '/storage/xxx_meh?older=%s' % ts) + res = res.json + self.assertEquals(sorted(res), ['01', '02']) + + def test_handling_of_invalid_json_in_bso_uploads(self): + # Single upload with JSON that's not a BSO. + bso = "notabso" + res = self.retry_put_json(self.root + '/storage/xxx_col2/invalid', bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + + bso = 42 + res = self.retry_put_json(self.root + '/storage/xxx_col2/invalid', bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + + bso = {'id': ["01", "02"], 'payload': {'3': '4'}} + res = self.retry_put_json(self.root + '/storage/xxx_col2/invalid', bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + + # Batch upload with JSON that's not a list of BSOs + bsos = "notalist" + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + + bsos = 42 + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + + # Batch upload a list with something that's not a valid data dict. + # It should fail out entirely, as the input is seriously broken. + bsos = [{'id': '01', 'payload': 'GOOD'}, "BAD"] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos, + status=400) + + # Batch upload a list with something that's an invalid BSO. + # It should process the good entry and fail for the bad. + bsos = [{'id': '01', 'payload': 'GOOD'}, {'id': '02', 'invalid': 'ya'}] + res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos) + res = res.json + self.assertEquals(len(res['success']), 1) + self.assertEquals(len(res['failed']), 1) + + def test_handling_of_invalid_bso_fields(self): + coll_url = self.root + "/storage/xxx_col2" + # Invalid ID - unacceptable characters. + # The newline cases are especially nuanced because \n + # gets special treatment from the regex library. + bso = {"id": "A\nB", "payload": "testing"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + bso = {"id": "A\n", "payload": "testing"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + bso = {"id": "\nN", "payload": "testing"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + bso = {"id": "A\tB", "payload": "testing"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + # Invalid ID - empty string is not acceptable. + bso = {"id": "", "payload": "testing"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + # Invalid ID - too long + bso = {"id": "X" * 65, "payload": "testing"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + # Commenting out this test. + # This uses the same invalid BSO from above, which should return a 400 + """ + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, status=404) + """ + # Invalid sortindex - not an integer + bso = {"id": "TEST", "payload": "testing", "sortindex": "xxx_meh"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + # Invalid sortindex - not an integer + bso = {"id": "TEST", "payload": "testing", "sortindex": "2.6"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + # Invalid sortindex - larger than max value + bso = {"id": "TEST", "payload": "testing", "sortindex": "1" + "0" * 9} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + # Invalid payload - not a string + bso = {"id": "TEST", "payload": 42} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + # Invalid ttl - not an integer + bso = {"id": "TEST", "payload": "testing", "ttl": "eh?"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + # Invalid ttl - not an integer + bso = {"id": "TEST", "payload": "testing", "ttl": "4.2"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, + status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + # Invalid BSO - unknown field + bso = {"id": "TEST", "unexpected": "spanish-inquisition"} + res = self.retry_post_json(coll_url, [bso]) + self.assertTrue(res.json["failed"] and not res.json["success"]) + res = self.retry_put_json(coll_url + "/" + bso["id"], bso, status=400) + self.assertEquals(res.json, WEAVE_INVALID_WBO) + + def test_that_batch_gets_are_limited_to_max_number_of_ids(self): + bso = {"id": "01", "payload": "testing"} + self.retry_put_json(self.root + "/storage/xxx_col2/01", bso) + + # Getting with less than the limit works OK. + ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS - 1)) + res = self.app.get(self.root + "/storage/xxx_col2?ids=" + ids) + self.assertEquals(res.json, ["01"]) + + # Getting with equal to the limit works OK. + ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS)) + res = self.app.get(self.root + "/storage/xxx_col2?ids=" + ids) + self.assertEquals(res.json, ["01"]) + + # Getting with more than the limit fails. + ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS + 1)) + self.app.get(self.root + "/storage/xxx_col2?ids=" + ids, status=400) + + def test_that_batch_deletes_are_limited_to_max_number_of_ids(self): + bso = {"id": "01", "payload": "testing"} + + # Deleting with less than the limit works OK. + self.retry_put_json(self.root + "/storage/xxx_col2/01", bso) + ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS - 1)) + self.retry_delete(self.root + "/storage/xxx_col2?ids=" + ids) + + # Deleting with equal to the limit works OK. + self.retry_put_json(self.root + "/storage/xxx_col2/01", bso) + ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS)) + self.retry_delete(self.root + "/storage/xxx_col2?ids=" + ids) + + # Deleting with more than the limit fails. + self.retry_put_json(self.root + "/storage/xxx_col2/01", bso) + ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS + 1)) + self.retry_delete(self.root + "/storage/xxx_col2?ids=" + ids, + status=400) + + def test_that_expired_items_can_be_overwritten_via_PUT(self): + # Upload something with a small ttl. + bso = {"payload": "XYZ", "ttl": 0} + self.retry_put_json(self.root + "/storage/xxx_col2/TEST", bso) + # Wait for it to expire. + time.sleep(0.02) + self.app.get(self.root + "/storage/xxx_col2/TEST", status=404) + # Overwriting it should still work. + bso = {"payload": "XYZ", "ttl": 42} + self.retry_put_json(self.root + "/storage/xxx_col2/TEST", bso) + + def test_if_modified_since_on_info_views(self): + # Store something, so the views have a modified time > 0. + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)] + self.retry_post_json(self.root + "/storage/xxx_col1", bsos) + INFO_VIEWS = ("/info/collections", "/info/quota", + "/info/collection_usage", "/info/collection_counts") + # Get the initial last-modified version. + r = self.app.get(self.root + "/info/collections") + ts1 = float(r.headers["X-Last-Modified"]) + self.assertTrue(ts1 > 0) + # With X-I-M-S set before latest change, all should give a 200. + headers = {"X-If-Modified-Since": str(ts1 - 1)} + for view in INFO_VIEWS: + self.app.get(self.root + view, headers=headers, status=200) + # With X-I-M-S set to after latest change , all should give a 304. + headers = {"X-If-Modified-Since": str(ts1)} + for view in INFO_VIEWS: + self.app.get(self.root + view, headers=headers, status=304) + # Change a collection. + bso = {"payload": "TEST"} + r = self.retry_put_json(self.root + "/storage/xxx_col2/TEST", bso) + ts2 = r.headers["X-Last-Modified"] + # Using the previous version should read the updated data. + headers = {"X-If-Modified-Since": str(ts1)} + for view in INFO_VIEWS: + self.app.get(self.root + view, headers=headers, status=200) + # Using the new timestamp should produce 304s. + headers = {"X-If-Modified-Since": str(ts2)} + for view in INFO_VIEWS: + self.app.get(self.root + view, headers=headers, status=304) + # XXX TODO: the storage-level timestamp is not tracked correctly + # after deleting a collection, so this test fails for now. + # # Delete a collection. + # r = self.retry_delete(self.root + "/storage/xxx_col2") + # ts3 = r.headers["X-Last-Modified"] + # # Using the previous timestamp should read the updated data. + # headers = {"X-If-Modified-Since": str(ts2)} + # for view in INFO_VIEWS: + # self.app.get(self.root + view, headers=headers, status=200) + # # Using the new timestamp should produce 304s. + # headers = {"X-If-Modified-Since": str(ts3)} + # for view in INFO_VIEWS: + # self.app.get(self.root + view, headers=headers, status=304) + + def test_that_x_last_modified_is_sent_for_all_get_requests(self): + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)] + self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + r = self.app.get(self.root + "/info/collections") + self.assertTrue("X-Last-Modified" in r.headers) + r = self.app.get(self.root + "/info/collection_counts") + self.assertTrue("X-Last-Modified" in r.headers) + r = self.app.get(self.root + "/storage/xxx_col2") + self.assertTrue("X-Last-Modified" in r.headers) + r = self.app.get(self.root + "/storage/xxx_col2/01") + self.assertTrue("X-Last-Modified" in r.headers) + + def test_update_of_ttl_without_sending_data(self): + bso = {"payload": "x", "ttl": 1} + self.retry_put_json(self.root + "/storage/xxx_col2/TEST1", bso) + self.retry_put_json(self.root + "/storage/xxx_col2/TEST2", bso) + # Before those expire, update ttl on one that exists + # and on one that does not. + time.sleep(0.2) + bso = {"ttl": 10} + self.retry_put_json(self.root + "/storage/xxx_col2/TEST2", bso) + self.retry_put_json(self.root + "/storage/xxx_col2/TEST3", bso) + # Update some other field on TEST1, which should leave ttl untouched. + bso = {"sortindex": 3} + self.retry_put_json(self.root + "/storage/xxx_col2/TEST1", bso) + # If we wait, TEST1 should expire but the others should not. + time.sleep(0.8) + items = self.app.get(self.root + "/storage/xxx_col2?full=1").json + items = dict((item["id"], item) for item in items) + self.assertEquals(sorted(list(items.keys())), ["TEST2", "TEST3"]) + # The existing item should have retained its payload. + # The new item should have got a default payload of empty string. + self.assertEquals(items["TEST2"]["payload"], "x") + self.assertEquals(items["TEST3"]["payload"], "") + ts2 = items["TEST2"]["modified"] + ts3 = items["TEST3"]["modified"] + self.assertTrue(ts2 < ts3) + + def test_bulk_update_of_ttls_without_sending_data(self): + # Create 5 BSOs with a ttl of 1 second. + bsos = [{"id": str(i).zfill(2), + "payload": "x", "ttl": 1} for i in range(5)] + r = self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + ts1 = float(r.headers["X-Last-Modified"]) + # Before they expire, bulk-update the ttl to something longer. + # Also send data for some that don't exist yet. + # And just to be really tricky, we're also going to update + # one of the payloads at the same time. + time.sleep(0.2) + bsos = [{"id": str(i).zfill(2), "ttl": 10} for i in range(3, 7)] + bsos[0]["payload"] = "xx" + r = self.retry_post_json(self.root + "/storage/xxx_col2", bsos) + self.assertEquals(len(r.json["success"]), 4) + ts2 = float(r.headers["X-Last-Modified"]) + # If we wait then items 0, 1, 2 should have expired. + # Items 3, 4, 5, 6 should still exist. + time.sleep(0.8) + items = self.app.get(self.root + "/storage/xxx_col2?full=1").json + items = dict((item["id"], item) for item in items) + self.assertEquals(sorted(list(items.keys())), ["03", "04", "05", "06"]) + # Items 3 and 4 should have the specified payloads. + # Items 5 and 6 should have payload defaulted to empty string. + self.assertEquals(items["03"]["payload"], "xx") + self.assertEquals(items["04"]["payload"], "x") + self.assertEquals(items["05"]["payload"], "") + self.assertEquals(items["06"]["payload"], "") + # All items created or modified by the request should get their + # timestamps update. Just bumping the ttl should not bump timestamp. + self.assertEquals(items["03"]["modified"], ts2) + self.assertEquals(items["04"]["modified"], ts1) + self.assertEquals(items["05"]["modified"], ts2) + self.assertEquals(items["06"]["modified"], ts2) + + def test_that_negative_integer_fields_are_not_accepted(self): + # ttls cannot be negative + self.retry_put_json(self.root + "/storage/xxx_col2/TEST", { + "payload": "TEST", + "ttl": -1, + }, status=400) + # limit cannot be negative + self.retry_put_json(self.root + "/storage/xxx_col2/TEST", + {"payload": "X"}) + self.app.get(self.root + "/storage/xxx_col2?limit=-1", status=400) + # X-If-Modified-Since cannot be negative + self.app.get(self.root + "/storage/xxx_col2", headers={ + "X-If-Modified-Since": "-3", + }, status=400) + # X-If-Unmodified-Since cannot be negative + self.retry_put_json(self.root + "/storage/xxx_col2/TEST", { + "payload": "TEST", + }, headers={ + "X-If-Unmodified-Since": "-3", + }, status=400) + # sortindex actually *can* be negative + self.retry_put_json(self.root + "/storage/xxx_col2/TEST", { + "payload": "TEST", + "sortindex": -42, + }, status=200) + + def test_meta_global_sanity(self): + # Memcache backend is configured to store 'meta' in write-through + # cache, so we want to check it explicitly. We might as well put it + # in the base tests because there's nothing memcached-specific here. + self.app.get(self.root + '/storage/meta/global', status=404) + res = self.app.get(self.root + '/storage/meta') + self.assertEquals(res.json, []) + self.retry_put_json(self.root + '/storage/meta/global', + {'payload': 'blob'}) + res = self.app.get(self.root + '/storage/meta') + self.assertEquals(res.json, ['global']) + res = self.app.get(self.root + '/storage/meta/global') + self.assertEquals(res.json['payload'], 'blob') + # It should not have extra keys. + keys = list(res.json.keys()) + keys.sort() + self.assertEquals(keys, ['id', 'modified', 'payload']) + # It should have a properly-formatted "modified" field. + modified_re = r"['\"]modified['\"]:\s*[0-9]+\.[0-9][0-9]\s*[,}]" + self.assertTrue(re.search(modified_re, res.body.decode('utf-8'))) + # Any client-specified "modified" field should be ignored + res = self.retry_put_json(self.root + '/storage/meta/global', + {'payload': 'blob', 'modified': 12}) + ts = float(res.headers['X-Weave-Timestamp']) + res = self.app.get(self.root + '/storage/meta/global') + self.assertEquals(res.json['modified'], ts) + + def test_that_404_responses_have_a_json_body(self): + res = self.app.get(self.root + '/nonexistent/url', status=404) + self.assertEquals(res.content_type, "application/json") + self.assertEquals(res.json, 0) + + def test_that_internal_server_fields_are_not_echoed(self): + self.retry_post_json(self.root + '/storage/xxx_col1', + [{'id': 'one', 'payload': 'blob'}]) + self.retry_put_json(self.root + '/storage/xxx_col1/two', + {'payload': 'blub'}) + res = self.app.get(self.root + '/storage/xxx_col1?full=1') + self.assertEquals(len(res.json), 2) + for item in res.json: + self.assertTrue("id" in item) + self.assertTrue("payload" in item) + self.assertFalse("payload_size" in item) + self.assertFalse("ttl" in item) + for id in ('one', 'two'): + res = self.app.get(self.root + '/storage/xxx_col1/' + id) + self.assertTrue("id" in res.json) + self.assertTrue("payload" in res.json) + self.assertFalse("payload_size" in res.json) + self.assertFalse("ttl" in res.json) + + def test_accessing_info_collections_with_an_expired_token(self): + # This can't be run against a live server because we + # have to forge an auth token to test things properly. + if self.distant: + raise unittest2.SkipTest + + # Write some items while we've got a good token. + bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)] + resp = self.retry_post_json(self.root + "/storage/xxx_col1", bsos) + ts = float(resp.headers["X-Last-Modified"]) + + # Check that we can read the info correctly. + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(list(resp.json.keys()), ["xxx_col1"]) + self.assertEquals(resp.json["xxx_col1"], ts) + + # Forge an expired token to use for the test. + auth_policy = self.config.registry.getUtility(IAuthenticationPolicy) + secret = auth_policy._get_token_secrets(self.host_url)[-1] + tm = tokenlib.TokenManager(secret=secret) + exp = time.time() - 60 + data = {"uid": self.user_id, "node": self.host_url, "expires": exp} + self.auth_token = tm.make_token(data) + self.auth_secret = tm.get_derived_secret(self.auth_token) + + # The expired token cannot be used for normal operations. + bsos = [{"id": str(i).zfill(2), "payload": "aaa"} for i in range(3)] + self.retry_post_json(self.root + "/storage/xxx_col1", bsos, status=401) + self.app.get(self.root + "/storage/xxx_col1", status=401) + + # But it still allows access to /info/collections. + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(list(resp.json.keys()), ["xxx_col1"]) + self.assertEquals(resp.json["xxx_col1"], ts) + + def test_pagination_with_newer_and_sort_by_oldest(self): + # Twelve bsos with three different modification times. + NUM_ITEMS = 12 + bsos = [] + timestamps = [] + for i in range(NUM_ITEMS): + bso = {'id': str(i).zfill(2), 'payload': 'x'} + bsos.append(bso) + if i % 4 == 3: + res = self.retry_post_json(self.root + '/storage/xxx_col2', + bsos) + ts = float(res.headers["X-Last-Modified"]) + timestamps.append((i, ts)) + bsos = [] + + # Try with several different pagination sizes, + # to hit various boundary conditions. + for limit in (2, 3, 4, 5, 6): + for (start, ts) in timestamps: + query_url = self.root + \ + '/storage/xxx_col2?full=true&sort=oldest' + query_url += '&newer=%s&limit=%s' % (ts, limit) + + # Paginated-ly fetch all items. + items = [] + res = self.app.get(query_url) + for item in res.json: + if items: + assert items[-1]['modified'] <= item['modified'] + items.append(item) + next_offset = res.headers.get('X-Weave-Next-Offset') + while next_offset is not None: + res = self.app.get(query_url + "&offset=" + next_offset) + for item in res.json: + assert items[-1]['modified'] <= item['modified'] + items.append(item) + next_offset = res.headers.get('X-Weave-Next-Offset') + + # They should all be in order, starting from the item + # *after* the one that was used for the newer= timestamp. + self.assertEquals(sorted(int(item['id']) for item in items), + list(range(start + 1, NUM_ITEMS))) + + def test_pagination_with_older_and_sort_by_newest(self): + # Twelve bsos with three different modification times. + NUM_ITEMS = 12 + bsos = [] + timestamps = [] + for i in range(NUM_ITEMS): + bso = {'id': str(i).zfill(2), 'payload': 'x'} + bsos.append(bso) + if i % 4 == 3: + res = self.retry_post_json(self.root + '/storage/xxx_col2', + bsos) + ts = float(res.headers["X-Last-Modified"]) + timestamps.append((i - 3, ts)) + bsos = [] + + # Try with several different pagination sizes, + # to hit various boundary conditions. + for limit in (2, 3, 4, 5, 6): + for (start, ts) in timestamps: + query_url = self.root + \ + '/storage/xxx_col2?full=true&sort=newest' + query_url += '&older=%s&limit=%s' % (ts, limit) + + # Paginated-ly fetch all items. + items = [] + res = self.app.get(query_url) + for item in res.json: + if items: + assert items[-1]['modified'] >= item['modified'] + items.append(item) + next_offset = res.headers.get('X-Weave-Next-Offset') + while next_offset is not None: + res = self.app.get(query_url + "&offset=" + next_offset) + for item in res.json: + assert items[-1]['modified'] >= item['modified'] + items.append(item) + next_offset = res.headers.get('X-Weave-Next-Offset') + + # They should all be in order, up to the item *before* + # the one that was used for the older= timestamp. + self.assertEquals(sorted(int(item['id']) for item in items), + list(range(0, start))) + + def assertCloseEnough(self, val1, val2, delta=0.05): + if abs(val1 - val2) < delta: + return True + raise AssertionError("abs(%.2f - %.2f) = %.2f > %.2f" + % (val1, val2, abs(val1 - val2), delta)) + + def test_batches(self): + + endpoint = self.root + '/storage/xxx_col2' + + bso1 = {'id': '12', 'payload': 'elegance'} + bso2 = {'id': '13', 'payload': 'slovenly'} + bsos = [bso1, bso2] + self.retry_post_json(endpoint, bsos) + + resp = self.app.get(endpoint + '/12') + orig_modified = resp.headers['X-Last-Modified'] + + bso3 = {'id': 'a', 'payload': 'internal'} + bso4 = {'id': 'b', 'payload': 'pancreas'} + resp = self.retry_post_json(endpoint + '?batch=true', [bso3, bso4]) + batch = resp.json["batch"] + + # The collection should not be reported as modified. + self.assertEquals(orig_modified, resp.headers['X-Last-Modified']) + + # And reading from it shouldn't show the new records yet. + resp = self.app.get(endpoint) + res = resp.json + res.sort() + self.assertEquals(res, ['12', '13']) + self.assertEquals(int(resp.headers['X-Weave-Records']), 2) + self.assertEquals(orig_modified, resp.headers['X-Last-Modified']) + + bso5 = {'id': 'c', 'payload': 'tinsel'} + bso6 = {'id': '13', 'payload': 'portnoy'} + bso0 = {'id': '14', 'payload': 'itsybitsy'} + commit = '?batch={0}&commit=true'.format(batch) + resp = self.retry_post_json(endpoint + commit, [bso5, bso6, bso0]) + committed = resp.json['modified'] + self.assertEquals(resp.json['modified'], + float(resp.headers['X-Last-Modified'])) + + # make sure /info/collections got updated + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(float(resp.headers['X-Last-Modified']), committed) + self.assertEquals(resp.json['xxx_col2'], committed) + + # make sure the changes applied + resp = self.app.get(endpoint) + res = resp.json + res.sort() + self.assertEquals(res, ['12', '13', '14', 'a', 'b', 'c']) + self.assertEquals(int(resp.headers['X-Weave-Records']), 6) + resp = self.app.get(endpoint + '/13') + self.assertEquals(resp.json['payload'], 'portnoy') + self.assertEquals(committed, float(resp.headers['X-Last-Modified'])) + self.assertEquals(committed, resp.json['modified']) + resp = self.app.get(endpoint + '/c') + self.assertEquals(resp.json['payload'], 'tinsel') + self.assertEquals(committed, resp.json['modified']) + resp = self.app.get(endpoint + '/14') + self.assertEquals(resp.json['payload'], 'itsybitsy') + self.assertEquals(committed, resp.json['modified']) + + # empty commit POST + bso7 = {'id': 'a', 'payload': 'burrito'} + bso8 = {'id': 'e', 'payload': 'chocolate'} + resp = self.retry_post_json(endpoint + '?batch=true', [bso7, bso8]) + batch = resp.json["batch"] + time.sleep(1) + commit = '?batch={0}&commit=true'.format(batch) + + resp1 = self.retry_post_json(endpoint + commit, []) + committed = resp1.json['modified'] + self.assertEquals(committed, float(resp1.headers['X-Last-Modified'])) + + resp2 = self.app.get(endpoint + '/a') + self.assertEquals(committed, float(resp2.headers['X-Last-Modified'])) + self.assertEquals(committed, resp2.json['modified']) + self.assertEquals(resp2.json['payload'], 'burrito') + + resp3 = self.app.get(endpoint + '/e') + self.assertEquals(committed, resp3.json['modified']) + + def test_we_dont_need_no_stinkin_batches(self): + endpoint = self.root + '/storage/xxx_col2' + + # invalid batch ID + bso1 = {'id': 'f', 'payload': 'pantomime'} + self.retry_post_json(endpoint + '?batch=sammich', [bso1], status=400) + + # commit with no batch ID + self.retry_post_json(endpoint + '?commit=true', [], status=400) + + def test_batch_size_limits(self): + limits = self.app.get(self.root + '/info/configuration').json + self.assertTrue('max_post_records' in limits) + self.assertTrue('max_post_bytes' in limits) + self.assertTrue('max_total_records' in limits) + self.assertTrue('max_total_bytes' in limits) + self.assertTrue('max_record_payload_bytes' in limits) + self.assertTrue('max_request_bytes' in limits) + + endpoint = self.root + '/storage/xxx_col2?batch=true' + # There are certain obvious constraints on these limits, + # violations of which would be very confusing for clients. +# +# self.assertTrue( +# limits['max_request_bytes'] > limits['max_post_bytes'] +# ) +# self.assertTrue( +# limits['max_post_bytes'] >= limits['max_record_payload_bytes'] +# ) +# self.assertTrue( +# limits['max_total_records'] >= limits['max_post_records'] +# ) +# self.assertTrue( +# limits['max_total_bytes'] >= limits['max_post_bytes'] +# ) +# +# # `max_post_records` is an (inclusive) limit on +# # the number of items in a single post. +# +# res = self.retry_post_json(endpoint, [], headers={ +# 'X-Weave-Records': str(limits['max_post_records']) +# }) +# self.assertFalse(res.json['failed']) +# res = self.retry_post_json(endpoint, [], headers={ +# 'X-Weave-Records': str(limits['max_post_records'] + 1) +# }, status=400) +# self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED) +# +# bsos = [{'id': str(x), 'payload': ''} +# for x in range(limits['max_post_records'])] +# res = self.retry_post_json(endpoint, bsos) +# self.assertFalse(res.json['failed']) +# bsos.append({'id': 'toomany', 'payload': ''}) +# res = self.retry_post_json(endpoint, bsos) +# self.assertEquals(res.json['failed']['toomany'], 'retry bso') +# +# # `max_total_records` is an (inclusive) limit on the +# # total number of items in a batch. We can only enforce +# # it if the client tells us this via header. +# +# self.retry_post_json(endpoint, [], headers={ +# 'X-Weave-Total-Records': str(limits['max_total_records']) +# }) +# res = self.retry_post_json(endpoint, [], headers={ +# 'X-Weave-Total-Records': str(limits['max_total_records'] + 1) +# }, status=400) +# self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED) +# +# # `max_post_bytes` is an (inclusive) limit on the +# # total size of payloads in a single post. +# +# self.retry_post_json(endpoint, [], headers={ +# 'X-Weave-Bytes': str(limits['max_post_bytes']) +# }) +# res = self.retry_post_json(endpoint, [], headers={ +# 'X-Weave-Bytes': str(limits['max_post_bytes'] + 1) +# }, status=400) +# self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED) + bsos = [ + {'id': 'little', 'payload': 'XXX'}, + {'id': 'big', 'payload': 'X' * (limits['max_post_bytes'] - 3)} + ] + res = self.retry_post_json(endpoint, bsos) + self.assertFalse(res.json['failed']) + bsos[1]['payload'] += 'X' + res = self.retry_post_json(endpoint, bsos) + self.assertEqual(res.json['success'], ['little']) + self.assertEqual(res.json['failed']['big'], 'retry bytes') + + # `max_total_bytes` is an (inclusive) limit on the + # total size of all payloads in a batch. We can only enforce + # it if the client tells us this via header. + + self.retry_post_json(endpoint, [], headers={ + 'X-Weave-Total-Bytes': str(limits['max_total_bytes']) + }) + res = self.retry_post_json(endpoint, [], headers={ + 'X-Weave-Total-Bytes': str(limits['max_total_bytes'] + 1) + }, status=400) + self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED) + + def test_batch_partial_update(self): + collection = self.root + '/storage/xxx_col2' + bsos = [ + {'id': 'a', 'payload': 'aai'}, + {'id': 'b', 'payload': 'bee', 'sortindex': 17} + ] + resp = self.retry_post_json(collection, bsos) + orig_ts = float(resp.headers['X-Last-Modified']) + + # Update one, and add a new one. + bsos = [ + {'id': 'b', 'payload': 'bii'}, + {'id': 'c', 'payload': 'sea'}, + ] + resp = self.retry_post_json(collection + '?batch=true', bsos) + batch = resp.json["batch"] + self.assertEquals(orig_ts, float(resp.headers['X-Last-Modified'])) + + # The updated item hasn't been written yet. + resp = self.app.get(collection + '?full=1') + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 2) + self.assertEquals(res[0]['payload'], 'aai') + self.assertEquals(res[1]['payload'], 'bee') + self.assertEquals(res[0]['modified'], orig_ts) + self.assertEquals(res[1]['modified'], orig_ts) + self.assertEquals(res[1]['sortindex'], 17) + + endpoint = collection + '?batch={0}&commit=true'.format(batch) + resp = self.retry_post_json(endpoint, []) + commit_ts = float(resp.headers['X-Last-Modified']) + + # The changes have now been applied. + resp = self.app.get(collection + '?full=1') + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 3) + self.assertEquals(res[0]['payload'], 'aai') + self.assertEquals(res[1]['payload'], 'bii') + self.assertEquals(res[2]['payload'], 'sea') + self.assertEquals(res[0]['modified'], orig_ts) + self.assertEquals(res[1]['modified'], commit_ts) + self.assertEquals(res[2]['modified'], commit_ts) + + # Fields not touched by the batch, should have been preserved. + self.assertEquals(res[1]['sortindex'], 17) + + def test_batch_ttl_update(self): + collection = self.root + '/storage/xxx_col2' + bsos = [ + {'id': 'a', 'payload': 'ayy'}, + {'id': 'b', 'payload': 'bea'}, + {'id': 'c', 'payload': 'see'} + ] + resp = self.retry_post_json(collection, bsos) + + # Bump ttls as a series of individual batch operations. + resp = self.retry_post_json(collection + '?batch=true', [], + status=202) + orig_ts = float(resp.headers['X-Last-Modified']) + batch = resp.json["batch"] + + endpoint = collection + '?batch={0}'.format(batch) + resp = self.retry_post_json(endpoint, [{'id': 'a', 'ttl': 2}], + status=202) + self.assertEquals(orig_ts, float(resp.headers['X-Last-Modified'])) + resp = self.retry_post_json(endpoint, [{'id': 'b', 'ttl': 2}], + status=202) + self.assertEquals(orig_ts, float(resp.headers['X-Last-Modified'])) + resp = self.retry_post_json(endpoint + '&commit=true', [], status=200) + + # The payloads should be unchanged + resp = self.app.get(collection + '?full=1') + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 3) + self.assertEquals(res[0]['payload'], 'ayy') + self.assertEquals(res[1]['payload'], 'bea') + self.assertEquals(res[2]['payload'], 'see') + + # If we wait, the ttls should kick in + time.sleep(2.1) + resp = self.app.get(collection + '?full=1') + res = resp.json + self.assertEquals(len(res), 1) + self.assertEquals(res[0]['payload'], 'see') + + def test_batch_ttl_is_based_on_commit_timestamp(self): + collection = self.root + '/storage/xxx_col2' + + resp = self.retry_post_json(collection + '?batch=true', [], status=202) + batch = resp.json["batch"] + endpoint = collection + '?batch={0}'.format(batch) + resp = self.retry_post_json(endpoint, [{'id': 'a', 'ttl': 3}], + status=202) + + # Put some time between upload timestamp and commit timestamp. + time.sleep(1.5) + + resp = self.retry_post_json(endpoint + '&commit=true', [], + status=200) + + # Wait a little; if ttl is taken from the time of the commit + # then it should not kick in just yet. + time.sleep(1.6) + resp = self.app.get(collection) + res = resp.json + self.assertEquals(len(res), 1) + self.assertEquals(res[0], 'a') + + # Wait some more, and the ttl should kick in. + time.sleep(1.6) + resp = self.app.get(collection) + res = resp.json + self.assertEquals(len(res), 0) + + def test_batch_with_immediate_commit(self): + collection = self.root + '/storage/xxx_col2' + bsos = [ + {'id': 'a', 'payload': 'aih'}, + {'id': 'b', 'payload': 'bie'}, + {'id': 'c', 'payload': 'cee'} + ] + + resp = self.retry_post_json(collection + '?batch=true&commit=true', + bsos, status=200) + self.assertTrue('batch' not in resp.json) + self.assertTrue('modified' in resp.json) + committed = resp.json['modified'] + + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(float(resp.headers['X-Last-Modified']), committed) + self.assertEquals(resp.json['xxx_col2'], committed) + + resp = self.app.get(collection + '?full=1') + self.assertEquals(float(resp.headers['X-Last-Modified']), committed) + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 3) + self.assertEquals(res[0]['payload'], 'aih') + self.assertEquals(res[1]['payload'], 'bie') + self.assertEquals(res[2]['payload'], 'cee') + + def test_batch_uploads_properly_update_info_collections(self): + collection1 = self.root + '/storage/xxx_col1' + collection2 = self.root + '/storage/xxx_col2' + bsos = [ + {'id': 'a', 'payload': 'aih'}, + {'id': 'b', 'payload': 'bie'}, + {'id': 'c', 'payload': 'cee'} + ] + + resp = self.retry_post_json(collection1, bsos) + ts1 = resp.json['modified'] + + resp = self.retry_post_json(collection2, bsos) + ts2 = resp.json['modified'] + + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(float(resp.headers['X-Last-Modified']), ts2) + self.assertEquals(resp.json['xxx_col1'], ts1) + self.assertEquals(resp.json['xxx_col2'], ts2) + + # Overwrite in place, timestamp should change. + resp = self.retry_post_json(collection2 + '?batch=true&commit=true', + bsos[:2]) + self.assertTrue(resp.json['modified'] > ts2) + ts2 = resp.json['modified'] + + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(float(resp.headers['X-Last-Modified']), ts2) + self.assertEquals(resp.json['xxx_col1'], ts1) + self.assertEquals(resp.json['xxx_col2'], ts2) + + # Add new items, timestamp should change + resp = self.retry_post_json(collection1 + '?batch=true&commit=true', + [{'id': 'd', 'payload': 'dee'}]) + self.assertTrue(resp.json['modified'] > ts1) + self.assertTrue(resp.json['modified'] >= ts2) + ts1 = resp.json['modified'] + + resp = self.app.get(self.root + '/info/collections') + self.assertEquals(float(resp.headers['X-Last-Modified']), ts1) + self.assertEquals(resp.json['xxx_col1'], ts1) + self.assertEquals(resp.json['xxx_col2'], ts2) + + def test_batch_with_failing_bsos(self): + collection = self.root + '/storage/xxx_col2' + bsos = [ + {'id': 'a', 'payload': 'aai'}, + {'id': 'b\n', 'payload': 'i am invalid', 'sortindex': 17} + ] + resp = self.retry_post_json(collection + '?batch=true', bsos) + self.assertEqual(len(resp.json['failed']), 1) + self.assertEqual(len(resp.json['success']), 1) + batch = resp.json["batch"] + + bsos = [ + {'id': 'c', 'payload': 'sea'}, + {'id': 'd', 'payload': 'dii', 'ttl': -12}, + ] + endpoint = collection + '?batch={0}&commit=true'.format(batch) + resp = self.retry_post_json(endpoint, bsos) + self.assertEqual(len(resp.json['failed']), 1) + self.assertEqual(len(resp.json['success']), 1) + + # To correctly match semantics of batchless POST, the batch + # should be committed including only the successful items. + # It is the client's responsibility to detect that some items + # failed, and decide whether to commit the batch. + resp = self.app.get(collection + '?full=1') + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 2) + self.assertEquals(res[0]['payload'], 'aai') + self.assertEquals(res[1]['payload'], 'sea') + + def test_batch_id_is_correctly_scoped_to_a_collection(self): + collection1 = self.root + '/storage/xxx_col1' + bsos = [ + {'id': 'a', 'payload': 'aih'}, + {'id': 'b', 'payload': 'bie'}, + {'id': 'c', 'payload': 'cee'} + ] + resp = self.retry_post_json(collection1 + '?batch=true', bsos) + batch = resp.json['batch'] + + # I should not be able to add to that batch in a different collection. + endpoint2 = self.root + '/storage/xxx_col2?batch={0}'.format(batch) + resp = self.retry_post_json( + endpoint2, + [{'id': 'd', 'payload': 'dii'}], + status=400) + + # I should not be able to commit that batch in a different collection. + resp = self.retry_post_json(endpoint2 + '&commit=true', [], status=400) + + # I should still be able to use the batch in the correct collection. + endpoint1 = collection1 + '?batch={0}'.format(batch) + resp = self.retry_post_json(endpoint1, + [{'id': 'd', 'payload': 'dii'}]) + resp = self.retry_post_json(endpoint1 + '&commit=true', []) + + resp = self.app.get(collection1 + '?full=1') + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 4) + self.assertEquals(res[0]['payload'], 'aih') + self.assertEquals(res[1]['payload'], 'bie') + self.assertEquals(res[2]['payload'], 'cee') + self.assertEquals(res[3]['payload'], 'dii') + + def test_users_with_the_same_batch_id_get_separate_data(self): + # Try to generate two users with the same batch-id. + # It might take a couple of attempts... + for _ in range(100): + bsos = [{'id': 'a', 'payload': 'aih'}] + req = '/storage/xxx_col1?batch=true' + resp = self.retry_post_json(self.root + req, bsos) + batch1 = resp.json['batch'] + with self._switch_user(): + bsos = [{'id': 'b', 'payload': 'bee'}] + req = '/storage/xxx_col1?batch=true' + resp = self.retry_post_json(self.root + req, bsos) + batch2 = resp.json['batch'] + # Let the second user commit their batch. + req = '/storage/xxx_col1?batch={0}&commit=true'.format(batch2) + self.retry_post_json(self.root + req, []) + # It should only have a single item. + resp = self.app.get(self.root + '/storage/xxx_col1') + self.assertEquals(resp.json, ['b']) + # The first user's collection should still be empty. + # Now have the first user commit their batch. + req = '/storage/xxx_col1?batch={0}&commit=true'.format(batch1) + self.retry_post_json(self.root + req, []) + # It should only have a single item. + resp = self.app.get(self.root + '/storage/xxx_col1') + self.assertEquals(resp.json, ['a']) + # If we didn't make a conflict, try again. + if batch1 == batch2: + break + else: + raise unittest2.SkipTest('failed to generate conflicting batchid') + + def test_that_we_dont_resurrect_committed_batches(self): + # This retry loop tries to trigger a situation where we: + # * create a batch with a single item + # * successfully commit that batch + # * create a new batch tht re-uses the same batchid + for _ in range(100): + bsos = [{'id': 'i', 'payload': 'aye'}] + req = '/storage/xxx_col1?batch=true' + resp = self.retry_post_json(self.root + req, bsos) + batch1 = resp.json['batch'] + req = '/storage/xxx_col1?batch={0}&commit=true'.format(batch1) + self.retry_post_json(self.root + req, []) + req = '/storage/xxx_col2?batch=true' + resp = self.retry_post_json(self.root + req, []) + batch2 = resp.json['batch'] + bsos = [{'id': 'j', 'payload': 'jay'}] + req = '/storage/xxx_col2?batch={0}&commit=true'.format(batch2) + self.retry_post_json(self.root + req, bsos) + # Retry if we failed to trigger re-use of the batchid. + if batch1 == batch2: + break + else: + raise unittest2.SkipTest('failed to trigger re-use of batchid') + # Despite having the same batchid, the second batch should + # be completely independent of the first. + resp = self.app.get(self.root + '/storage/xxx_col2') + self.assertEquals(resp.json, ['j']) + + def test_batch_id_is_correctly_scoped_to_a_user(self): + collection = self.root + '/storage/xxx_col1' + bsos = [ + {'id': 'a', 'payload': 'aih'}, + {'id': 'b', 'payload': 'bie'}, + {'id': 'c', 'payload': 'cee'} + ] + resp = self.retry_post_json(collection + '?batch=true', bsos) + batch = resp.json['batch'] + + with self._switch_user(): + # I should not be able to add to that batch as a different user. + endpoint = self.root + '/storage/xxx_col1?batch={0}'.format(batch) + resp = self.retry_post_json( + endpoint, + [{'id': 'd', 'payload': 'di'}], + status=400) + + # I should not be able to commit that batch as a different user. + resp = self.retry_post_json(endpoint + '&commit=true', [], + status=400) + + # I should still be able to use the batch in the original user. + endpoint = collection + '?batch={0}'.format(batch) + resp = self.retry_post_json(endpoint, [{'id': 'd', 'payload': 'di'}]) + resp = self.retry_post_json(endpoint + '&commit=true', []) + + resp = self.app.get(collection + '?full=1') + res = resp.json + res.sort(key=lambda bso: bso['id']) + self.assertEquals(len(res), 4) + self.assertEquals(res[0]['payload'], 'aih') + self.assertEquals(res[1]['payload'], 'bie') + self.assertEquals(res[2]['payload'], 'cee') + self.assertEquals(res[3]['payload'], 'di') + + # bug 1332552 make sure ttl:null use the default ttl + def test_create_bso_with_null_ttl(self): + bso = {"payload": "x", "ttl": None} + self.retry_put_json(self.root + "/storage/xxx_col2/TEST1", bso) + time.sleep(0.1) + res = self.app.get(self.root + "/storage/xxx_col2/TEST1?full=1") + self.assertEquals(res.json["payload"], "x") + + def test_rejection_of_known_bad_payloads(self): + bso = { + "id": "keys", + "payload": json_dumps({ + "ciphertext": "IDontKnowWhatImDoing", + "IV": "AAAAAAAAAAAAAAAAAAAAAA==", + }) + } + # Fishy IVs are rejected on the "crypto" collection. + self.retry_put_json(self.root + "/storage/crypto/keys", bso, + status=400) + self.retry_put_json(self.root + "/storage/crypto/blerg", bso, + status=400) + self.retry_post_json(self.root + "/storage/crypto", [bso], status=400) + # But are allowed on other collections. + self.retry_put_json(self.root + "/storage/xxx_col2/keys", bso, + status=200) + self.retry_post_json(self.root + "/storage/xxx_col2", [bso], + status=200) + + # bug 1397357 + def test_batch_empty_commit(self): + def testEmptyCommit(contentType, body, status=200): + bsos = [{'id': str(i).zfill(2), 'payload': 'X'} for i in range(5)] + res = self.retry_post_json(self.root+'/storage/xxx_col?batch=true', + bsos) + self.assertEquals(len(res.json['success']), 5) + self.assertEquals(len(res.json['failed']), 0) + batch = res.json["batch"] + self.app.post( + self.root+'/storage/xxx_col?commit=true&batch='+batch, + body, headers={"Content-Type": contentType}, + status=status + ) + + testEmptyCommit("application/json", "[]") + testEmptyCommit("application/json", "{}", status=400) + testEmptyCommit("application/json", "", status=400) + + testEmptyCommit("application/newlines", "") + testEmptyCommit("application/newlines", "\n", status=400) + testEmptyCommit("application/newlines", "{}", status=400) + testEmptyCommit("application/newlines", "[]", status=400) diff --git a/tools/integration_tests/test_support.py b/tools/integration_tests/test_support.py new file mode 100644 index 00000000..4e1ad5b7 --- /dev/null +++ b/tools/integration_tests/test_support.py @@ -0,0 +1,849 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +""" Base test class, with an instanciated app. +""" + +import contextlib +import functools +from konfig import Config, SettingsDict +import hawkauthlib +import os +import optparse +from pyramid.authorization import ACLAuthorizationPolicy +from pyramid.config import Configurator +from pyramid.interfaces import IAuthenticationPolicy +from pyramid.request import Request +from pyramid.util import DottedNameResolver +from pyramid_hawkauth import HawkAuthenticationPolicy +import random +import re +import csv +import binascii +from collections import defaultdict +import sys +import time +import tokenlib +import urllib.parse as urlparse +import unittest2 +import uuid +from webtest import TestApp +from zope.interface import implementer + + +global_secret = None +VALID_FXA_ID_REGEX = re.compile("^[A-Za-z0-9=\\-_]{1,64}$") + + +class Secrets(object): + """Load node-specific secrets from a file. + This class provides a method to get a list of secrets for a node + ordered by timestamps. The secrets are stored in a CSV file which + is loaded when the object is created. + Options: + - **filename**: a list of file paths, or a single path. + """ + def __init__(self, filename=None): + self._secrets = defaultdict(list) + if filename is not None: + self.load(filename) + + def keys(self): + return self._secrets.keys() + + def load(self, filename): + if not isinstance(filename, (list, tuple)): + filename = [filename] + + for name in filename: + with open(name, 'rb') as f: + + reader = csv.reader(f, delimiter=',') + for line, row in enumerate(reader): + if len(row) < 2: + continue + node = row[0] + if node in self._secrets: + raise ValueError("Duplicate node line %d" % line) + secrets = [] + for secret in row[1:]: + secret = secret.split(':') + if len(secret) != 2: + raise ValueError("Invalid secret line %d" % line) + secrets.append(tuple(secret)) + secrets.sort() + self._secrets[node] = secrets + + def save(self, filename): + with open(filename, 'wb') as f: + writer = csv.writer(f, delimiter=',') + for node, secrets in self._secrets.items(): + secrets = ['%s:%s' % (timestamp, secret) + for timestamp, secret in secrets] + secrets.insert(0, node) + writer.writerow(secrets) + + def get(self, node): + return [secret for timestamp, secret in self._secrets[node]] + + def add(self, node, size=256): + timestamp = str(int(time.time())) + secret = binascii.b2a_hex(os.urandom(size))[:size] + # The new secret *must* sort at the end of the list. + # This forbids you from adding multiple secrets per second. + try: + if timestamp <= self._secrets[node][-1][0]: + assert False, "You can only add one secret per second" + except IndexError: + pass + self._secrets[node].append((timestamp, secret)) + + +class FixedSecrets(object): + """Use a fixed set of secrets for all nodes. + This class provides the same API as the Secrets class, but uses a + single list of secrets for all nodes rather than using different + secrets for each node. + Options: + - **secrets**: a list of hex-encoded secrets to use for all nodes. + """ + def __init__(self, secrets): + if isinstance(secrets, str): + secrets = secrets.split() + self._secrets = secrets + + def get(self, node): + return list(self._secrets) + + def keys(self): + return [] + + +def resolve_name(name, package=None): + """Resolve dotted name into a python object. + This function resolves a dotted name as a reference to a python object, + returning whatever object happens to live at that path. It's a simple + convenience wrapper around pyramid's DottedNameResolver. + The optional argument 'package' specifies the package name for relative + imports. If not specified, only absolute paths will be supported. + """ + return DottedNameResolver(package).resolve(name) + + +def load_into_settings(filename, settings): + """Load config file contents into a Pyramid settings dict. + This is a helper function for initialising a Pyramid settings dict from + a config file. It flattens the config file sections into dotted settings + names and updates the given dictionary in place. + You would typically use this when constructing a Pyramid Configurator + object, like so:: + def main(global_config, **settings): + config_file = global_config['__file__'] + load_info_settings(config_file, settings) + config = Configurator(settings=settings) + """ + filename = os.path.expandvars(os.path.expanduser(filename)) + filename = os.path.abspath(os.path.normpath(filename)) + config = Config(filename) + + # Konfig keywords are added to every section when present, we have to + # filter them out, otherwise plugin.load_from_config and + # plugin.load_from_settings are unable to create instances. + konfig_keywords = ['extends', 'overrides'] + + # Put values from the config file into the pyramid settings dict. + for section in config.sections(): + setting_prefix = section.replace(":", ".") + for name, value in config.get_map(section).items(): + if name not in konfig_keywords: + settings[setting_prefix + "." + name] = value + + # Store a reference to the Config object itself for later retrieval. + settings['config'] = config + return config + + +def get_test_configurator(root, ini_file="tests.ini"): + """Find a file with testing settings, turn it into a configurator.""" + ini_dir = root + while True: + ini_path = os.path.join(ini_dir, ini_file) + if os.path.exists(ini_path): + break + if ini_path == ini_file or ini_path == "/" + ini_file: + raise RuntimeError("cannot locate " + ini_file) + ini_dir = os.path.split(ini_dir)[0] + # print("finding configurator for", ini_path) + config = get_configurator({"__file__": ini_path}) + authz_policy = ACLAuthorizationPolicy() + config.set_authorization_policy(authz_policy) + authn_policy = TokenServerAuthenticationPolicy.from_settings( + config.get_settings()) + config.set_authentication_policy(authn_policy) + return config + + +def get_configurator(global_config, **settings): + """Create a pyramid Configurator and populate it with sensible defaults. + This function is a helper to create and pre-populate a Configurator + object using the given paste-deploy settings dicts. It uses the + mozsvc.config module to flatten the config paste-deploy config file + into the settings dict so that non-mozsvc pyramid apps can read values + from it easily. + """ + # Populate a SettingsDict with settings from the deployment file. + settings = SettingsDict(settings) + config_file = global_config.get('__file__') + if config_file is not None: + load_into_settings(config_file, settings) + # Update with default pyramid settings, and then insert for all to use. + config = Configurator(settings={}) + settings.setdefaults(config.registry.settings) + config.registry.settings = settings + return config + + +def restore_env(*keys): + """Decorator that ensures os.environ gets restored after a test. + + Given a list of environment variable keys, this decorator will save the + current values of those environment variables at the start of the call + and restore them to those values at the end. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwds): + values = [os.environ.get(key) for key in keys] + try: + return func(*args, **kwds) + finally: + for key, value in zip(keys, values): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + return wrapper + return decorator + + +class TestCase(unittest2.TestCase): + """TestCase with some generic helper methods.""" + + def setUp(self): + super(TestCase, self).setUp() + self.config = self.get_configurator() + + def tearDown(self): + self.config.end() + super(TestCase, self).tearDown() + + def get_configurator(self): + """Load the configurator to use for the tests.""" + # Load config from the .ini file. + # print("get_configurator", self, getattr(self, "TEST_INI_FILE", None)) + if not hasattr(self, "ini_file"): + if hasattr(self, "TEST_INI_FILE"): + self.ini_file = self.TEST_INI_FILE + else: + # The file to use may be specified in the environment. + self.ini_file = os.environ.get("MOZSVC_TEST_INI_FILE", + "tests.ini") + __file__ = sys.modules[self.__class__.__module__].__file__ + config = get_test_configurator(__file__, self.ini_file) + config.begin() + return config + + """ + def make_request(self, *args, **kwds): + config = kwds.pop("config", self.config) + return make_request(config, *args, **kwds) + """ + + +class StorageTestCase(TestCase): + """TestCase class with automatic cleanup of database files.""" + + @restore_env("MOZSVC_TEST_INI_FILE") + def setUp(self): + # Put a fresh UUID into the environment. + # This can be used in e.g. config files to create unique paths. + os.environ["MOZSVC_UUID"] = str(uuid.uuid4()) + # Ensure a default sqluri if none is provided in the environment. + # We use an in-memory sqlite db by default, except for tests that + # explicitly require an on-disk file. + if "MOZSVC_SQLURI" not in os.environ: + os.environ["MOZSVC_SQLURI"] = "sqlite:///:memory:" + if "MOZSVC_ONDISK_SQLURI" not in os.environ: + ondisk_sqluri = os.environ["MOZSVC_SQLURI"] + if ":memory:" in ondisk_sqluri: + ondisk_sqluri = "sqlite:////tmp/tests-sync-%s.db" + ondisk_sqluri %= (os.environ["MOZSVC_UUID"],) + os.environ["MOZSVC_ONDISK_SQLURI"] = ondisk_sqluri + # Allow subclasses to override default ini file. + if hasattr(self, "TEST_INI_FILE"): + if "MOZSVC_TEST_INI_FILE" not in os.environ: + os.environ["MOZSVC_TEST_INI_FILE"] = self.TEST_INI_FILE + super(StorageTestCase, self).setUp() + + def tearDown(self): + self._cleanup_test_databases() + # clear the pyramid threadlocals + self.config.end() + super(StorageTestCase, self).tearDown() + del os.environ["MOZSVC_UUID"] + + def get_configurator(self): + config = super(StorageTestCase, self).get_configurator() + # config.include("syncstorage") + return config + + def _cleanup_test_databases(self): + """Clean up any database used during the tests.""" + # Find and clean up any in-use databases + for key, storage in self.config.registry.items(): + if not key.startswith("syncstorage:storage:"): + continue + while hasattr(storage, "storage"): + storage = storage.storage + # For server-based dbs, drop the tables to clear them. + if storage.dbconnector.driver in ("mysql", "postgres"): + with storage.dbconnector.connect() as c: + c.execute('DROP TABLE bso') + c.execute('DROP TABLE user_collections') + c.execute('DROP TABLE collections') + c.execute('DROP TABLE batch_uploads') + c.execute('DROP TABLE batch_upload_items') + # Explicitly free any pooled connections. + storage.dbconnector.engine.dispose() + # Find any sqlite database files and delete them. + for key, value in self.config.registry.settings.items(): + if key.endswith(".sqluri"): + sqluri = urlparse.urlparse(value) + if sqluri.scheme == 'sqlite' and ":memory:" not in value: + if os.path.isfile(sqluri.path): + os.remove(sqluri.path) + + +class FunctionalTestCase(TestCase): + """TestCase for writing functional tests using WebTest. + This TestCase subclass provides an easy mechanism to write functional + tests using WebTest. It exposes a TestApp instance as self.app. + If the environment variable MOZSVC_TEST_REMOTE is set to a URL, then + self.app will be a WSGIProxy application that forwards all requests to + that server. This allows the functional tests to be easily run against + a live server instance. + """ + + def setUp(self): + super(FunctionalTestCase, self).setUp() + + # now that we're testing against a rust server, we're always distant. + # but some tests don't run if we're set to distant. so let's set + # distant to false, figure out which tests we still want, and + # delete the ones that don't work with distant = True along + # with the need for self.distant. + self.distant = False + self.host_url = "http://localhost:8000" + # This call implicitly commits the configurator. We probably still + # want it for the side effects. + self.config.make_wsgi_app() + + host_url = urlparse.urlparse(self.host_url) + self.app = TestApp(self.host_url, extra_environ={ + "HTTP_HOST": host_url.netloc, + "wsgi.url_scheme": host_url.scheme or "http", + "SERVER_NAME": host_url.hostname, + "REMOTE_ADDR": "127.0.0.1", + "SCRIPT_NAME": host_url.path, + }) + + +class StorageFunctionalTestCase(FunctionalTestCase, StorageTestCase): + """Abstract base class for functional testing of a storage API.""" + + def setUp(self): + super(StorageFunctionalTestCase, self).setUp() + + # Generate userid and auth token crednentials. + # This can be overridden by subclasses. + self.config.commit() + self._authenticate() + + # Monkey-patch the app to sign all requests with the token. + def new_do_request(req, *args, **kwds): + hawkauthlib.sign_request(req, self.auth_token, self.auth_secret) + return orig_do_request(req, *args, **kwds) + orig_do_request = self.app.do_request + self.app.do_request = new_do_request + + def basic_testing_authenticate(self): + # For basic testing, use a random uid and sign our own tokens. + # Subclasses might like to override this and use a live tokenserver. + pass + + def _authenticate(self): + policy = self.config.registry.getUtility(IAuthenticationPolicy) + if global_secret is not None: + policy.secrets._secrets = [global_secret] + self.user_id = random.randint(1, 100000) + auth_policy = self.config.registry.getUtility(IAuthenticationPolicy) + req = Request.blank(self.host_url) + creds = auth_policy.encode_hawk_id( + req, self.user_id, extra={ + # Include a hashed_fxa_uid to trigger uid/kid extraction + "hashed_fxa_uid": str(uuid.uuid4()), + "fxa_uid": str(uuid.uuid4()), + "fxa_kid": str(uuid.uuid4()), + } + ) + self.auth_token, self.auth_secret = creds + + @contextlib.contextmanager + def _switch_user(self): + # It's hard to reliably switch users when testing a live server. + if self.distant: + raise unittest2.SkipTest("Skipped when testing a live server") + # Temporarily authenticate as a different user. + orig_user_id = self.user_id + orig_auth_token = self.auth_token + orig_auth_secret = self.auth_secret + try: + # We loop because the userids are randomly generated, + # so there's a small change we'll get the same one again. + for retry_count in range(10): + self._authenticate() + if self.user_id != orig_user_id: + break + else: + raise RuntimeError("Failed to switch to new user id") + yield + finally: + self.user_id = orig_user_id + self.auth_token = orig_auth_token + self.auth_secret = orig_auth_secret + + def _cleanup_test_databases(self): + # Don't cleanup databases unless we created them ourselves. + if not self.distant: + super(StorageFunctionalTestCase, self)._cleanup_test_databases() + + +MOCKMYID_DOMAIN = "mockmyid.s3-us-west-2.amazonaws.com" +MOCKMYID_PRIVATE_KEY = None +MOCKMYID_PRIVATE_KEY_DATA = { + "algorithm": "DS", + "x": "385cb3509f086e110c5e24bdd395a84b335a09ae", + "y": "738ec929b559b604a232a9b55a5295afc368063bb9c20fac4e53a74970a4db795" + "6d48e4c7ed523405f629b4cc83062f13029c4d615bbacb8b97f5e56f0c7ac9bc1" + "d4e23809889fa061425c984061fca1826040c399715ce7ed385c4dd0d40225691" + "2451e03452d3c961614eb458f188e3e8d2782916c43dbe2e571251ce38262", + "p": "ff600483db6abfc5b45eab78594b3533d550d9f1bf2a992a7a8daa6dc34f8045a" + "d4e6e0c429d334eeeaaefd7e23d4810be00e4cc1492cba325ba81ff2d5a5b305a" + "8d17eb3bf4a06a349d392e00d329744a5179380344e82a18c47933438f891e22a" + "eef812d69c8f75e326cb70ea000c3f776dfdbd604638c2ef717fc26d02e17", + "q": "e21e04f911d1ed7991008ecaab3bf775984309c3", + "g": "c52a4a0ff3b7e61fdf1867ce84138369a6154f4afa92966e3c827e25cfa6cf508b" + "90e5de419e1337e07a2e9e2a3cd5dea704d175f8ebf6af397d69e110b96afb17c7" + "a03259329e4829b0d03bbc7896b15b4ade53e130858cc34d96269aa89041f40913" + "6c7242a38895c9d5bccad4f389af1d7a4bd1398bd072dffa896233397a", +} + + +class PermissiveNonceCache(object): + """Object for not really managing a cache of used nonce values. + This class implements the timestamp/nonce checking interface required + by hawkauthlib, but doesn't actually check them. Instead it just logs + timestamps that are too far out of the timestamp window for future + analysis. + """ + + def __init__(self, log_window=60, get_time=None): + self.log_window = log_window + self.get_time = get_time or time.time + + def __len__(self): + raise NotImplementedError + + def check_nonce(self, timestamp, nonce): + """Check if the given timestamp+nonce is fresh.""" + now = self.get_time() + skew = now - timestamp + if abs(skew) > self.log_window: + print("Large timestamp skew detected: %d", skew) + return True + + +@implementer(IAuthenticationPolicy) +class TokenServerAuthenticationPolicy(HawkAuthenticationPolicy): + """Pyramid authentication policy for use with Tokenserver auth tokens. + This class provides an IAuthenticationPolicy implementation based on + the Mozilla TokenServer authentication tokens as described here: + https://docs.services.mozilla.com/token/ + For verification of token signatures, this plugin can use either a + single fixed secret (via the argument 'secret') or a file mapping + node hostnames to secrets (via the argument 'secrets_file'). The + two arguments are mutually exclusive. + """ + + def __init__(self, secrets=None, **kwds): + if not secrets: + # Using secret=None will cause tokenlib to use a randomly-generated + # secret. This is useful for getting started without having to + # twiddle any configuration files, but probably not what anyone + # wants to use long-term. + secrets = None + msgs = ["WARNING: using a randomly-generated token secret.", + "You probably want to set 'secret' or 'secrets_file' in " + "the [hawkauth] section of your configuration"] + for msg in msgs: + print("warn:", msg) + elif isinstance(secrets, (str, list)): + secrets = FixedSecrets(secrets) + elif isinstance(secrets, dict): + secrets = resolve_name(secrets.pop("backend"))(**secrets) + self.secrets = secrets + if kwds.get("nonce_cache") is None: + kwds["nonce_cache"] = PermissiveNonceCache() + super(TokenServerAuthenticationPolicy, self).__init__(**kwds) + + @classmethod + def _parse_settings(cls, settings): + """Parse settings for an instance of this class.""" + supercls = super(TokenServerAuthenticationPolicy, cls) + kwds = supercls._parse_settings(settings) + # collect leftover settings into a config for a Secrets object, + # wtih some b/w compat for old-style secret-handling settings. + secrets_prefix = "secrets." + secrets = {} + if "secrets_file" in settings: + if "secret" in settings: + raise ValueError("can't use both 'secret' and 'secrets_file'") + secrets["backend"] = "test_support.Secrets" + secrets["filename"] = settings.pop("secrets_file") + elif "secret" in settings: + secrets["backend"] = "test_support.FixedSecrets" + secrets["secrets"] = settings.pop("secret") + for name in settings.keys(): + if name.startswith(secrets_prefix): + secrets[name[len(secrets_prefix):]] = settings.pop(name) + kwds['secrets'] = secrets + return kwds + + def decode_hawk_id(self, request, tokenid): + """Decode a Hawk token id into its userid and secret key. + This method determines the appropriate secrets to use for the given + request, then passes them on to tokenlib to handle the given Hawk + token. + If the id is invalid then ValueError will be raised. + """ + # There might be multiple secrets in use, if we're in the + # process of transitioning from one to another. Try each + # until we find one that works. + node_name = self._get_node_name(request) + secrets = self._get_token_secrets(node_name) + for secret in secrets: + try: + data = tokenlib.parse_token(tokenid, secret=secret) + userid = data["uid"] + token_node_name = data["node"] + if token_node_name != node_name: + raise ValueError("incorrect node for this token") + key = tokenlib.get_derived_secret(tokenid, secret=secret) + break + except (ValueError, KeyError): + pass + else: + print("warn: Authentication Failed: invalid hawk id") + raise ValueError("invalid Hawk id") + return userid, key + + def encode_hawk_id(self, request, userid, extra=None): + """Encode the given userid into a Hawk id and secret key. + This method is essentially the reverse of decode_hawk_id. It is + not needed for consuming authentication tokens, but is very useful + when building them for testing purposes. + """ + node_name = self._get_node_name(request) + # There might be multiple secrets in use, if we're in the + # process of transitioning from one to another. Always use + # the last one aka the "most recent" secret. + secret = self._get_token_secrets(node_name)[-1] + data = {"uid": userid, "node": node_name} + tokenid = tokenlib.make_token(data, secret=secret) + key = tokenlib.get_derived_secret(tokenid, secret=secret) + return tokenid, key + + def _get_node_name(self, request): + """Get the canonical node name for the given request.""" + # Secrets are looked up by hostname. + # We need to normalize some port information for this work right. + node_name = request.host_url + if node_name.startswith("http:") and node_name.endswith(":80"): + node_name = node_name[:-3] + elif node_name.startswith("https:") and node_name.endswith(":443"): + node_name = node_name[:-4] + return node_name + request.script_name + + def _get_token_secrets(self, node_name): + """Get the list of possible secrets for signing tokens.""" + if self.secrets is None: + return [None] + return self.secrets.get(node_name) + + +@implementer(IAuthenticationPolicy) +class SyncStorageAuthenticationPolicy(TokenServerAuthenticationPolicy): + """Pyramid authentication policy with special handling of expired tokens. + + This class extends the standard mozsvc TokenServerAuthenticationPolicy + to (carefully) allow some access by holders of expired tokens. Presenting + an expired token will result in a principal of "expired:" rather than + just "", allowing this case to be specially detected and handled for + some resources without interfering with the usual authentication rules. + """ + + def __init__(self, secrets=None, **kwds): + self.expired_token_timeout = kwds.pop("expired_token_timeout", None) + if self.expired_token_timeout is None: + self.expired_token_timeout = 300 + super(SyncStorageAuthenticationPolicy, self).__init__(secrets, **kwds) + + @classmethod + def _parse_settings(cls, settings): + """Parse settings for an instance of this class.""" + supercls = super(SyncStorageAuthenticationPolicy, cls) + kwds = supercls._parse_settings(settings) + expired_token_timeout = settings.pop("expired_token_timeout", None) + if expired_token_timeout is not None: + kwds["expired_token_timeout"] = int(expired_token_timeout) + return kwds + + def decode_hawk_id(self, request, tokenid): + """Decode a Hawk token id into its userid and secret key. + + This method determines the appropriate secrets to use for the given + request, then passes them on to tokenlib to handle the given Hawk + token. If the id is invalid then ValueError will be raised. + + Unlike the superclass method, this implementation allows expired + tokens to be used up to a configurable timeout. The effective userid + for expired tokens is changed to be "expired:". + """ + now = time.time() + node_name = self._get_node_name(request) + # There might be multiple secrets in use, + # so try each until we find one that works. + secrets = self._get_token_secrets(node_name) + for secret in secrets: + try: + tm = tokenlib.TokenManager(secret=secret) + # Check for a proper valid signature first. + # If that failed because of an expired token, check if + # it falls within the allowable expired-token window. + try: + data = self._parse_token(tm, tokenid, now) + userid = data["uid"] + except tokenlib.errors.ExpiredTokenError: + recently = now - self.expired_token_timeout + data = self._parse_token(tm, tokenid, recently) + # We replace the uid with a special string to ensure that + # calling code doesn't accidentally treat the token as + # valid. If it wants to use the expired uid, it will have + # to explicitly dig it back out from `request.user`. + data["expired_uid"] = data["uid"] + userid = data["uid"] = "expired:%d" % (data["uid"],) + except tokenlib.errors.InvalidSignatureError: + # Token signature check failed, try the next secret. + continue + except TypeError as e: + # Something went wrong when validating the contained data. + raise ValueError(str(e)) + else: + # Token signature check succeeded, quit the loop. + break + else: + # The token failed to validate using any secret. + print("warn Authentication Failed: invalid hawk id") + raise ValueError("invalid Hawk id") + + # Let the app access all user data from the token. + request.user.update(data) + request.metrics["metrics_uid"] = data.get("hashed_fxa_uid") + request.metrics["metrics_device_id"] = data.get("hashed_device_id") + + # Sanity-check that we're on the right node. + if data["node"] != node_name: + msg = "incorrect node for this token: %s" + raise ValueError(msg % (data["node"],)) + + # Calculate the matching request-signing secret. + key = tokenlib.get_derived_secret(tokenid, secret=secret) + + return userid, key + + def encode_hawk_id(self, request, userid, extra=None): + """Encode the given userid into a Hawk id and secret key. + + This method is essentially the reverse of decode_hawk_id. It is + not needed for consuming authentication tokens, but is very useful + when building them for testing purposes. + + Unlike its superclass method, this one allows the caller to specify + a dict of additional user data to include in the auth token. + """ + node_name = self._get_node_name(request) + secret = self._get_token_secrets(node_name)[-1] + data = {"uid": userid, "node": node_name} + if extra is not None: + data.update(extra) + tokenid = tokenlib.make_token(data, secret=secret) + key = tokenlib.get_derived_secret(tokenid, secret=secret) + return tokenid, key + + def _parse_token(self, tokenmanager, tokenid, now): + """Parse, validate and normalize user data from a tokenserver token. + + This is a thin wrapper around tokenmanager.parse_token to apply + some extra validation to the contained user data. The data is + signed and trusted, but it's still coming from outside the system + so it's good defense-in-depth to validate it at our app boundary. + + We also deal with some historical baggage by renaming fields + as needed. + """ + data = tokenmanager.parse_token(tokenid, now=now) + user = {} + + # It should always contain an integer userid. + try: + user["uid"] = data["uid"] + except KeyError: + raise ValueError("missing uid in token data") + else: + if not isinstance(user["uid"], int) or user["uid"] < 0: + raise ValueError("invalid uid in token data") + + # It should always contain a string node name. + try: + user["node"] = data["node"] + except KeyError: + raise ValueError("missing node in token data") + else: + if not isinstance(user["node"], str): + raise ValueError("invalid node in token data") + + # It might contain additional user identifiers for + # storage and metrics purposes. + # + # There's some historical baggage here. + # + # Old versions of tokenserver would send a hashed "metrics uid" as the + # "fxa_uid" key, attempting a small amount of anonymization. Newer + # versions of tokenserver send the raw uid as "fxa_uid" and the hashed + # version as "hashed_fxa_uid". The raw version may be used associating + # stored data with a specific user, but the hashed version is the one + # that we want for metrics. + + if "hashed_fxa_uid" in data: + user["hashed_fxa_uid"] = data["hashed_fxa_uid"] + if not VALID_FXA_ID_REGEX.match(user["hashed_fxa_uid"]): + raise ValueError("invalid hashed_fxa_uid in token data") + try: + user["fxa_uid"] = data["fxa_uid"] + except KeyError: + raise ValueError("missing fxa_uid in token data") + else: + if not VALID_FXA_ID_REGEX.match(user["fxa_uid"]): + raise ValueError("invalid fxa_uid in token data") + try: + user["fxa_kid"] = data["fxa_kid"] + except KeyError: + raise ValueError("missing fxa_kid in token data") + else: + if not VALID_FXA_ID_REGEX.match(user["fxa_kid"]): + raise ValueError("invalid fxa_kid in token data") + elif "fxa_uid" in data: + user["hashed_fxa_uid"] = data["fxa_uid"] + if not VALID_FXA_ID_REGEX.match(user["hashed_fxa_uid"]): + raise ValueError("invalid fxa_uid in token data") + + if "hashed_device_id" in data: + user["hashed_device_id"] = data["hashed_device_id"] + if not VALID_FXA_ID_REGEX.match(user["hashed_device_id"]): + raise ValueError("invalid hashed_device_id in token data") + elif "device_id" in data: + user["hashed_device_id"] = data.get("device_id") + if not VALID_FXA_ID_REGEX.match(user["hashed_device_id"]): + raise ValueError("invalid device_id in token data") + return user + + +def run_live_functional_tests(TestCaseClass, argv=None): + """Execute the given suite of testcases against a live server.""" + if argv is None: + argv = sys.argv + + # This will only work using a StorageFunctionalTestCase subclass, + # since we override the _authenticate() method. + assert issubclass(TestCaseClass, StorageFunctionalTestCase) + + usage = "Usage: %prog [options] " + parser = optparse.OptionParser(usage=usage) + parser.add_option("-x", "--failfast", action="store_true", + help="stop after the first failed test") + parser.add_option("", "--config-file", + help="name of the config file in use by the server") + parser.add_option("", "--use-token-server", action="store_true", + help="the given URL is a tokenserver, not an endpoint") + parser.add_option("", "--email", + help="email address to use for tokenserver tests") + parser.add_option("", "--audience", + help="assertion audience to use for tokenserver tests") + + try: + opts, args = parser.parse_args(argv) + except SystemExit as e: + return e.args[0] + if len(args) != 2: + parser.print_usage() + return 2 + + url = args[1] + if opts.config_file is not None: + os.environ["MOZSVC_TEST_INI_FILE"] = opts.config_file + + # If we're not using the tokenserver, the default implementation of + # _authenticate will do just fine. We optionally accept the token + # signing secret in the url hash fragement. + if opts.email is not None: + msg = "cant specify email address unless using live tokenserver" + raise ValueError(msg) + if opts.audience is not None: + msg = "cant specify audience unless using live tokenserver" + raise ValueError(msg) + host_url = urlparse.urlparse(url) + if host_url.fragment: + global global_secret + global_secret = host_url.fragment + host_url = host_url._replace(fragment="") + os.environ["MOZSVC_TEST_REMOTE"] = 'localhost' + + # Now use the unittest2 runner to execute them. + suite = unittest2.TestSuite() + import test_storage + test_prefix = os.environ.get("SYNC_TEST_PREFIX", "test") + suite.addTest(unittest2.findTestCases(test_storage, test_prefix)) + # suite.addTest(unittest2.makeSuite(LiveTestCases, prefix=test_prefix)) + runner = unittest2.TextTestRunner( + stream=sys.stderr, + failfast=opts.failfast, + verbosity=2, + ) + res = runner.run(suite) + if not res.wasSuccessful(): + return 1 + return 0 + + +# Tell over-zealous test discovery frameworks that this isn't a real test. +run_live_functional_tests.__test__ = False diff --git a/tools/integration_tests/tests-no-batch.ini b/tools/integration_tests/tests-no-batch.ini new file mode 100644 index 00000000..0948e94b --- /dev/null +++ b/tools/integration_tests/tests-no-batch.ini @@ -0,0 +1,21 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:SyncStorage + +[storage] +backend = syncstorage.storage.sql.SQLStorage +sqluri = ${MOZSVC_SQLURI} +standard_collections = true +quota_size = 5242880 +pool_size = 100 +pool_recycle = 3600 +reset_on_return = true +create_tables = true +max_post_records = 4000 + +[hawkauth] +secret = "TED KOPPEL IS A ROBOT" diff --git a/tools/integration_tests/tests-paginated.ini b/tools/integration_tests/tests-paginated.ini new file mode 100644 index 00000000..103c5c99 --- /dev/null +++ b/tools/integration_tests/tests-paginated.ini @@ -0,0 +1,23 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:SyncStorage + +[storage] +backend = syncstorage.storage.sql.SQLStorage +sqluri = ${MOZSVC_SQLURI} +standard_collections = true +quota_size = 5242880 +pool_size = 100 +pool_recycle = 3600 +reset_on_return = true +create_tables = true +batch_upload_enabled = true +# Use a small batch-size to help test internal pagination usage. +pagination_batch_size = 4 + +[hawkauth] +secret = "TED KOPPEL IS A ROBOT" diff --git a/tools/integration_tests/tests.ini b/tools/integration_tests/tests.ini new file mode 100644 index 00000000..b1863c7e --- /dev/null +++ b/tools/integration_tests/tests.ini @@ -0,0 +1,26 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:SyncStorage + +[storage] +backend = syncstorage.storage.sql.SQLStorage +sqluri = ${MOZSVC_SQLURI} +standard_collections = true +quota_size = 5242880 +pool_size = 100 +pool_recycle = 3600 +reset_on_return = true +create_tables = true +max_post_records = 4000 +batch_upload_enabled = true +force_consistent_sort_order = true + +[hawkauth] +secret = "TED KOPPEL IS A ROBOT" + +[endpoints] +syncstorage-rs = http://localhost:8000/1.5/1