mirrors/syncstorage-rs
1
0
Fork 0
mirror of https://github.com/mozilla-services/syncstorage-rs.git synced 2025-08-06 11:56:58 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity
d6e6c2c0a1
syncstorage-rs/tools/integration_tests/test_storage.py
Donovan Preston d6e6c2c0a1
Py3 integration tests (#917) 
* feat: Port integration tests to py3, and run them in the same docker container as the rust server.

Fix #911
2020-12-01 21:20:59 -05:00

2086 lines
90 KiB
Python
Raw Blame History

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Functional tests for the SyncStorage server protocol.
This file runs tests to ensure the correct operation of the server
as specified in:
http://docs.services.mozilla.com/storage/apis-1.5.html
If there's an aspect of that spec that's not covered by a test in this file,
consider it a bug.
"""
import unittest2
import re
import json
import time
import random
import string
import urllib
import webtest
import contextlib
# import math
import simplejson
from pyramid.interfaces import IAuthenticationPolicy
from test_support import StorageFunctionalTestCase
import tokenlib
class ConflictError(Exception):
pass
class BackendError(Exception):
pass
WEAVE_INVALID_WBO = 8 # Invalid Weave Basic Object
WEAVE_SIZE_LIMIT_EXCEEDED = 17 # Size limit exceeded
BATCH_MAX_IDS = 100
def get_limit_config(request, limit):
"""Get the configured value for the named size limit."""
return request.registry.settings["storage." + limit]
def json_dumps(value):
"""Decimal-aware version of json.dumps()."""
return simplejson.dumps(value, use_decimal=True)
def json_loads(value):
"""Decimal-aware version of json.loads()."""
return simplejson.loads(value, use_decimal=True)
_PLD = '*' * 500
_ASCII = string.ascii_letters + string.digits
def randtext(size=10):
return ''.join([random.choice(_ASCII) for i in range(size)])
class TestStorage(StorageFunctionalTestCase):
"""Storage testcases that only use the web API.
These tests are suitable for running against both in-process and live
external web servers.
"""
def setUp(self):
super(TestStorage, self).setUp()
self.root = '/1.5/%d' % (self.user_id,)
# Reset the storage to a known state, aka "empty".
self.retry_delete(self.root)
@contextlib.contextmanager
def _switch_user(self):
orig_root = self.root
try:
with super(TestStorage, self)._switch_user():
self.root = '/1.5/%d' % (self.user_id,)
yield
finally:
self.root = orig_root
def retry_post_json(self, *args, **kwargs):
return self._retry_send(self.app.post_json, *args, **kwargs)
def retry_put_json(self, *args, **kwargs):
return self._retry_send(self.app.put_json, *args, **kwargs)
def retry_delete(self, *args, **kwargs):
return self._retry_send(self.app.delete, *args, **kwargs)
def _retry_send(self, func, *args, **kwargs):
try:
return func(*args, **kwargs)
except webtest.AppError as ex:
if "409 " not in ex.args[0] and "503 " not in ex.args[0]:
raise ex
time.sleep(0.01)
return func(*args, **kwargs)
def test_get_info_collections(self):
# xxx_col1 gets 3 items, xxx_col2 gets 5 items.
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)]
resp = self.retry_post_json(self.root + "/storage/xxx_col1", bsos)
ts1 = resp.json["modified"]
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
resp = self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
ts2 = resp.json["modified"]
# only those collections should appear in the query.
resp = self.app.get(self.root + '/info/collections')
res = resp.json
keys = sorted(list(res.keys()))
self.assertEquals(keys, ["xxx_col1", "xxx_col2"])
self.assertEquals(res["xxx_col1"], ts1)
self.assertEquals(res["xxx_col2"], ts2)
# Updating items in xxx_col2, check timestamps.
bsos = [{"id": str(i).zfill(2), "payload": "yyy"} for i in range(2)]
resp = self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
self.assertTrue(ts2 < resp.json["modified"])
ts2 = resp.json["modified"]
resp = self.app.get(self.root + '/info/collections')
res = resp.json
keys = sorted(list(res.keys()))
self.assertEquals(keys, ["xxx_col1", "xxx_col2"])
self.assertEquals(res["xxx_col1"], ts1)
self.assertEquals(res["xxx_col2"], ts2)
def test_get_collection_count(self):
# xxx_col1 gets 3 items, xxx_col2 gets 5 items.
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)]
self.retry_post_json(self.root + "/storage/xxx_col1", bsos)
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
# those counts should be reflected back in query.
resp = self.app.get(self.root + '/info/collection_counts')
res = resp.json
self.assertEquals(len(res), 2)
self.assertEquals(res["xxx_col1"], 3)
self.assertEquals(res["xxx_col2"], 5)
def test_bad_cache(self):
# fixes #637332
# the collection name <-> id mapper is temporarely cached to
# save a few requests.
# but should get purged when new collections are added
# 1. get collection info
resp = self.app.get(self.root + '/info/collections')
numcols = len(resp.json)
# 2. add a new collection + stuff
bso = {'id': '125', 'payload': _PLD}
self.retry_put_json(self.root + '/storage/xxxx/125', bso)
# 3. get collection info again, should find the new ones
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(len(resp.json), numcols + 1)
def test_get_collection_only(self):
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
# non-existent collections appear as empty
resp = self.app.get(self.root + '/storage/nonexistent')
res = resp.json
self.assertEquals(res, [])
# try just getting all items at once.
resp = self.app.get(self.root + '/storage/xxx_col2')
res = resp.json
res.sort()
self.assertEquals(res, ['00', '01', '02', '03', '04'])
self.assertEquals(int(resp.headers['X-Weave-Records']), 5)
# trying various filters
# "ids"
# Returns the ids for objects in the collection that are in the
# provided comma-separated list.
res = self.app.get(self.root + '/storage/xxx_col2?ids=01,03,17')
res = res.json
res.sort()
self.assertEquals(res, ['01', '03'])
# "newer"
# Returns only ids for objects in the collection that have been last
# modified after the timestamp given.
self.retry_delete(self.root + '/storage/xxx_col2')
bso = {'id': '128', 'payload': 'x'}
res = self.retry_put_json(self.root + '/storage/xxx_col2/128', bso)
ts1 = float(res.headers["X-Last-Modified"])
bso = {'id': '129', 'payload': 'x'}
res = self.retry_put_json(self.root + '/storage/xxx_col2/129', bso)
ts2 = float(res.headers["X-Last-Modified"])
self.assertTrue(ts1 < ts2)
res = self.app.get(self.root + '/storage/xxx_col2?newer=%s' % ts1)
self.assertEquals(res.json, ['129'])
res = self.app.get(self.root + '/storage/xxx_col2?newer=%s' % ts2)
self.assertEquals(res.json, [])
res = self.app.get(
self.root + '/storage/xxx_col2?newer=%s' % (ts1 - 1))
self.assertEquals(sorted(res.json), ['128', '129'])
# "older"
# Returns only ids for objects in the collection that have been last
# modified before the timestamp given.
self.retry_delete(self.root + '/storage/xxx_col2')
bso = {'id': '128', 'payload': 'x'}
res = self.retry_put_json(self.root + '/storage/xxx_col2/128', bso)
ts1 = float(res.headers["X-Last-Modified"])
bso = {'id': '129', 'payload': 'x'}
res = self.retry_put_json(self.root + '/storage/xxx_col2/129', bso)
ts2 = float(res.headers["X-Last-Modified"])
self.assertTrue(ts1 < ts2)
res = self.app.get(self.root + '/storage/xxx_col2?older=%s' % ts1)
self.assertEquals(res.json, [])
res = self.app.get(self.root + '/storage/xxx_col2?older=%s' % ts2)
self.assertEquals(res.json, ['128'])
res = self.app.get(
self.root + '/storage/xxx_col2?older=%s' % (ts2 + 1))
self.assertEquals(sorted(res.json), ['128', '129'])
qs = '?older=%s&newer=%s' % (ts2 + 1, ts1)
res = self.app.get(self.root + '/storage/xxx_col2' + qs)
self.assertEquals(sorted(res.json), ['129'])
# "full"
# If defined, returns the full BSO, rather than just the id.
res = self.app.get(self.root + '/storage/xxx_col2?full=1')
keys = list(res.json[0].keys())
keys.sort()
wanted = ['id', 'modified', 'payload']
self.assertEquals(keys, wanted)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertTrue(isinstance(res.json, list))
# "limit"
# Sets the maximum number of ids that will be returned
self.retry_delete(self.root + '/storage/xxx_col2')
bsos = []
for i in range(10):
bso = {'id': str(i).zfill(2), 'payload': 'x', 'sortindex': i}
bsos.append(bso)
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
query_url = self.root + '/storage/xxx_col2?sort=index'
res = self.app.get(query_url)
all_items = res.json
self.assertEquals(len(all_items), 10)
res = self.app.get(query_url + '&limit=2')
self.assertEquals(res.json, all_items[:2])
# "offset"
# Skips over items that have already been returned.
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&limit=3&offset=' + next_offset)
self.assertEquals(res.json, all_items[2:5])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
self.assertTrue("X-Weave-Next-Offset" not in res.headers)
res = self.app.get(
query_url + '&limit=10000&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
self.assertTrue("X-Weave-Next-Offset" not in res.headers)
# "offset" again, this time ordering by descending timestamp.
query_url = self.root + '/storage/xxx_col2?sort=newest'
res = self.app.get(query_url)
all_items = res.json
self.assertEquals(len(all_items), 10)
res = self.app.get(query_url + '&limit=2')
self.assertEquals(res.json, all_items[:2])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&limit=3&offset=' + next_offset)
self.assertEquals(res.json, all_items[2:5])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
self.assertTrue("X-Weave-Next-Offset" not in res.headers)
res = self.app.get(
query_url + '&limit=10000&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
# "offset" again, this time ordering by ascending timestamp.
query_url = self.root + '/storage/xxx_col2?sort=oldest'
res = self.app.get(query_url)
all_items = res.json
self.assertEquals(len(all_items), 10)
res = self.app.get(query_url + '&limit=2')
self.assertEquals(res.json, all_items[:2])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&limit=3&offset=' + next_offset)
self.assertEquals(res.json, all_items[2:5])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
self.assertTrue("X-Weave-Next-Offset" not in res.headers)
res = self.app.get(
query_url + '&limit=10000&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
# "offset" once more, this time with no explicit ordering
query_url = self.root + '/storage/xxx_col2?'
res = self.app.get(query_url)
all_items = res.json
self.assertEquals(len(all_items), 10)
res = self.app.get(query_url + '&limit=2')
self.assertEquals(res.json, all_items[:2])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&limit=3&offset=' + next_offset)
self.assertEquals(res.json, all_items[2:5])
next_offset = res.headers["X-Weave-Next-Offset"]
res = self.app.get(query_url + '&offset=' + next_offset)
self.assertEquals(res.json, all_items[5:])
self.assertTrue("X-Weave-Next-Offset" not in res.headers)
res = self.app.get(
query_url + '&limit=10000&offset=' + next_offset)
# "sort"
# 'newest': Orders by timestamp number (newest first)
# 'oldest': Orders by timestamp number (oldest first)
# 'index': Orders by the sortindex descending (highest weight first)
self.retry_delete(self.root + '/storage/xxx_col2')
for index, sortindex in (('00', -1), ('01', 34), ('02', 12)):
bso = {'id': index, 'payload': 'x', 'sortindex': sortindex}
self.retry_post_json(self.root + '/storage/xxx_col2', [bso])
res = self.app.get(self.root + '/storage/xxx_col2?sort=newest')
res = res.json
self.assertEquals(res, ['02', '01', '00'])
res = self.app.get(self.root + '/storage/xxx_col2?sort=oldest')
res = res.json
self.assertEquals(res, ['00', '01', '02'])
res = self.app.get(self.root + '/storage/xxx_col2?sort=index')
res = res.json
self.assertEquals(res, ['01', '02', '00'])
def test_alternative_formats(self):
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
# application/json
res = self.app.get(self.root + '/storage/xxx_col2',
headers=[('Accept', 'application/json')])
self.assertEquals(res.content_type.split(";")[0], 'application/json')
res = res.json
res.sort()
self.assertEquals(res, ['00', '01', '02', '03', '04'])
# application/newlines
res = self.app.get(self.root + '/storage/xxx_col2',
headers=[('Accept', 'application/newlines')])
self.assertEquals(res.content_type, 'application/newlines')
self.assertTrue(res.body.endswith(b'\n'))
res = [json_loads(line) for line in res.body.decode(
'utf-8').strip().split('\n')]
res.sort()
self.assertEquals(res, ['00', '01', '02', '03', '04'])
# unspecified format defaults to json
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(res.content_type.split(";")[0], 'application/json')
# unkown format gets a 406
self.app.get(self.root + '/storage/xxx_col2',
headers=[('Accept', 'x/yy')], status=406)
def test_set_collection_with_if_modified_since(self):
# Create five items with different timestamps.
for i in range(5):
bsos = [{"id": str(i).zfill(2), "payload": "xxx"}]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
# Get them all, along with their timestamps.
res = self.app.get(self.root + '/storage/xxx_col2?full=true').json
self.assertEquals(len(res), 5)
timestamps = sorted([r["modified"] for r in res])
# The timestamp of the collection should be the max of all those.
self.app.get(self.root + "/storage/xxx_col2", headers={
"X-If-Modified-Since": str(timestamps[0])
}, status=200)
res = self.app.get(self.root + "/storage/xxx_col2", headers={
"X-If-Modified-Since": str(timestamps[-1])
}, status=304)
self.assertTrue("X-Last-Modified" in res.headers)
def test_get_item(self):
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
# grabbing object 1 from xxx_col2
res = self.app.get(self.root + '/storage/xxx_col2/01')
res = res.json
keys = list(res.keys())
keys.sort()
self.assertEquals(keys, ['id', 'modified', 'payload'])
self.assertEquals(res['id'], '01')
# unexisting object
self.app.get(self.root + '/storage/xxx_col2/99', status=404)
# using x-if-modified-since header.
self.app.get(self.root + '/storage/xxx_col2/01', headers={
"X-If-Modified-Since": str(res["modified"])
}, status=304)
self.app.get(self.root + '/storage/xxx_col2/01', headers={
"X-If-Modified-Since": str(res["modified"] + 1)
}, status=304)
res = self.app.get(self.root + '/storage/xxx_col2/01', headers={
"X-If-Modified-Since": str(res["modified"] - 1)
})
self.assertEquals(res.json['id'], '01')
def test_set_item(self):
# let's create an object
bso = {'payload': _PLD}
self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
res = self.app.get(self.root + '/storage/xxx_col2/12345')
res = res.json
self.assertEquals(res['payload'], _PLD)
# now let's update it
bso = {'payload': 'YYY'}
self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
res = self.app.get(self.root + '/storage/xxx_col2/12345')
res = res.json
self.assertEquals(res['payload'], 'YYY')
def test_set_collection(self):
# sending two bsos
bso1 = {'id': '12', 'payload': _PLD}
bso2 = {'id': '13', 'payload': _PLD}
bsos = [bso1, bso2]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
# checking what we did
res = self.app.get(self.root + '/storage/xxx_col2/12')
res = res.json
self.assertEquals(res['payload'], _PLD)
res = self.app.get(self.root + '/storage/xxx_col2/13')
res = res.json
self.assertEquals(res['payload'], _PLD)
# one more time, with changes
bso1 = {'id': '13', 'payload': 'XyX'}
bso2 = {'id': '14', 'payload': _PLD}
bsos = [bso1, bso2]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
# checking what we did
res = self.app.get(self.root + '/storage/xxx_col2/14')
res = res.json
self.assertEquals(res['payload'], _PLD)
res = self.app.get(self.root + '/storage/xxx_col2/13')
res = res.json
self.assertEquals(res['payload'], 'XyX')
# sending two bsos with one bad sortindex
bso1 = {'id': 'one', 'payload': _PLD}
bso2 = {'id': 'two', 'payload': _PLD,
'sortindex': 'FAIL'}
bsos = [bso1, bso2]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
self.app.get(self.root + '/storage/xxx_col2/two', status=404)
def test_set_collection_input_formats(self):
# If we send with application/newlines it should work.
bso1 = {'id': '12', 'payload': _PLD}
bso2 = {'id': '13', 'payload': _PLD}
bsos = [bso1, bso2]
body = "\n".join(json_dumps(bso) for bso in bsos)
self.app.post(self.root + '/storage/xxx_col2', body, headers={
"Content-Type": "application/newlines"
})
items = self.app.get(self.root + "/storage/xxx_col2").json
self.assertEquals(len(items), 2)
# If we send an unknown content type, we get an error.
self.retry_delete(self.root + "/storage/xxx_col2")
body = json_dumps(bsos)
self.app.post(self.root + '/storage/xxx_col2', body, headers={
"Content-Type": "application/octet-stream"
}, status=415)
items = self.app.get(self.root + "/storage/xxx_col2").json
self.assertEquals(len(items), 0)
def test_set_item_input_formats(self):
# If we send with application/json it should work.
body = json_dumps({'payload': _PLD})
self.app.put(self.root + '/storage/xxx_col2/TEST', body, headers={
"Content-Type": "application/json"
})
item = self.app.get(self.root + "/storage/xxx_col2/TEST").json
self.assertEquals(item["payload"], _PLD)
# If we send json with some other content type, it should fail
self.retry_delete(self.root + "/storage/xxx_col2")
self.app.put(self.root + '/storage/xxx_col2/TEST', body, headers={
"Content-Type": "application/octet-stream"
}, status=415)
self.app.get(self.root + "/storage/xxx_col2/TEST", status=404)
# Unless we use text/plain, which is a special bw-compat case.
self.app.put(self.root + '/storage/xxx_col2/TEST', body, headers={
"Content-Type": "text/plain"
})
item = self.app.get(self.root + "/storage/xxx_col2/TEST").json
self.assertEquals(item["payload"], _PLD)
def test_app_newlines_when_payloads_contain_newlines(self):
# Send some application/newlines with embedded newline chars.
bsos = [
{'id': '01', 'payload': 'hello\nworld'},
{'id': '02', 'payload': '\nmarco\npolo\n'},
]
body = "\n".join(json_dumps(bso) for bso in bsos)
self.assertEquals(len(body.split("\n")), 2)
self.app.post(self.root + '/storage/xxx_col2', body, headers={
"Content-Type": "application/newlines"
})
# Read them back as JSON list, check payloads.
items = self.app.get(self.root + "/storage/xxx_col2?full=1").json
self.assertEquals(len(items), 2)
items.sort(key=lambda bso: bso["id"])
self.assertEquals(items[0]["payload"], bsos[0]["payload"])
self.assertEquals(items[1]["payload"], bsos[1]["payload"])
# Read them back as application/newlines, check payloads.
res = self.app.get(self.root + "/storage/xxx_col2?full=1", headers={
"Accept": "application/newlines",
})
items = [json_loads(line) for line in res.body.decode(
'utf-8').strip().split('\n')]
self.assertEquals(len(items), 2)
items.sort(key=lambda bso: bso["id"])
self.assertEquals(items[0]["payload"], bsos[0]["payload"])
self.assertEquals(items[1]["payload"], bsos[1]["payload"])
def test_collection_usage(self):
self.retry_delete(self.root + "/storage")
bso1 = {'id': '13', 'payload': 'XyX'}
bso2 = {'id': '14', 'payload': _PLD}
bsos = [bso1, bso2]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = self.app.get(self.root + '/info/collection_usage')
usage = res.json
xxx_col2_size = usage['xxx_col2']
wanted = (len(bso1['payload']) + len(bso2['payload'])) / 1024.0
self.assertEqual(round(xxx_col2_size, 2), round(wanted, 2))
def test_delete_collection_items(self):
# creating a collection of three
bso1 = {'id': '12', 'payload': _PLD}
bso2 = {'id': '13', 'payload': _PLD}
bso3 = {'id': '14', 'payload': _PLD}
bsos = [bso1, bso2, bso3]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 3)
# deleting all items
self.retry_delete(self.root + '/storage/xxx_col2')
items = self.app.get(self.root + '/storage/xxx_col2').json
self.assertEquals(len(items), 0)
# Deletes the ids for objects in the collection that are in the
# provided comma-separated list.
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 3)
self.retry_delete(self.root + '/storage/xxx_col2?ids=12,14')
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 1)
self.retry_delete(self.root + '/storage/xxx_col2?ids=13')
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 0)
def test_delete_item(self):
# creating a collection of three
bso1 = {'id': '12', 'payload': _PLD}
bso2 = {'id': '13', 'payload': _PLD}
bso3 = {'id': '14', 'payload': _PLD}
bsos = [bso1, bso2, bso3]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 3)
ts = float(res.headers['X-Last-Modified'])
# deleting item 13
self.retry_delete(self.root + '/storage/xxx_col2/13')
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 2)
# unexisting item should return a 404
self.retry_delete(self.root + '/storage/xxx_col2/12982', status=404)
# The collection should get an updated timestsamp.
res = self.app.get(self.root + '/info/collections')
self.assertTrue(ts < float(res.headers['X-Last-Modified']))
def test_delete_storage(self):
# creating a collection of three
bso1 = {'id': '12', 'payload': _PLD}
bso2 = {'id': '13', 'payload': _PLD}
bso3 = {'id': '14', 'payload': _PLD}
bsos = [bso1, bso2, bso3]
self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 3)
# deleting all
self.retry_delete(self.root + '/storage')
items = self.app.get(self.root + '/storage/xxx_col2').json
self.assertEquals(len(items), 0)
self.retry_delete(self.root + '/storage/xxx_col2', status=200)
self.assertEquals(len(items), 0)
def test_x_timestamp_header(self):
# This can't be run against a live server.
if self.distant:
raise unittest2.SkipTest
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
now = round(time.time(), 2)
time.sleep(0.01)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertTrue(now <= float(res.headers['X-Weave-Timestamp']))
# getting the timestamp with a PUT
now = round(time.time(), 2)
time.sleep(0.01)
bso = {'payload': _PLD}
res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
self.assertTrue(now <= float(res.headers['X-Weave-Timestamp']))
self.assertTrue(abs(now -
float(res.headers['X-Weave-Timestamp'])) <= 200)
# getting the timestamp with a POST
now = round(time.time(), 2)
time.sleep(0.01)
bso1 = {'id': '12', 'payload': _PLD}
bso2 = {'id': '13', 'payload': _PLD}
bsos = [bso1, bso2]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
self.assertTrue(now <= float(res.headers['X-Weave-Timestamp']))
def test_ifunmodifiedsince(self):
bso = {'id': '12345', 'payload': _PLD}
res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
# Using an X-If-Unmodified-Since in the past should cause 412s.
ts = str(float(res.headers['X-Last-Modified']) - 1)
bso = {'id': '12345', 'payload': _PLD + "XXX"}
res = self.retry_put_json(
self.root + '/storage/xxx_col2/12345', bso,
headers=[('X-If-Unmodified-Since', ts)],
status=412)
self.assertTrue("X-Last-Modified" in res.headers)
res = self.retry_delete(
self.root + '/storage/xxx_col2/12345',
headers=[('X-If-Unmodified-Since', ts)],
status=412)
self.assertTrue("X-Last-Modified" in res.headers)
self.retry_post_json(
self.root + '/storage/xxx_col2', [bso],
headers=[('X-If-Unmodified-Since', ts)],
status=412)
self.retry_delete(
self.root + '/storage/xxx_col2?ids=12345',
headers=[('X-If-Unmodified-Since', ts)],
status=412)
self.app.get(
self.root + '/storage/xxx_col2/12345',
headers=[('X-If-Unmodified-Since', ts)],
status=412)
self.app.get(
self.root + '/storage/xxx_col2',
headers=[('X-If-Unmodified-Since', ts)],
status=412)
# Deleting items from a collection should give 412 even if some
# other, unrelated item in the collection has been modified.
ts = res.headers['X-Last-Modified']
res2 = self.retry_put_json(self.root + '/storage/xxx_col2/54321', {
'payload': _PLD,
})
self.retry_delete(
self.root + '/storage/xxx_col2?ids=12345',
headers=[('X-If-Unmodified-Since', ts)],
status=412)
ts = res2.headers['X-Last-Modified']
# All of those should have left the BSO unchanged
res2 = self.app.get(self.root + '/storage/xxx_col2/12345')
self.assertEquals(res2.json['payload'], _PLD)
self.assertEquals(res2.headers['X-Last-Modified'],
res.headers['X-Last-Modified'])
# Using an X-If-Unmodified-Since equal to
# X-Last-Modified should allow the request to succeed.
res = self.retry_post_json(
self.root + '/storage/xxx_col2', [bso],
headers=[('X-If-Unmodified-Since', ts)],
status=200)
ts = res.headers['X-Last-Modified']
self.app.get(
self.root + '/storage/xxx_col2/12345',
headers=[('X-If-Unmodified-Since', ts)],
status=200)
self.retry_delete(
self.root + '/storage/xxx_col2/12345',
headers=[('X-If-Unmodified-Since', ts)],
status=200)
res = self.retry_put_json(
self.root + '/storage/xxx_col2/12345', bso,
headers=[('X-If-Unmodified-Since', '0')],
status=200)
ts = res.headers['X-Last-Modified']
self.app.get(
self.root + '/storage/xxx_col2',
headers=[('X-If-Unmodified-Since', ts)],
status=200)
self.retry_delete(
self.root + '/storage/xxx_col2?ids=12345',
headers=[('X-If-Unmodified-Since', ts)],
status=200)
def test_quota(self):
res = self.app.get(self.root + '/info/quota')
old_used = res.json[0]
bso = {'payload': _PLD}
self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
res = self.app.get(self.root + '/info/quota')
used = res.json[0]
self.assertEquals(used - old_used, len(_PLD) / 1024.0)
def test_overquota(self):
# This can't be run against a live server.
raise unittest2.SkipTest
if self.distant:
raise unittest2.SkipTest
# Clear out any data that's already in the store.
self.retry_delete(self.root + "/storage")
# Set a low quota for the storage.
self.config.registry.settings["storage.quota_size"] = 700
# Check the the remaining quota is correctly reported.
bso = {'payload': _PLD}
res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
wanted = str(round(200 / 1024.0, 2))
self.assertEquals(res.headers['X-Weave-Quota-Remaining'], wanted)
# Set the quota so that they're over their limit.
self.config.registry.settings["storage.quota_size"] = 10
bso = {'payload': _PLD}
res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso,
status=403)
self.assertEquals(res.content_type.split(";")[0], 'application/json')
self.assertEquals(res.json["status"], "quota-exceeded")
def test_get_collection_ttl(self):
bso = {'payload': _PLD, 'ttl': 0}
res = self.retry_put_json(self.root + '/storage/xxx_col2/12345', bso)
time.sleep(1.1)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(res.json, [])
bso = {'payload': _PLD, 'ttl': 2}
res = self.retry_put_json(self.root + '/storage/xxx_col2/123456', bso)
# it should exists now
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 1)
# trying a second put again
self.retry_put_json(self.root + '/storage/xxx_col2/123456', bso)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 1)
time.sleep(2.1)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(len(res.json), 0)
def test_multi_item_post_limits(self):
res = self.app.get(self.root + '/info/configuration')
try:
max_bytes = res.json['max_post_bytes']
max_count = res.json['max_post_records']
max_req_bytes = res.json['max_request_bytes']
except KeyError:
# Can't run against live server if it doesn't
# report the right config options.
if self.distant:
raise unittest2.SkipTest
max_bytes = get_limit_config(self.config, 'max_post_bytes')
max_count = get_limit_config(self.config, 'max_post_records')
max_req_bytes = get_limit_config(self.config, 'max_request_bytes')
# Uploading max_count-5 small objects should succeed.
bsos = [{'id': str(i).zfill(2),
'payload': 'X'} for i in range(max_count - 5)]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = res.json
self.assertEquals(len(res['success']), max_count - 5)
self.assertEquals(len(res['failed']), 0)
# Uploading max_count+5 items should produce five failures.
bsos = [{'id': str(i).zfill(2),
'payload': 'X'} for i in range(max_count + 5)]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = res.json
self.assertEquals(len(res['success']), max_count)
self.assertEquals(len(res['failed']), 5)
# Uploading items such that the last item puts us over the
# cumulative limit on payload size, should produce 1 failure.
# The item_size here is arbitrary, so I made it a prime in kB.
item_size = (227 * 1024)
max_items, leftover = divmod(max_bytes, item_size)
bsos = [{'id': str(i).zfill(2), 'payload': 'X' * item_size}
for i in range(max_items)]
bsos.append({'id': str(max_items), 'payload': 'X' * (leftover + 1)})
# Check that we don't go over the limit on raw request bytes,
# which would get us rejected in production with a 413.
self.assertTrue(len(json.dumps(bsos)) < max_req_bytes)
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = res.json
self.assertEquals(len(res['success']), max_items)
self.assertEquals(len(res['failed']), 1)
def test_weird_args(self):
# pushing some data in xxx_col2
bsos = [{'id': str(i).zfill(2), 'payload': _PLD} for i in range(10)]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = res.json
# trying weird args and make sure the server returns 400s
# Note: "Offset" is a string since the bsoid could be anything.
# skipping that for now.
args = ('newer', 'older', 'limit', 'offset')
for arg in args:
value = randtext()
self.app.get(self.root + '/storage/xxx_col2?%s=%s' % (arg, value),
status=400)
# what about a crazy ids= string ?
ids = ','.join([randtext(10) for i in range(100)])
res = self.app.get(self.root + '/storage/xxx_col2?ids=%s' % ids)
self.assertEquals(res.json, [])
# trying unexpected args - they should not break
self.app.get(self.root + '/storage/xxx_col2?blabla=1',
status=200)
def test_guid_deletion(self):
# pushing some data in xxx_col2
bsos = [{'id': '6820f3ca-6e8a-4ff4-8af7-8b3625d7d65%d' % i,
'payload': _PLD} for i in range(5)]
res = self.retry_post_json(self.root + '/storage/passwords', bsos)
res = res.json
self.assertEquals(len(res["success"]), 5)
# now deleting some of them
ids = ','.join(['6820f3ca-6e8a-4ff4-8af7-8b3625d7d65%d' % i
for i in range(2)])
self.retry_delete(self.root + '/storage/passwords?ids=%s' % ids)
res = self.app.get(self.root + '/storage/passwords?ids=%s' % ids)
self.assertEqual(len(res.json), 0)
res = self.app.get(self.root + '/storage/passwords')
self.assertEqual(len(res.json), 3)
def test_specifying_ids_with_percent_encoded_query_string(self):
# create some items
bsos = [{'id': 'test-%d' % i, 'payload': _PLD} for i in range(5)]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = res.json
self.assertEquals(len(res["success"]), 5)
# now delete some of them
ids = ','.join(['test-%d' % i for i in range(2)])
ids = urllib.request.quote(ids)
self.retry_delete(self.root + '/storage/xxx_col2?ids=%s' % ids)
# check that the correct items were deleted
res = self.app.get(self.root + '/storage/xxx_col2?ids=%s' % ids)
self.assertEqual(len(res.json), 0)
res = self.app.get(self.root + '/storage/xxx_col2')
self.assertEqual(len(res.json), 3)
def test_timestamp_numbers_are_decimals(self):
# Create five items with different timestamps.
for i in range(5):
bsos = [{"id": str(i).zfill(2), "payload": "xxx"}]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
# make sure the server returns only proper precision timestamps.
resp = self.app.get(self.root + '/storage/xxx_col2?full=1')
bsos = json_loads(resp.body)
timestamps = []
for bso in bsos:
ts = bso['modified']
# timestamps could be on the hundred seconds (.10) or on the
# second (.0) and the zero could be dropped. We just don't want
# anything beyond milisecond.
self.assertLessEqual(len(str(ts).split(".")[-1]), 2)
timestamps.append(ts)
timestamps.sort()
# try a newer filter now, to get the last two objects
ts = float(timestamps[-3])
# Returns only ids for objects in the collection that have been
# last modified since the timestamp given.
res = self.app.get(self.root + '/storage/xxx_col2?newer=%s' % ts)
res = res.json
try:
self.assertEquals(sorted(res), ['03', '04'])
except AssertionError:
# need to display the whole collection to understand the issue
msg = 'Timestamp used: %s' % ts
msg += ' ' + self.app.get(self.root +
'/storage/xxx_col2?full=1').body
msg += ' Timestamps received: %s' % str(timestamps)
msg += ' Result of newer query: %s' % res
raise AssertionError(msg)
def test_strict_newer(self):
# send two bsos in the 'meh' collection
bso1 = {'id': '01', 'payload': _PLD}
bso2 = {'id': '02', 'payload': _PLD}
bsos = [bso1, bso2]
res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos)
ts = float(res.headers["X-Last-Modified"])
# send two more bsos
bso3 = {'id': '03', 'payload': _PLD}
bso4 = {'id': '04', 'payload': _PLD}
bsos = [bso3, bso4]
res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos)
# asking for bsos using newer=ts where newer is the timestamp
# of bso 1 and 2, should not return them
res = self.app.get(self.root + '/storage/xxx_meh?newer=%s' % ts)
res = res.json
self.assertEquals(sorted(res), ['03', '04'])
def test_strict_older(self):
# send two bsos in the 'xxx_meh' collection
bso1 = {'id': '01', 'payload': _PLD}
bso2 = {'id': '02', 'payload': _PLD}
bsos = [bso1, bso2]
res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos)
# send two more bsos
bso3 = {'id': '03', 'payload': _PLD}
bso4 = {'id': '04', 'payload': _PLD}
bsos = [bso3, bso4]
res = self.retry_post_json(self.root + '/storage/xxx_meh', bsos)
ts = float(res.headers["X-Last-Modified"])
# asking for bsos using older=ts where older is the timestamp
# of bso 3 and 4, should not return them
res = self.app.get(self.root + '/storage/xxx_meh?older=%s' % ts)
res = res.json
self.assertEquals(sorted(res), ['01', '02'])
def test_handling_of_invalid_json_in_bso_uploads(self):
# Single upload with JSON that's not a BSO.
bso = "notabso"
res = self.retry_put_json(self.root + '/storage/xxx_col2/invalid', bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
bso = 42
res = self.retry_put_json(self.root + '/storage/xxx_col2/invalid', bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
bso = {'id': ["01", "02"], 'payload': {'3': '4'}}
res = self.retry_put_json(self.root + '/storage/xxx_col2/invalid', bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Batch upload with JSON that's not a list of BSOs
bsos = "notalist"
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
bsos = 42
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Batch upload a list with something that's not a valid data dict.
# It should fail out entirely, as the input is seriously broken.
bsos = [{'id': '01', 'payload': 'GOOD'}, "BAD"]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos,
status=400)
# Batch upload a list with something that's an invalid BSO.
# It should process the good entry and fail for the bad.
bsos = [{'id': '01', 'payload': 'GOOD'}, {'id': '02', 'invalid': 'ya'}]
res = self.retry_post_json(self.root + '/storage/xxx_col2', bsos)
res = res.json
self.assertEquals(len(res['success']), 1)
self.assertEquals(len(res['failed']), 1)
def test_handling_of_invalid_bso_fields(self):
coll_url = self.root + "/storage/xxx_col2"
# Invalid ID - unacceptable characters.
# The newline cases are especially nuanced because \n
# gets special treatment from the regex library.
bso = {"id": "A\nB", "payload": "testing"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
bso = {"id": "A\n", "payload": "testing"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
bso = {"id": "\nN", "payload": "testing"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
bso = {"id": "A\tB", "payload": "testing"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
# Invalid ID - empty string is not acceptable.
bso = {"id": "", "payload": "testing"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
# Invalid ID - too long
bso = {"id": "X" * 65, "payload": "testing"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
# Commenting out this test.
# This uses the same invalid BSO from above, which should return a 400
"""
res = self.retry_put_json(coll_url + "/" + bso["id"], bso, status=404)
"""
# Invalid sortindex - not an integer
bso = {"id": "TEST", "payload": "testing", "sortindex": "xxx_meh"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Invalid sortindex - not an integer
bso = {"id": "TEST", "payload": "testing", "sortindex": "2.6"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Invalid sortindex - larger than max value
bso = {"id": "TEST", "payload": "testing", "sortindex": "1" + "0" * 9}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Invalid payload - not a string
bso = {"id": "TEST", "payload": 42}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Invalid ttl - not an integer
bso = {"id": "TEST", "payload": "testing", "ttl": "eh?"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Invalid ttl - not an integer
bso = {"id": "TEST", "payload": "testing", "ttl": "4.2"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso,
status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
# Invalid BSO - unknown field
bso = {"id": "TEST", "unexpected": "spanish-inquisition"}
res = self.retry_post_json(coll_url, [bso])
self.assertTrue(res.json["failed"] and not res.json["success"])
res = self.retry_put_json(coll_url + "/" + bso["id"], bso, status=400)
self.assertEquals(res.json, WEAVE_INVALID_WBO)
def test_that_batch_gets_are_limited_to_max_number_of_ids(self):
bso = {"id": "01", "payload": "testing"}
self.retry_put_json(self.root + "/storage/xxx_col2/01", bso)
# Getting with less than the limit works OK.
ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS - 1))
res = self.app.get(self.root + "/storage/xxx_col2?ids=" + ids)
self.assertEquals(res.json, ["01"])
# Getting with equal to the limit works OK.
ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS))
res = self.app.get(self.root + "/storage/xxx_col2?ids=" + ids)
self.assertEquals(res.json, ["01"])
# Getting with more than the limit fails.
ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS + 1))
self.app.get(self.root + "/storage/xxx_col2?ids=" + ids, status=400)
def test_that_batch_deletes_are_limited_to_max_number_of_ids(self):
bso = {"id": "01", "payload": "testing"}
# Deleting with less than the limit works OK.
self.retry_put_json(self.root + "/storage/xxx_col2/01", bso)
ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS - 1))
self.retry_delete(self.root + "/storage/xxx_col2?ids=" + ids)
# Deleting with equal to the limit works OK.
self.retry_put_json(self.root + "/storage/xxx_col2/01", bso)
ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS))
self.retry_delete(self.root + "/storage/xxx_col2?ids=" + ids)
# Deleting with more than the limit fails.
self.retry_put_json(self.root + "/storage/xxx_col2/01", bso)
ids = ",".join(str(i).zfill(2) for i in range(BATCH_MAX_IDS + 1))
self.retry_delete(self.root + "/storage/xxx_col2?ids=" + ids,
status=400)
def test_that_expired_items_can_be_overwritten_via_PUT(self):
# Upload something with a small ttl.
bso = {"payload": "XYZ", "ttl": 0}
self.retry_put_json(self.root + "/storage/xxx_col2/TEST", bso)
# Wait for it to expire.
time.sleep(0.02)
self.app.get(self.root + "/storage/xxx_col2/TEST", status=404)
# Overwriting it should still work.
bso = {"payload": "XYZ", "ttl": 42}
self.retry_put_json(self.root + "/storage/xxx_col2/TEST", bso)
def test_if_modified_since_on_info_views(self):
# Store something, so the views have a modified time > 0.
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)]
self.retry_post_json(self.root + "/storage/xxx_col1", bsos)
INFO_VIEWS = ("/info/collections", "/info/quota",
"/info/collection_usage", "/info/collection_counts")
# Get the initial last-modified version.
r = self.app.get(self.root + "/info/collections")
ts1 = float(r.headers["X-Last-Modified"])
self.assertTrue(ts1 > 0)
# With X-I-M-S set before latest change, all should give a 200.
headers = {"X-If-Modified-Since": str(ts1 - 1)}
for view in INFO_VIEWS:
self.app.get(self.root + view, headers=headers, status=200)
# With X-I-M-S set to after latest change , all should give a 304.
headers = {"X-If-Modified-Since": str(ts1)}
for view in INFO_VIEWS:
self.app.get(self.root + view, headers=headers, status=304)
# Change a collection.
bso = {"payload": "TEST"}
r = self.retry_put_json(self.root + "/storage/xxx_col2/TEST", bso)
ts2 = r.headers["X-Last-Modified"]
# Using the previous version should read the updated data.
headers = {"X-If-Modified-Since": str(ts1)}
for view in INFO_VIEWS:
self.app.get(self.root + view, headers=headers, status=200)
# Using the new timestamp should produce 304s.
headers = {"X-If-Modified-Since": str(ts2)}
for view in INFO_VIEWS:
self.app.get(self.root + view, headers=headers, status=304)
# XXX TODO: the storage-level timestamp is not tracked correctly
# after deleting a collection, so this test fails for now.
# # Delete a collection.
# r = self.retry_delete(self.root + "/storage/xxx_col2")
# ts3 = r.headers["X-Last-Modified"]
# # Using the previous timestamp should read the updated data.
# headers = {"X-If-Modified-Since": str(ts2)}
# for view in INFO_VIEWS:
# self.app.get(self.root + view, headers=headers, status=200)
# # Using the new timestamp should produce 304s.
# headers = {"X-If-Modified-Since": str(ts3)}
# for view in INFO_VIEWS:
# self.app.get(self.root + view, headers=headers, status=304)
def test_that_x_last_modified_is_sent_for_all_get_requests(self):
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(5)]
self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
r = self.app.get(self.root + "/info/collections")
self.assertTrue("X-Last-Modified" in r.headers)
r = self.app.get(self.root + "/info/collection_counts")
self.assertTrue("X-Last-Modified" in r.headers)
r = self.app.get(self.root + "/storage/xxx_col2")
self.assertTrue("X-Last-Modified" in r.headers)
r = self.app.get(self.root + "/storage/xxx_col2/01")
self.assertTrue("X-Last-Modified" in r.headers)
def test_update_of_ttl_without_sending_data(self):
bso = {"payload": "x", "ttl": 1}
self.retry_put_json(self.root + "/storage/xxx_col2/TEST1", bso)
self.retry_put_json(self.root + "/storage/xxx_col2/TEST2", bso)
# Before those expire, update ttl on one that exists
# and on one that does not.
time.sleep(0.2)
bso = {"ttl": 10}
self.retry_put_json(self.root + "/storage/xxx_col2/TEST2", bso)
self.retry_put_json(self.root + "/storage/xxx_col2/TEST3", bso)
# Update some other field on TEST1, which should leave ttl untouched.
bso = {"sortindex": 3}
self.retry_put_json(self.root + "/storage/xxx_col2/TEST1", bso)
# If we wait, TEST1 should expire but the others should not.
time.sleep(0.8)
items = self.app.get(self.root + "/storage/xxx_col2?full=1").json
items = dict((item["id"], item) for item in items)
self.assertEquals(sorted(list(items.keys())), ["TEST2", "TEST3"])
# The existing item should have retained its payload.
# The new item should have got a default payload of empty string.
self.assertEquals(items["TEST2"]["payload"], "x")
self.assertEquals(items["TEST3"]["payload"], "")
ts2 = items["TEST2"]["modified"]
ts3 = items["TEST3"]["modified"]
self.assertTrue(ts2 < ts3)
def test_bulk_update_of_ttls_without_sending_data(self):
# Create 5 BSOs with a ttl of 1 second.
bsos = [{"id": str(i).zfill(2),
"payload": "x", "ttl": 1} for i in range(5)]
r = self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
ts1 = float(r.headers["X-Last-Modified"])
# Before they expire, bulk-update the ttl to something longer.
# Also send data for some that don't exist yet.
# And just to be really tricky, we're also going to update
# one of the payloads at the same time.
time.sleep(0.2)
bsos = [{"id": str(i).zfill(2), "ttl": 10} for i in range(3, 7)]
bsos[0]["payload"] = "xx"
r = self.retry_post_json(self.root + "/storage/xxx_col2", bsos)
self.assertEquals(len(r.json["success"]), 4)
ts2 = float(r.headers["X-Last-Modified"])
# If we wait then items 0, 1, 2 should have expired.
# Items 3, 4, 5, 6 should still exist.
time.sleep(0.8)
items = self.app.get(self.root + "/storage/xxx_col2?full=1").json
items = dict((item["id"], item) for item in items)
self.assertEquals(sorted(list(items.keys())), ["03", "04", "05", "06"])
# Items 3 and 4 should have the specified payloads.
# Items 5 and 6 should have payload defaulted to empty string.
self.assertEquals(items["03"]["payload"], "xx")
self.assertEquals(items["04"]["payload"], "x")
self.assertEquals(items["05"]["payload"], "")
self.assertEquals(items["06"]["payload"], "")
# All items created or modified by the request should get their
# timestamps update. Just bumping the ttl should not bump timestamp.
self.assertEquals(items["03"]["modified"], ts2)
self.assertEquals(items["04"]["modified"], ts1)
self.assertEquals(items["05"]["modified"], ts2)
self.assertEquals(items["06"]["modified"], ts2)
def test_that_negative_integer_fields_are_not_accepted(self):
# ttls cannot be negative
self.retry_put_json(self.root + "/storage/xxx_col2/TEST", {
"payload": "TEST",
"ttl": -1,
}, status=400)
# limit cannot be negative
self.retry_put_json(self.root + "/storage/xxx_col2/TEST",
{"payload": "X"})
self.app.get(self.root + "/storage/xxx_col2?limit=-1", status=400)
# X-If-Modified-Since cannot be negative
self.app.get(self.root + "/storage/xxx_col2", headers={
"X-If-Modified-Since": "-3",
}, status=400)
# X-If-Unmodified-Since cannot be negative
self.retry_put_json(self.root + "/storage/xxx_col2/TEST", {
"payload": "TEST",
}, headers={
"X-If-Unmodified-Since": "-3",
}, status=400)
# sortindex actually *can* be negative
self.retry_put_json(self.root + "/storage/xxx_col2/TEST", {
"payload": "TEST",
"sortindex": -42,
}, status=200)
def test_meta_global_sanity(self):
# Memcache backend is configured to store 'meta' in write-through
# cache, so we want to check it explicitly. We might as well put it
# in the base tests because there's nothing memcached-specific here.
self.app.get(self.root + '/storage/meta/global', status=404)
res = self.app.get(self.root + '/storage/meta')
self.assertEquals(res.json, [])
self.retry_put_json(self.root + '/storage/meta/global',
{'payload': 'blob'})
res = self.app.get(self.root + '/storage/meta')
self.assertEquals(res.json, ['global'])
res = self.app.get(self.root + '/storage/meta/global')
self.assertEquals(res.json['payload'], 'blob')
# It should not have extra keys.
keys = list(res.json.keys())
keys.sort()
self.assertEquals(keys, ['id', 'modified', 'payload'])
# It should have a properly-formatted "modified" field.
modified_re = r"['\"]modified['\"]:\s*[0-9]+\.[0-9][0-9]\s*[,}]"
self.assertTrue(re.search(modified_re, res.body.decode('utf-8')))
# Any client-specified "modified" field should be ignored
res = self.retry_put_json(self.root + '/storage/meta/global',
{'payload': 'blob', 'modified': 12})
ts = float(res.headers['X-Weave-Timestamp'])
res = self.app.get(self.root + '/storage/meta/global')
self.assertEquals(res.json['modified'], ts)
def test_that_404_responses_have_a_json_body(self):
res = self.app.get(self.root + '/nonexistent/url', status=404)
self.assertEquals(res.content_type, "application/json")
self.assertEquals(res.json, 0)
def test_that_internal_server_fields_are_not_echoed(self):
self.retry_post_json(self.root + '/storage/xxx_col1',
[{'id': 'one', 'payload': 'blob'}])
self.retry_put_json(self.root + '/storage/xxx_col1/two',
{'payload': 'blub'})
res = self.app.get(self.root + '/storage/xxx_col1?full=1')
self.assertEquals(len(res.json), 2)
for item in res.json:
self.assertTrue("id" in item)
self.assertTrue("payload" in item)
self.assertFalse("payload_size" in item)
self.assertFalse("ttl" in item)
for id in ('one', 'two'):
res = self.app.get(self.root + '/storage/xxx_col1/' + id)
self.assertTrue("id" in res.json)
self.assertTrue("payload" in res.json)
self.assertFalse("payload_size" in res.json)
self.assertFalse("ttl" in res.json)
def test_accessing_info_collections_with_an_expired_token(self):
# This can't be run against a live server because we
# have to forge an auth token to test things properly.
if self.distant:
raise unittest2.SkipTest
# Write some items while we've got a good token.
bsos = [{"id": str(i).zfill(2), "payload": "xxx"} for i in range(3)]
resp = self.retry_post_json(self.root + "/storage/xxx_col1", bsos)
ts = float(resp.headers["X-Last-Modified"])
# Check that we can read the info correctly.
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(list(resp.json.keys()), ["xxx_col1"])
self.assertEquals(resp.json["xxx_col1"], ts)
# Forge an expired token to use for the test.
auth_policy = self.config.registry.getUtility(IAuthenticationPolicy)
secret = auth_policy._get_token_secrets(self.host_url)[-1]
tm = tokenlib.TokenManager(secret=secret)
exp = time.time() - 60
data = {"uid": self.user_id, "node": self.host_url, "expires": exp}
self.auth_token = tm.make_token(data)
self.auth_secret = tm.get_derived_secret(self.auth_token)
# The expired token cannot be used for normal operations.
bsos = [{"id": str(i).zfill(2), "payload": "aaa"} for i in range(3)]
self.retry_post_json(self.root + "/storage/xxx_col1", bsos, status=401)
self.app.get(self.root + "/storage/xxx_col1", status=401)
# But it still allows access to /info/collections.
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(list(resp.json.keys()), ["xxx_col1"])
self.assertEquals(resp.json["xxx_col1"], ts)
def test_pagination_with_newer_and_sort_by_oldest(self):
# Twelve bsos with three different modification times.
NUM_ITEMS = 12
bsos = []
timestamps = []
for i in range(NUM_ITEMS):
bso = {'id': str(i).zfill(2), 'payload': 'x'}
bsos.append(bso)
if i % 4 == 3:
res = self.retry_post_json(self.root + '/storage/xxx_col2',
bsos)
ts = float(res.headers["X-Last-Modified"])
timestamps.append((i, ts))
bsos = []
# Try with several different pagination sizes,
# to hit various boundary conditions.
for limit in (2, 3, 4, 5, 6):
for (start, ts) in timestamps:
query_url = self.root + \
'/storage/xxx_col2?full=true&sort=oldest'
query_url += '&newer=%s&limit=%s' % (ts, limit)
# Paginated-ly fetch all items.
items = []
res = self.app.get(query_url)
for item in res.json:
if items:
assert items[-1]['modified'] <= item['modified']
items.append(item)
next_offset = res.headers.get('X-Weave-Next-Offset')
while next_offset is not None:
res = self.app.get(query_url + "&offset=" + next_offset)
for item in res.json:
assert items[-1]['modified'] <= item['modified']
items.append(item)
next_offset = res.headers.get('X-Weave-Next-Offset')
# They should all be in order, starting from the item
# *after* the one that was used for the newer= timestamp.
self.assertEquals(sorted(int(item['id']) for item in items),
list(range(start + 1, NUM_ITEMS)))
def test_pagination_with_older_and_sort_by_newest(self):
# Twelve bsos with three different modification times.
NUM_ITEMS = 12
bsos = []
timestamps = []
for i in range(NUM_ITEMS):
bso = {'id': str(i).zfill(2), 'payload': 'x'}
bsos.append(bso)
if i % 4 == 3:
res = self.retry_post_json(self.root + '/storage/xxx_col2',
bsos)
ts = float(res.headers["X-Last-Modified"])
timestamps.append((i - 3, ts))
bsos = []
# Try with several different pagination sizes,
# to hit various boundary conditions.
for limit in (2, 3, 4, 5, 6):
for (start, ts) in timestamps:
query_url = self.root + \
'/storage/xxx_col2?full=true&sort=newest'
query_url += '&older=%s&limit=%s' % (ts, limit)
# Paginated-ly fetch all items.
items = []
res = self.app.get(query_url)
for item in res.json:
if items:
assert items[-1]['modified'] >= item['modified']
items.append(item)
next_offset = res.headers.get('X-Weave-Next-Offset')
while next_offset is not None:
res = self.app.get(query_url + "&offset=" + next_offset)
for item in res.json:
assert items[-1]['modified'] >= item['modified']
items.append(item)
next_offset = res.headers.get('X-Weave-Next-Offset')
# They should all be in order, up to the item *before*
# the one that was used for the older= timestamp.
self.assertEquals(sorted(int(item['id']) for item in items),
list(range(0, start)))
def assertCloseEnough(self, val1, val2, delta=0.05):
if abs(val1 - val2) < delta:
return True
raise AssertionError("abs(%.2f - %.2f) = %.2f > %.2f"
% (val1, val2, abs(val1 - val2), delta))
def test_batches(self):
endpoint = self.root + '/storage/xxx_col2'
bso1 = {'id': '12', 'payload': 'elegance'}
bso2 = {'id': '13', 'payload': 'slovenly'}
bsos = [bso1, bso2]
self.retry_post_json(endpoint, bsos)
resp = self.app.get(endpoint + '/12')
orig_modified = resp.headers['X-Last-Modified']
bso3 = {'id': 'a', 'payload': 'internal'}
bso4 = {'id': 'b', 'payload': 'pancreas'}
resp = self.retry_post_json(endpoint + '?batch=true', [bso3, bso4])
batch = resp.json["batch"]
# The collection should not be reported as modified.
self.assertEquals(orig_modified, resp.headers['X-Last-Modified'])
# And reading from it shouldn't show the new records yet.
resp = self.app.get(endpoint)
res = resp.json
res.sort()
self.assertEquals(res, ['12', '13'])
self.assertEquals(int(resp.headers['X-Weave-Records']), 2)
self.assertEquals(orig_modified, resp.headers['X-Last-Modified'])
bso5 = {'id': 'c', 'payload': 'tinsel'}
bso6 = {'id': '13', 'payload': 'portnoy'}
bso0 = {'id': '14', 'payload': 'itsybitsy'}
commit = '?batch={0}&commit=true'.format(batch)
resp = self.retry_post_json(endpoint + commit, [bso5, bso6, bso0])
committed = resp.json['modified']
self.assertEquals(resp.json['modified'],
float(resp.headers['X-Last-Modified']))
# make sure /info/collections got updated
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(float(resp.headers['X-Last-Modified']), committed)
self.assertEquals(resp.json['xxx_col2'], committed)
# make sure the changes applied
resp = self.app.get(endpoint)
res = resp.json
res.sort()
self.assertEquals(res, ['12', '13', '14', 'a', 'b', 'c'])
self.assertEquals(int(resp.headers['X-Weave-Records']), 6)
resp = self.app.get(endpoint + '/13')
self.assertEquals(resp.json['payload'], 'portnoy')
self.assertEquals(committed, float(resp.headers['X-Last-Modified']))
self.assertEquals(committed, resp.json['modified'])
resp = self.app.get(endpoint + '/c')
self.assertEquals(resp.json['payload'], 'tinsel')
self.assertEquals(committed, resp.json['modified'])
resp = self.app.get(endpoint + '/14')
self.assertEquals(resp.json['payload'], 'itsybitsy')
self.assertEquals(committed, resp.json['modified'])
# empty commit POST
bso7 = {'id': 'a', 'payload': 'burrito'}
bso8 = {'id': 'e', 'payload': 'chocolate'}
resp = self.retry_post_json(endpoint + '?batch=true', [bso7, bso8])
batch = resp.json["batch"]
time.sleep(1)
commit = '?batch={0}&commit=true'.format(batch)
resp1 = self.retry_post_json(endpoint + commit, [])
committed = resp1.json['modified']
self.assertEquals(committed, float(resp1.headers['X-Last-Modified']))
resp2 = self.app.get(endpoint + '/a')
self.assertEquals(committed, float(resp2.headers['X-Last-Modified']))
self.assertEquals(committed, resp2.json['modified'])
self.assertEquals(resp2.json['payload'], 'burrito')
resp3 = self.app.get(endpoint + '/e')
self.assertEquals(committed, resp3.json['modified'])
def test_we_dont_need_no_stinkin_batches(self):
endpoint = self.root + '/storage/xxx_col2'
# invalid batch ID
bso1 = {'id': 'f', 'payload': 'pantomime'}
self.retry_post_json(endpoint + '?batch=sammich', [bso1], status=400)
# commit with no batch ID
self.retry_post_json(endpoint + '?commit=true', [], status=400)
def test_batch_size_limits(self):
limits = self.app.get(self.root + '/info/configuration').json
self.assertTrue('max_post_records' in limits)
self.assertTrue('max_post_bytes' in limits)
self.assertTrue('max_total_records' in limits)
self.assertTrue('max_total_bytes' in limits)
self.assertTrue('max_record_payload_bytes' in limits)
self.assertTrue('max_request_bytes' in limits)
endpoint = self.root + '/storage/xxx_col2?batch=true'
# There are certain obvious constraints on these limits,
# violations of which would be very confusing for clients.
#
# self.assertTrue(
# limits['max_request_bytes'] > limits['max_post_bytes']
# )
# self.assertTrue(
# limits['max_post_bytes'] >= limits['max_record_payload_bytes']
# )
# self.assertTrue(
# limits['max_total_records'] >= limits['max_post_records']
# )
# self.assertTrue(
# limits['max_total_bytes'] >= limits['max_post_bytes']
# )
#
# # `max_post_records` is an (inclusive) limit on
# # the number of items in a single post.
#
# res = self.retry_post_json(endpoint, [], headers={
# 'X-Weave-Records': str(limits['max_post_records'])
# })
# self.assertFalse(res.json['failed'])
# res = self.retry_post_json(endpoint, [], headers={
# 'X-Weave-Records': str(limits['max_post_records'] + 1)
# }, status=400)
# self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED)
#
# bsos = [{'id': str(x), 'payload': ''}
# for x in range(limits['max_post_records'])]
# res = self.retry_post_json(endpoint, bsos)
# self.assertFalse(res.json['failed'])
# bsos.append({'id': 'toomany', 'payload': ''})
# res = self.retry_post_json(endpoint, bsos)
# self.assertEquals(res.json['failed']['toomany'], 'retry bso')
#
# # `max_total_records` is an (inclusive) limit on the
# # total number of items in a batch. We can only enforce
# # it if the client tells us this via header.
#
# self.retry_post_json(endpoint, [], headers={
# 'X-Weave-Total-Records': str(limits['max_total_records'])
# })
# res = self.retry_post_json(endpoint, [], headers={
# 'X-Weave-Total-Records': str(limits['max_total_records'] + 1)
# }, status=400)
# self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED)
#
# # `max_post_bytes` is an (inclusive) limit on the
# # total size of payloads in a single post.
#
# self.retry_post_json(endpoint, [], headers={
# 'X-Weave-Bytes': str(limits['max_post_bytes'])
# })
# res = self.retry_post_json(endpoint, [], headers={
# 'X-Weave-Bytes': str(limits['max_post_bytes'] + 1)
# }, status=400)
# self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED)
bsos = [
{'id': 'little', 'payload': 'XXX'},
{'id': 'big', 'payload': 'X' * (limits['max_post_bytes'] - 3)}
]
res = self.retry_post_json(endpoint, bsos)
self.assertFalse(res.json['failed'])
bsos[1]['payload'] += 'X'
res = self.retry_post_json(endpoint, bsos)
self.assertEqual(res.json['success'], ['little'])
self.assertEqual(res.json['failed']['big'], 'retry bytes')
# `max_total_bytes` is an (inclusive) limit on the
# total size of all payloads in a batch. We can only enforce
# it if the client tells us this via header.
self.retry_post_json(endpoint, [], headers={
'X-Weave-Total-Bytes': str(limits['max_total_bytes'])
})
res = self.retry_post_json(endpoint, [], headers={
'X-Weave-Total-Bytes': str(limits['max_total_bytes'] + 1)
}, status=400)
self.assertEquals(res.json, WEAVE_SIZE_LIMIT_EXCEEDED)
def test_batch_partial_update(self):
collection = self.root + '/storage/xxx_col2'
bsos = [
{'id': 'a', 'payload': 'aai'},
{'id': 'b', 'payload': 'bee', 'sortindex': 17}
]
resp = self.retry_post_json(collection, bsos)
orig_ts = float(resp.headers['X-Last-Modified'])
# Update one, and add a new one.
bsos = [
{'id': 'b', 'payload': 'bii'},
{'id': 'c', 'payload': 'sea'},
]
resp = self.retry_post_json(collection + '?batch=true', bsos)
batch = resp.json["batch"]
self.assertEquals(orig_ts, float(resp.headers['X-Last-Modified']))
# The updated item hasn't been written yet.
resp = self.app.get(collection + '?full=1')
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 2)
self.assertEquals(res[0]['payload'], 'aai')
self.assertEquals(res[1]['payload'], 'bee')
self.assertEquals(res[0]['modified'], orig_ts)
self.assertEquals(res[1]['modified'], orig_ts)
self.assertEquals(res[1]['sortindex'], 17)
endpoint = collection + '?batch={0}&commit=true'.format(batch)
resp = self.retry_post_json(endpoint, [])
commit_ts = float(resp.headers['X-Last-Modified'])
# The changes have now been applied.
resp = self.app.get(collection + '?full=1')
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 3)
self.assertEquals(res[0]['payload'], 'aai')
self.assertEquals(res[1]['payload'], 'bii')
self.assertEquals(res[2]['payload'], 'sea')
self.assertEquals(res[0]['modified'], orig_ts)
self.assertEquals(res[1]['modified'], commit_ts)
self.assertEquals(res[2]['modified'], commit_ts)
# Fields not touched by the batch, should have been preserved.
self.assertEquals(res[1]['sortindex'], 17)
def test_batch_ttl_update(self):
collection = self.root + '/storage/xxx_col2'
bsos = [
{'id': 'a', 'payload': 'ayy'},
{'id': 'b', 'payload': 'bea'},
{'id': 'c', 'payload': 'see'}
]
resp = self.retry_post_json(collection, bsos)
# Bump ttls as a series of individual batch operations.
resp = self.retry_post_json(collection + '?batch=true', [],
status=202)
orig_ts = float(resp.headers['X-Last-Modified'])
batch = resp.json["batch"]
endpoint = collection + '?batch={0}'.format(batch)
resp = self.retry_post_json(endpoint, [{'id': 'a', 'ttl': 2}],
status=202)
self.assertEquals(orig_ts, float(resp.headers['X-Last-Modified']))
resp = self.retry_post_json(endpoint, [{'id': 'b', 'ttl': 2}],
status=202)
self.assertEquals(orig_ts, float(resp.headers['X-Last-Modified']))
resp = self.retry_post_json(endpoint + '&commit=true', [], status=200)
# The payloads should be unchanged
resp = self.app.get(collection + '?full=1')
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 3)
self.assertEquals(res[0]['payload'], 'ayy')
self.assertEquals(res[1]['payload'], 'bea')
self.assertEquals(res[2]['payload'], 'see')
# If we wait, the ttls should kick in
time.sleep(2.1)
resp = self.app.get(collection + '?full=1')
res = resp.json
self.assertEquals(len(res), 1)
self.assertEquals(res[0]['payload'], 'see')
def test_batch_ttl_is_based_on_commit_timestamp(self):
collection = self.root + '/storage/xxx_col2'
resp = self.retry_post_json(collection + '?batch=true', [], status=202)
batch = resp.json["batch"]
endpoint = collection + '?batch={0}'.format(batch)
resp = self.retry_post_json(endpoint, [{'id': 'a', 'ttl': 3}],
status=202)
# Put some time between upload timestamp and commit timestamp.
time.sleep(1.5)
resp = self.retry_post_json(endpoint + '&commit=true', [],
status=200)
# Wait a little; if ttl is taken from the time of the commit
# then it should not kick in just yet.
time.sleep(1.6)
resp = self.app.get(collection)
res = resp.json
self.assertEquals(len(res), 1)
self.assertEquals(res[0], 'a')
# Wait some more, and the ttl should kick in.
time.sleep(1.6)
resp = self.app.get(collection)
res = resp.json
self.assertEquals(len(res), 0)
def test_batch_with_immediate_commit(self):
collection = self.root + '/storage/xxx_col2'
bsos = [
{'id': 'a', 'payload': 'aih'},
{'id': 'b', 'payload': 'bie'},
{'id': 'c', 'payload': 'cee'}
]
resp = self.retry_post_json(collection + '?batch=true&commit=true',
bsos, status=200)
self.assertTrue('batch' not in resp.json)
self.assertTrue('modified' in resp.json)
committed = resp.json['modified']
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(float(resp.headers['X-Last-Modified']), committed)
self.assertEquals(resp.json['xxx_col2'], committed)
resp = self.app.get(collection + '?full=1')
self.assertEquals(float(resp.headers['X-Last-Modified']), committed)
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 3)
self.assertEquals(res[0]['payload'], 'aih')
self.assertEquals(res[1]['payload'], 'bie')
self.assertEquals(res[2]['payload'], 'cee')
def test_batch_uploads_properly_update_info_collections(self):
collection1 = self.root + '/storage/xxx_col1'
collection2 = self.root + '/storage/xxx_col2'
bsos = [
{'id': 'a', 'payload': 'aih'},
{'id': 'b', 'payload': 'bie'},
{'id': 'c', 'payload': 'cee'}
]
resp = self.retry_post_json(collection1, bsos)
ts1 = resp.json['modified']
resp = self.retry_post_json(collection2, bsos)
ts2 = resp.json['modified']
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(float(resp.headers['X-Last-Modified']), ts2)
self.assertEquals(resp.json['xxx_col1'], ts1)
self.assertEquals(resp.json['xxx_col2'], ts2)
# Overwrite in place, timestamp should change.
resp = self.retry_post_json(collection2 + '?batch=true&commit=true',
bsos[:2])
self.assertTrue(resp.json['modified'] > ts2)
ts2 = resp.json['modified']
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(float(resp.headers['X-Last-Modified']), ts2)
self.assertEquals(resp.json['xxx_col1'], ts1)
self.assertEquals(resp.json['xxx_col2'], ts2)
# Add new items, timestamp should change
resp = self.retry_post_json(collection1 + '?batch=true&commit=true',
[{'id': 'd', 'payload': 'dee'}])
self.assertTrue(resp.json['modified'] > ts1)
self.assertTrue(resp.json['modified'] >= ts2)
ts1 = resp.json['modified']
resp = self.app.get(self.root + '/info/collections')
self.assertEquals(float(resp.headers['X-Last-Modified']), ts1)
self.assertEquals(resp.json['xxx_col1'], ts1)
self.assertEquals(resp.json['xxx_col2'], ts2)
def test_batch_with_failing_bsos(self):
collection = self.root + '/storage/xxx_col2'
bsos = [
{'id': 'a', 'payload': 'aai'},
{'id': 'b\n', 'payload': 'i am invalid', 'sortindex': 17}
]
resp = self.retry_post_json(collection + '?batch=true', bsos)
self.assertEqual(len(resp.json['failed']), 1)
self.assertEqual(len(resp.json['success']), 1)
batch = resp.json["batch"]
bsos = [
{'id': 'c', 'payload': 'sea'},
{'id': 'd', 'payload': 'dii', 'ttl': -12},
]
endpoint = collection + '?batch={0}&commit=true'.format(batch)
resp = self.retry_post_json(endpoint, bsos)
self.assertEqual(len(resp.json['failed']), 1)
self.assertEqual(len(resp.json['success']), 1)
# To correctly match semantics of batchless POST, the batch
# should be committed including only the successful items.
# It is the client's responsibility to detect that some items
# failed, and decide whether to commit the batch.
resp = self.app.get(collection + '?full=1')
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 2)
self.assertEquals(res[0]['payload'], 'aai')
self.assertEquals(res[1]['payload'], 'sea')
def test_batch_id_is_correctly_scoped_to_a_collection(self):
collection1 = self.root + '/storage/xxx_col1'
bsos = [
{'id': 'a', 'payload': 'aih'},
{'id': 'b', 'payload': 'bie'},
{'id': 'c', 'payload': 'cee'}
]
resp = self.retry_post_json(collection1 + '?batch=true', bsos)
batch = resp.json['batch']
# I should not be able to add to that batch in a different collection.
endpoint2 = self.root + '/storage/xxx_col2?batch={0}'.format(batch)
resp = self.retry_post_json(
endpoint2,
[{'id': 'd', 'payload': 'dii'}],
status=400)
# I should not be able to commit that batch in a different collection.
resp = self.retry_post_json(endpoint2 + '&commit=true', [], status=400)
# I should still be able to use the batch in the correct collection.
endpoint1 = collection1 + '?batch={0}'.format(batch)
resp = self.retry_post_json(endpoint1,
[{'id': 'd', 'payload': 'dii'}])
resp = self.retry_post_json(endpoint1 + '&commit=true', [])
resp = self.app.get(collection1 + '?full=1')
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 4)
self.assertEquals(res[0]['payload'], 'aih')
self.assertEquals(res[1]['payload'], 'bie')
self.assertEquals(res[2]['payload'], 'cee')
self.assertEquals(res[3]['payload'], 'dii')
def test_users_with_the_same_batch_id_get_separate_data(self):
# Try to generate two users with the same batch-id.
# It might take a couple of attempts...
for _ in range(100):
bsos = [{'id': 'a', 'payload': 'aih'}]
req = '/storage/xxx_col1?batch=true'
resp = self.retry_post_json(self.root + req, bsos)
batch1 = resp.json['batch']
with self._switch_user():
bsos = [{'id': 'b', 'payload': 'bee'}]
req = '/storage/xxx_col1?batch=true'
resp = self.retry_post_json(self.root + req, bsos)
batch2 = resp.json['batch']
# Let the second user commit their batch.
req = '/storage/xxx_col1?batch={0}&commit=true'.format(batch2)
self.retry_post_json(self.root + req, [])
# It should only have a single item.
resp = self.app.get(self.root + '/storage/xxx_col1')
self.assertEquals(resp.json, ['b'])
# The first user's collection should still be empty.
# Now have the first user commit their batch.
req = '/storage/xxx_col1?batch={0}&commit=true'.format(batch1)
self.retry_post_json(self.root + req, [])
# It should only have a single item.
resp = self.app.get(self.root + '/storage/xxx_col1')
self.assertEquals(resp.json, ['a'])
# If we didn't make a conflict, try again.
if batch1 == batch2:
break
else:
raise unittest2.SkipTest('failed to generate conflicting batchid')
def test_that_we_dont_resurrect_committed_batches(self):
# This retry loop tries to trigger a situation where we:
# * create a batch with a single item
# * successfully commit that batch
# * create a new batch tht re-uses the same batchid
for _ in range(100):
bsos = [{'id': 'i', 'payload': 'aye'}]
req = '/storage/xxx_col1?batch=true'
resp = self.retry_post_json(self.root + req, bsos)
batch1 = resp.json['batch']
req = '/storage/xxx_col1?batch={0}&commit=true'.format(batch1)
self.retry_post_json(self.root + req, [])
req = '/storage/xxx_col2?batch=true'
resp = self.retry_post_json(self.root + req, [])
batch2 = resp.json['batch']
bsos = [{'id': 'j', 'payload': 'jay'}]
req = '/storage/xxx_col2?batch={0}&commit=true'.format(batch2)
self.retry_post_json(self.root + req, bsos)
# Retry if we failed to trigger re-use of the batchid.
if batch1 == batch2:
break
else:
raise unittest2.SkipTest('failed to trigger re-use of batchid')
# Despite having the same batchid, the second batch should
# be completely independent of the first.
resp = self.app.get(self.root + '/storage/xxx_col2')
self.assertEquals(resp.json, ['j'])
def test_batch_id_is_correctly_scoped_to_a_user(self):
collection = self.root + '/storage/xxx_col1'
bsos = [
{'id': 'a', 'payload': 'aih'},
{'id': 'b', 'payload': 'bie'},
{'id': 'c', 'payload': 'cee'}
]
resp = self.retry_post_json(collection + '?batch=true', bsos)
batch = resp.json['batch']
with self._switch_user():
# I should not be able to add to that batch as a different user.
endpoint = self.root + '/storage/xxx_col1?batch={0}'.format(batch)
resp = self.retry_post_json(
endpoint,
[{'id': 'd', 'payload': 'di'}],
status=400)
# I should not be able to commit that batch as a different user.
resp = self.retry_post_json(endpoint + '&commit=true', [],
status=400)
# I should still be able to use the batch in the original user.
endpoint = collection + '?batch={0}'.format(batch)
resp = self.retry_post_json(endpoint, [{'id': 'd', 'payload': 'di'}])
resp = self.retry_post_json(endpoint + '&commit=true', [])
resp = self.app.get(collection + '?full=1')
res = resp.json
res.sort(key=lambda bso: bso['id'])
self.assertEquals(len(res), 4)
self.assertEquals(res[0]['payload'], 'aih')
self.assertEquals(res[1]['payload'], 'bie')
self.assertEquals(res[2]['payload'], 'cee')
self.assertEquals(res[3]['payload'], 'di')
# bug 1332552 make sure ttl:null use the default ttl
def test_create_bso_with_null_ttl(self):
bso = {"payload": "x", "ttl": None}
self.retry_put_json(self.root + "/storage/xxx_col2/TEST1", bso)
time.sleep(0.1)
res = self.app.get(self.root + "/storage/xxx_col2/TEST1?full=1")
self.assertEquals(res.json["payload"], "x")
def test_rejection_of_known_bad_payloads(self):
bso = {
"id": "keys",
"payload": json_dumps({
"ciphertext": "IDontKnowWhatImDoing",
"IV": "AAAAAAAAAAAAAAAAAAAAAA==",
})
}
# Fishy IVs are rejected on the "crypto" collection.
self.retry_put_json(self.root + "/storage/crypto/keys", bso,
status=400)
self.retry_put_json(self.root + "/storage/crypto/blerg", bso,
status=400)
self.retry_post_json(self.root + "/storage/crypto", [bso], status=400)
# But are allowed on other collections.
self.retry_put_json(self.root + "/storage/xxx_col2/keys", bso,
status=200)
self.retry_post_json(self.root + "/storage/xxx_col2", [bso],
status=200)
# bug 1397357
def test_batch_empty_commit(self):
def testEmptyCommit(contentType, body, status=200):
bsos = [{'id': str(i).zfill(2), 'payload': 'X'} for i in range(5)]
res = self.retry_post_json(self.root+'/storage/xxx_col?batch=true',
bsos)
self.assertEquals(len(res.json['success']), 5)
self.assertEquals(len(res.json['failed']), 0)
batch = res.json["batch"]
self.app.post(
self.root+'/storage/xxx_col?commit=true&batch='+batch,
body, headers={"Content-Type": contentType},
status=status
)
testEmptyCommit("application/json", "[]")
testEmptyCommit("application/json", "{}", status=400)
testEmptyCommit("application/json", "", status=400)
testEmptyCommit("application/newlines", "")
testEmptyCommit("application/newlines", "\n", status=400)
testEmptyCommit("application/newlines", "{}", status=400)
testEmptyCommit("application/newlines", "[]", status=400)
Reference in New Issue View Git Blame Copy Permalink