mirrors/syncstorage-rs
1
0
Fork 0
mirror of https://github.com/mozilla-services/syncstorage-rs.git synced 2026-05-05 04:06:16 +02:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

Merge pull request #1467 from mozilla-services/release/0.13-merge

Browse Source 
chore: tag 0.13.6 (etc)
This commit is contained in:
Philip Jenvey 2023-03-07 17:25:09 -08:00 committed by GitHub
commit f416d8a8c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 789 additions and 500 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
CHANGELOG.md
View File

@ -1,3 +1,55 @@
<a name="0.13.6"></a>
## 0.13.6 (2023-03-07)
#### Chore
* update tempfile crate ([670d6832](https://github.com/mozilla-services/syncstorage-rs/commit/670d68325d48f1f0f7b02e431807aa6dcd252e5f))
#### Bug Fixes
* connect to the db once instead of every loop iteration ([31192d52](https://github.com/mozilla-services/syncstorage-rs/commit/31192d52c9677e5b5def9ffc62fd43099e499bd1))
<a name="0.13.5"></a>
## 0.13.5 (2023-03-03)
#### Bug Fixes
* handle nullable (None) keys_changed_at values (#1464) ([7e298c2d](https://github.com/mozilla-services/syncstorage-rs/commit/7e298c2dd06dc12a0dbc2d7e6d5aab8ab8bdfba6))
<a name="0.13.4"></a>
## 0.13.4 (2023-02-24)
* Re-tag 0.13.3
<a name="0.13.3"></a>
## 0.13.3 (2023-02-24)
#### Chore
* add another missing file to docker for process_account_events.py (#1463) ([6ee39da4](https://github.com/mozilla-services/syncstorage-rs/commit/6ee39da4a0926e6352bf513206d1d01b63232a2e))
<a name="0.13.2"></a>
## 0.13.2 (2023-02-06)
#### Chore
* add missing util.py to docker for process_account_events.py (#1455) (#1457) ([d2f6cf65](https://github.com/mozilla-services/syncstorage-rs/commit/d2f6cf65ff412676935e6f4306311e4599e697e9))
<a name="0.13.1"></a>
## 0.13.1 (2022-12-16)

1083
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

4
Dockerfile
View File

@ -82,9 +82,7 @@ COPY --from=builder /app/bin /app/bin
COPY --from=builder /app/syncserver/version.json /app
COPY --from=builder /app/tools/spanner /app/tools/spanner
COPY --from=builder /app/tools/integration_tests /app/tools/integration_tests
COPY --from=builder /app/tools/tokenserver/process_account_events.py /app/tools/tokenserver/process_account_events.py
COPY --from=builder /app/tools/tokenserver/util.py /app/tools/tokenserver/util.py
COPY --from=builder /app/tools/tokenserver/requirements.txt /app/tools/tokenserver/requirements.txt
COPY --from=builder /app/tools/tokenserver /app/tools/tokenserver
COPY --from=builder /app/scripts/prepare-spanner.sh /app/scripts/prepare-spanner.sh
COPY --from=builder /app/syncstorage-spanner/src/schema.ddl /app/schema.ddl

2
syncserver-common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncserver-common"
version = "0.13.1"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncserver-db-common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncserver-db-common"
version = "0.13.1"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncserver-settings/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncserver-settings"
version = "0.13.1"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncserver/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncserver"
version = "0.13.1"
version = "0.13.6"
license = "MPL-2.0"
authors = [
"Ben Bangert <ben@groovie.org>",

2
syncstorage-db-common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncstorage-db-common"
version = "0.12.3"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncstorage-db/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncstorage-db"
version = "0.12.3"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncstorage-mysql/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncstorage-mysql"
version = "0.12.3"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncstorage-settings/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncstorage-settings"
version = "0.13.1"
version = "0.13.6"
edition = "2021"
[dependencies]

2
syncstorage-spanner/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "syncstorage-spanner"
version = "0.12.3"
version = "0.13.6"
edition = "2021"
[dependencies]

2
tokenserver-auth/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "tokenserver-auth"
version = "0.12.3"
version = "0.13.6"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

2
tokenserver-common/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "tokenserver-common"
version = "0.13.1"
version = "0.13.6"
edition = "2021"
[dependencies]

2
tokenserver-db/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "tokenserver-db"
version = "0.12.3"
version = "0.13.6"
edition = "2021"
[dependencies]

2
tokenserver-settings/Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "tokenserver-settings"
version = "0.13.1"
version = "0.13.6"
edition = "2021"
[dependencies]

35
tools/tokenserver/database.py
View File

@ -334,15 +334,12 @@ class Database:
res = self._execute_sql(_UPDATE_USER_RECORD_IN_PLACE, **params)
res.close()
user['generation'] = max([x
for x
in [generation, user['generation']]
if x is not None])
user['keys_changed_at'] = max([x
for x
in [keys_changed_at,
user['keys_changed_at']]
if x is not None])
if generation is not None:
user['generation'] = max(user['generation'], generation)
user['keys_changed_at'] = max_keys_changed_at(
user,
keys_changed_at
)
else:
# Reject previously-seen client-state strings.
if client_state is None:
@ -369,10 +366,7 @@ class Database:
generation = max(user['generation'], generation)
else:
generation = user['generation']
if keys_changed_at is not None:
keys_changed_at = max(user['keys_changed_at'], keys_changed_at)
else:
keys_changed_at = user['keys_changed_at']
keys_changed_at = max_keys_changed_at(user, keys_changed_at)
now = get_timestamp()
params = {
'service': self._get_service_id(SERVICE_NAME),
@ -650,3 +644,18 @@ class Database:
if row is None:
raise Exception('unknown node: ' + node)
return row
def max_keys_changed_at(user, keys_changed_at):
"""Return the largest `keys_changed_at` between the user record and the
specified value.
May return `None` as the column is nullable.
"""
it = (
x
for x in (keys_changed_at, user['keys_changed_at'])
if x is not None
)
return max(it, default=None)

6
tools/tokenserver/process_account_events.py
View File

@ -51,6 +51,7 @@ def process_account_events(queue_name, aws_region=None, queue_wait_time=20):
to interrupt execution you'll need to e.g. SIGINT the process.
"""
logger.info("Processing account events from %s", queue_name)
database = Database()
try:
# Connect to the SQS queue.
# If no region is given, infer it from the instance metadata.
@ -68,7 +69,7 @@ def process_account_events(queue_name, aws_region=None, queue_wait_time=20):
msg = queue.read(wait_time_seconds=queue_wait_time)
if msg is None:
continue
process_account_event(msg.get_body())
process_account_event(database, msg.get_body())
# This intentionally deletes the event even if it was some
# unrecognized type. Not point leaving a backlog.
queue.delete_message(msg)
@ -77,9 +78,8 @@ def process_account_events(queue_name, aws_region=None, queue_wait_time=20):
raise
def process_account_event(body):
def process_account_event(database, body):
"""Parse and process a single account event."""
database = Database()
# Try very hard not to error out if there's junk in the queue.
email = None
event_type = None

83
tools/tokenserver/test_process_account_events.py
View File

@ -61,6 +61,9 @@ class TestProcessAccountEvents(unittest.TestCase):
def clearLogs(self):
del self.logs.records[:]
def process_account_event(self, body):
process_account_event(self.database, body)
def test_delete_user(self):
self.database.allocate_user(EMAIL)
user = self.database.get_user(EMAIL)
@ -69,7 +72,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.assertEquals(len(records), 2)
self.assertTrue(records[0]["replaced_at"] is not None)
process_account_event(message_body(
self.process_account_event(message_body(
event="delete",
uid=UID,
iss=ISS,
@ -88,7 +91,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.assertEquals(len(records), 2)
self.assertTrue(records[0]["replaced_at"] is not None)
process_account_event(message_body(
self.process_account_event(message_body(
event="delete",
uid=EMAIL,
))
@ -102,7 +105,7 @@ class TestProcessAccountEvents(unittest.TestCase):
records = list(self.database.get_user_records(EMAIL))
self.assertEquals(len(records), 0)
process_account_event(message_body(
self.process_account_event(message_body(
event="delete",
uid=UID,
iss=ISS
@ -114,7 +117,7 @@ class TestProcessAccountEvents(unittest.TestCase):
def test_reset_user(self):
self.database.allocate_user(EMAIL, generation=12)
process_account_event(message_body(
self.process_account_event(message_body(
event="reset",
uid=UID,
iss=ISS,
@ -127,7 +130,7 @@ class TestProcessAccountEvents(unittest.TestCase):
def test_reset_user_by_legacy_uid_format(self):
self.database.allocate_user(EMAIL, generation=12)
process_account_event(message_body(
self.process_account_event(message_body(
event="reset",
uid=EMAIL,
generation=43,
@ -140,7 +143,7 @@ class TestProcessAccountEvents(unittest.TestCase):
records = list(self.database.get_user_records(EMAIL))
self.assertEquals(len(records), 0)
process_account_event(message_body(
self.process_account_event(message_body(
event="reset",
uid=UID,
iss=ISS,
@ -153,7 +156,7 @@ class TestProcessAccountEvents(unittest.TestCase):
def test_password_change(self):
self.database.allocate_user(EMAIL, generation=12)
process_account_event(message_body(
self.process_account_event(message_body(
event="passwordChange",
uid=UID,
iss=ISS,
@ -167,7 +170,7 @@ class TestProcessAccountEvents(unittest.TestCase):
records = list(self.database.get_user_records(EMAIL))
self.assertEquals(len(records), 0)
process_account_event(message_body(
self.process_account_event(message_body(
event="passwordChange",
uid=UID,
iss=ISS,
@ -180,7 +183,7 @@ class TestProcessAccountEvents(unittest.TestCase):
def test_malformed_events(self):
# Unknown event type.
process_account_event(message_body(
self.process_account_event(message_body(
event="party",
uid=UID,
iss=ISS,
@ -190,7 +193,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.clearLogs()
# Missing event type.
process_account_event(message_body(
self.process_account_event(message_body(
uid=UID,
iss=ISS,
generation=43,
@ -199,7 +202,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.clearLogs()
# Missing uid.
process_account_event(message_body(
self.process_account_event(message_body(
event="delete",
iss=ISS,
))
@ -207,7 +210,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.clearLogs()
# Missing generation for reset events.
process_account_event(message_body(
self.process_account_event(message_body(
event="reset",
uid=UID,
iss=ISS,
@ -216,7 +219,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.clearLogs()
# Missing generation for passwordChange events.
process_account_event(message_body(
self.process_account_event(message_body(
event="passwordChange",
uid=UID,
iss=ISS,
@ -225,7 +228,7 @@ class TestProcessAccountEvents(unittest.TestCase):
self.clearLogs()
# Missing issuer with nonemail uid
process_account_event(message_body(
self.process_account_event(message_body(
event="delete",
uid=UID,
))
@ -233,16 +236,62 @@ class TestProcessAccountEvents(unittest.TestCase):
self.clearLogs()
# Non-JSON garbage.
process_account_event("wat")
self.process_account_event("wat")
self.assertMessageWasLogged("Invalid account message")
self.clearLogs()
# Non-JSON garbage in Message field.
process_account_event('{ "Message": "wat" }')
self.process_account_event('{ "Message": "wat" }')
self.assertMessageWasLogged("Invalid account message")
self.clearLogs()
# Badly-typed JSON value in Message field.
process_account_event('{ "Message": "[1, 2, 3"] }')
self.process_account_event('{ "Message": "[1, 2, 3"] }')
self.assertMessageWasLogged("Invalid account message")
self.clearLogs()
def test_update_with_no_keys_changed_at(self):
user = self.database.allocate_user(
EMAIL,
generation=12,
keys_changed_at=None
)
# These update_user calls previously failed (SYNC-3633)
self.database.update_user(user, generation=13)
self.database.update_user(
user,
generation=14,
client_state="abcdef",
keys_changed_at=13
)
self.process_account_event(message_body(
event="reset",
uid=UID,
iss=ISS,
generation=43,
))
user = self.database.get_user(EMAIL)
self.assertEquals(user["generation"], 42)
def test_update_with_no_keys_changed_at2(self):
user = self.database.allocate_user(
EMAIL,
generation=12,
keys_changed_at=None
)
# Mark the current record as replaced. This can probably only occur
# during a race condition in row creation
self.database.replace_user_record(user["uid"])
self.process_account_event(message_body(
event="reset",
uid=UID,
iss=ISS,
generation=43,
))
user = self.database.get_user(EMAIL)
self.assertEquals(user["generation"], 42)