From 02f0b317ae51a91d336535b8dfc84ad017c7f38d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Rafael=20Siqueira?= Date: Tue, 25 Mar 2025 22:38:30 -0300 Subject: [PATCH] remove prefix --- layercake/layercake/dynamodb.py | 139 ++++++++++++++++++++++--------- layercake/pyproject.toml | 1 + layercake/tests/test_dynamodb.py | 37 +++++++- layercake/uv.lock | 52 ++++++++++++ 4 files changed, 186 insertions(+), 43 deletions(-) diff --git a/layercake/layercake/dynamodb.py b/layercake/layercake/dynamodb.py index a32fe0d..319c77a 100644 --- a/layercake/layercake/dynamodb.py +++ b/layercake/layercake/dynamodb.py @@ -1,8 +1,12 @@ +import json import os from abc import ABC, abstractmethod +from base64 import urlsafe_b64decode, urlsafe_b64encode +from dataclasses import dataclass from datetime import datetime from ipaddress import IPv4Address -from typing import Any, Type, TypedDict +from typing import TYPE_CHECKING, Any, Type, TypedDict +from urllib.parse import quote, unquote from uuid import UUID from aws_lambda_powertools import Logger @@ -49,34 +53,70 @@ def deserialize(value: dict) -> dict: return {k: deserializer.deserialize(v) for k, v in value.items()} -def ComposeKey( - keyparts: str | tuple[str, ...], - *, - prefix: str | None = None, - delimiter: str = DELIMITER, -) -> str: - """Creates a composite key by joining string parts with a specified delimiter. - If a prefix is provided, it is added at the beginning of the key parts. +if TYPE_CHECKING: - Example - ------- - >>> ComposeKey(('abc', 'xyz'), prefix='examples', delimiter='#') - 'examples#abc#xyz' - """ + @dataclass + class ComposeKey(str): + """Creates a composite key by joining string parts with a specified delimiter. + If a prefix is provided, it is added at the beginning of the key parts. - if not prefix and not isinstance(keyparts, tuple): - return keyparts + Example + ------- + >>> ComposeKey(('abc', 'xyz'), prefix='examples', delimiter='#') + 'examples#abc#xyz' + """ - if isinstance(keyparts, str): - keyparts = (keyparts,) + keyparts: str | tuple[str, ...] + prefix: str | None = None + delimiter: str = '#' +else: - if prefix: - keyparts = (prefix,) + keyparts + class ComposeKey(str): + def __new__( + cls, + keyparts: str | tuple[str, ...], + *, + prefix: str | None = None, + delimiter: str = '#', + ) -> str: + if isinstance(keyparts, str): + keyparts = (keyparts,) - return delimiter.join(keyparts) + if prefix: + keyparts = (prefix,) + keyparts + + return super().__new__(cls, delimiter.join(keyparts)) + + def __init__( + self, + keyparts: str | tuple[str, ...], + *, + prefix: str | None = None, + delimiter: str = '#', + ) -> None: + # __init__ is used to store the parameters for later reference. + # For immutable types like str, __init__ cannot change the instance's value. + self.keyparts = keyparts + self.prefix = prefix + self.delimiter = delimiter -class PrimaryKey(ABC, dict): +if TYPE_CHECKING: + + @dataclass + class PrefixKey(str): + prefix: str + +else: + + class PrefixKey(str): + def __init__(self, prefix: str | None = None) -> None: + # __init__ is used to store the parameters for later reference. + # For immutable types like str, __init__ cannot change the instance's value. + self.prefix = prefix + + +class Key(ABC, dict): @abstractmethod def expr_attr_name(self) -> dict: ... @@ -84,11 +124,11 @@ class PrimaryKey(ABC, dict): def expr_attr_values(self) -> dict: ... -class PartitionKey(PrimaryKey): +class PartitionKey(Key): """Represents a partition key for DynamoDB queries""" def __init__(self, pk: str) -> None: - super().__init__(**{PK: pk}) + super().__init__(**{PK: pk, SK: None}) def expr_attr_name(self) -> dict: return {'#pk': PK} @@ -97,7 +137,7 @@ class PartitionKey(PrimaryKey): return {':pk': self[PK]} -class KeyPair(PrimaryKey): +class KeyPair(Key): """Represents a composite key (partition key and sort key) for DynamoDB queries""" def __init__(self, pk: str, sk: str) -> None: @@ -341,7 +381,7 @@ class DynamoDBPersistenceLayer: attrs['Limit'] = limit try: - response = self.dynamodb_client.query( + res = self.dynamodb_client.query( TableName=self.table_name, KeyConditionExpression=key_cond_expr, **attrs, @@ -351,17 +391,17 @@ class DynamoDBPersistenceLayer: raise else: return dict( - items=[deserialize(v) for v in response.get('Items', [])], - last_key=response.get('LastEvaluatedKey', None), + items=[deserialize(v) for v in res.get('Items', [])], + last_key=res.get('LastEvaluatedKey', None), ) def get_item(self, key: dict) -> dict: """The GetItem operation returns a set of attributes for the item with the given primary key. - If there is no matching item, GetItem does not return any data and there will be no Item element in the response. + If there is no matching item, GetItem does not return any data and there will be no Item element in the res. """ try: - response = self.dynamodb_client.get_item( + res = self.dynamodb_client.get_item( TableName=self.table_name, Key=serialize(key), ) @@ -369,7 +409,7 @@ class DynamoDBPersistenceLayer: logger.exception(err) raise else: - return deserialize(response.get('Item', {})) + return deserialize(res.get('Item', {})) def put_item(self, item: dict, *, cond_expr: str | None = None) -> bool: attrs = {} @@ -456,17 +496,14 @@ class DynamoDBPersistenceLayer: def transact_get_items(self, transact_items: TransactItems) -> list[dict]: try: - response = self.dynamodb_client.transact_get_items( + res = self.dynamodb_client.transact_get_items( TransactItems=transact_items.items ) except ClientError as err: logger.exception(err) raise else: - return [ - deserialize(response.get('Item', {})) - for response in response.get('Responses', []) - ] + return [deserialize(res.get('Item', {})) for res in res.get('ress', [])] def transact_write_items(self, transact_items: TransactItems) -> bool: try: @@ -516,7 +553,7 @@ class DynamoDBCollection: collect = DynamoDBCollection(...) collect.get_items( - KeyPair('b3511b5a-cb32-4833-a373-f8223f2088d4', 'emails), + KeyPair('b3511b5a-cb32-4833-a373-f8223f2088d4', 'emails'), ) """ @@ -619,17 +656,37 @@ class DynamoDBCollection: expr_attr_name.update(key.expr_attr_name()) expr_attr_values.update(key.expr_attr_values()) - response = self.persistence_layer.query( + res = self.persistence_layer.query( key_cond_expr=key_cond_expr, expr_attr_name=expr_attr_name, expr_attr_values=expr_attr_values, filter_expr=filter_expr, index_forward=index_forward, limit=limit, - # start_key=start_key if start_key else {}, + start_key=_startkey_b64decode(start_key) if start_key else {}, ) + items = res['items'] + last_key = _startkey_b64encode(res['last_key']) if res['last_key'] else None + + # Remove prefix from Sort Key if `key[SK]` is a PrefixKey + if isinstance(key[SK], PrefixKey): + prefix = key[SK].prefix + items = [item | {SK: item[SK].removeprefix(prefix)} for item in items] + return { - 'items': response['items'], - 'last_key': response['last_key'] if 'last_key' in response else None, + 'items': items, + 'last_key': last_key, } + + +def _startkey_b64encode(obj: dict) -> str: + s = json.dumps(obj) + b = urlsafe_b64encode(s.encode('utf-8')).decode('utf-8') + return quote(b) + + +def _startkey_b64decode(s: str) -> dict: + b = unquote(s).encode('utf-8') + s = urlsafe_b64decode(b).decode('utf-8') + return json.loads(s) diff --git a/layercake/pyproject.toml b/layercake/pyproject.toml index 33a5dcb..eab8359 100644 --- a/layercake/pyproject.toml +++ b/layercake/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "pydantic-extra-types>=2.10.3", "pytz>=2025.1", "shortuuid>=1.0.13", + "requests>=2.32.3", ] [dependency-groups] diff --git a/layercake/tests/test_dynamodb.py b/layercake/tests/test_dynamodb.py index 8dbc377..7fa542a 100644 --- a/layercake/tests/test_dynamodb.py +++ b/layercake/tests/test_dynamodb.py @@ -12,6 +12,7 @@ from layercake.dynamodb import ( KeyPair, MissingError, PartitionKey, + PrefixKey, TransactItems, serialize, ) @@ -34,7 +35,11 @@ def test_serialize(): def test_composekey(): - assert ComposeKey(('122', 'abc'), prefix='schedules') == 'schedules#122#abc' + key = ComposeKey(('122', 'abc'), prefix='schedules', delimiter=':') + assert key == 'schedules:122:abc' + assert key.prefix == 'schedules' + assert key.delimiter == ':' + assert ComposeKey(('122', 'abc')) == '122#abc' assert ComposeKey('122') == '122' @@ -51,6 +56,12 @@ def test_keypair(): assert KeyPair('123', 'abc').expr_attr_values() == {':pk': '123', ':sk': 'abc'} +def test_prefixkey(): + key = PrefixKey('emails') + assert key == 'emails' + assert isinstance(key, PrefixKey) + + def test_transact_write_items( dynamodb_seeds, dynamodb_persistence_layer: DynamoDBPersistenceLayer, @@ -63,7 +74,10 @@ def test_transact_write_items( cond_expr='attribute_not_exists(sk)', ) transact.put( - item=KeyPair('5OxmMjL-ujoR5IMGegQz', 'emails#sergio@somosbeta.com.br'), + item=KeyPair( + '5OxmMjL-ujoR5IMGegQz', + ComposeKey('sergio@somosbeta.com.br', prefix='emails'), + ), cond_expr='attribute_not_exists(sk)', ) @@ -163,3 +177,22 @@ def test_collection_get_items( ), ) assert len(data['items']) == 2 + + # This data was added from seeds + data = collect.get_items( + KeyPair('5OxmMjL-ujoR5IMGegQz', PrefixKey('emails#')), + ) + assert data == { + 'items': [ + { + 'email_verified': True, + 'mx_record_exists': True, + 'sk': 'sergio@somosbeta.com.br', # Removed prefix from Sort Key + 'email_primary': True, + 'id': '5OxmMjL-ujoR5IMGegQz', + 'create_date': '2019-03-25T00:00:00-03:00', + 'update_date': '2023-11-09T12:13:04.308986-03:00', + } + ], + 'last_key': None, + } diff --git a/layercake/uv.lock b/layercake/uv.lock index dcba7c5..8ba0209 100644 --- a/layercake/uv.lock +++ b/layercake/uv.lock @@ -149,6 +149,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, ] +[[package]] +name = "charset-normalizer" +version = "3.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/9a/dd1e1cdceb841925b7798369a09279bd1cf183cef0f9ddf15a3a6502ee45/charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545", size = 196105 }, + { url = "https://files.pythonhosted.org/packages/d3/8c/90bfabf8c4809ecb648f39794cf2a84ff2e7d2a6cf159fe68d9a26160467/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7", size = 140404 }, + { url = "https://files.pythonhosted.org/packages/ad/8f/e410d57c721945ea3b4f1a04b74f70ce8fa800d393d72899f0a40526401f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757", size = 150423 }, + { url = "https://files.pythonhosted.org/packages/f0/b8/e6825e25deb691ff98cf5c9072ee0605dc2acfca98af70c2d1b1bc75190d/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa", size = 143184 }, + { url = "https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d", size = 145268 }, + { url = "https://files.pythonhosted.org/packages/74/94/8a5277664f27c3c438546f3eb53b33f5b19568eb7424736bdc440a88a31f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616", size = 147601 }, + { url = "https://files.pythonhosted.org/packages/7c/5f/6d352c51ee763623a98e31194823518e09bfa48be2a7e8383cf691bbb3d0/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b", size = 141098 }, + { url = "https://files.pythonhosted.org/packages/78/d4/f5704cb629ba5ab16d1d3d741396aec6dc3ca2b67757c45b0599bb010478/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d", size = 149520 }, + { url = "https://files.pythonhosted.org/packages/c5/96/64120b1d02b81785f222b976c0fb79a35875457fa9bb40827678e54d1bc8/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a", size = 152852 }, + { url = "https://files.pythonhosted.org/packages/84/c9/98e3732278a99f47d487fd3468bc60b882920cef29d1fa6ca460a1fdf4e6/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9", size = 150488 }, + { url = "https://files.pythonhosted.org/packages/13/0e/9c8d4cb99c98c1007cc11eda969ebfe837bbbd0acdb4736d228ccaabcd22/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1", size = 146192 }, + { url = "https://files.pythonhosted.org/packages/b2/21/2b6b5b860781a0b49427309cb8670785aa543fb2178de875b87b9cc97746/charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35", size = 95550 }, + { url = "https://files.pythonhosted.org/packages/21/5b/1b390b03b1d16c7e382b561c5329f83cc06623916aab983e8ab9239c7d5c/charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f", size = 102785 }, + { url = "https://files.pythonhosted.org/packages/38/94/ce8e6f63d18049672c76d07d119304e1e2d7c6098f0841b51c666e9f44a0/charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda", size = 195698 }, + { url = "https://files.pythonhosted.org/packages/24/2e/dfdd9770664aae179a96561cc6952ff08f9a8cd09a908f259a9dfa063568/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313", size = 140162 }, + { url = "https://files.pythonhosted.org/packages/24/4e/f646b9093cff8fc86f2d60af2de4dc17c759de9d554f130b140ea4738ca6/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9", size = 150263 }, + { url = "https://files.pythonhosted.org/packages/5e/67/2937f8d548c3ef6e2f9aab0f6e21001056f692d43282b165e7c56023e6dd/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b", size = 142966 }, + { url = "https://files.pythonhosted.org/packages/52/ed/b7f4f07de100bdb95c1756d3a4d17b90c1a3c53715c1a476f8738058e0fa/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11", size = 144992 }, + { url = "https://files.pythonhosted.org/packages/96/2c/d49710a6dbcd3776265f4c923bb73ebe83933dfbaa841c5da850fe0fd20b/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f", size = 147162 }, + { url = "https://files.pythonhosted.org/packages/b4/41/35ff1f9a6bd380303dea55e44c4933b4cc3c4850988927d4082ada230273/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd", size = 140972 }, + { url = "https://files.pythonhosted.org/packages/fb/43/c6a0b685fe6910d08ba971f62cd9c3e862a85770395ba5d9cad4fede33ab/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2", size = 149095 }, + { url = "https://files.pythonhosted.org/packages/4c/ff/a9a504662452e2d2878512115638966e75633519ec11f25fca3d2049a94a/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886", size = 152668 }, + { url = "https://files.pythonhosted.org/packages/6c/71/189996b6d9a4b932564701628af5cee6716733e9165af1d5e1b285c530ed/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601", size = 150073 }, + { url = "https://files.pythonhosted.org/packages/e4/93/946a86ce20790e11312c87c75ba68d5f6ad2208cfb52b2d6a2c32840d922/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd", size = 145732 }, + { url = "https://files.pythonhosted.org/packages/cd/e5/131d2fb1b0dddafc37be4f3a2fa79aa4c037368be9423061dccadfd90091/charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407", size = 95391 }, + { url = "https://files.pythonhosted.org/packages/27/f2/4f9a69cc7712b9b5ad8fdb87039fd89abba997ad5cbe690d1835d40405b0/charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971", size = 102702 }, + { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -396,6 +431,7 @@ dependencies = [ { name = "pydantic", extra = ["email"] }, { name = "pydantic-extra-types" }, { name = "pytz" }, + { name = "requests" }, { name = "shortuuid" }, ] @@ -420,6 +456,7 @@ requires-dist = [ { name = "pydantic", extras = ["email"], specifier = ">=2.10.6" }, { name = "pydantic-extra-types", specifier = ">=2.10.3" }, { name = "pytz", specifier = ">=2025.1" }, + { name = "requests", specifier = ">=2.32.3" }, { name = "shortuuid", specifier = ">=1.0.13" }, ] @@ -664,6 +701,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930 }, ] +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, +] + [[package]] name = "ruff" version = "0.11.1"