From 743fd57bafc7d85ed5fe5a7dd475f3385dbf10f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Rafael=20Siqueira?= Date: Mon, 14 Jul 2025 17:25:32 -0300 Subject: [PATCH] add deduplication window --- http-api/app/app.py | 1 + http-api/app/routes/enrollments/__init__.py | 5 ++-- http-api/app/routes/enrollments/cancel.py | 2 +- .../enrollments/deduplication_window.py | 29 +++++++++++++++++++ http-api/app/rules/enrollment.py | 8 +++-- http-api/seeds/test-enrollments.jsonl | 1 + http-api/uv.lock | 24 ++++++++++++++- .../app/events/docs_into_eventbus.py | 25 ++++++++++++++-- streams-events/app/meili.py | 10 ++++++- streams-events/app/utils.py | 25 ++++++++++++---- streams-events/uv.lock | 13 ++++++++- 11 files changed, 125 insertions(+), 18 deletions(-) create mode 100644 http-api/app/routes/enrollments/deduplication_window.py diff --git a/http-api/app/app.py b/http-api/app/app.py index a158a9e..85d5aab 100644 --- a/http-api/app/app.py +++ b/http-api/app/app.py @@ -56,6 +56,7 @@ app.include_router(enrollments.router, prefix='/enrollments') app.include_router(enrollments.slots, prefix='/enrollments') app.include_router(enrollments.enroll, prefix='/enrollments') app.include_router(enrollments.cancel, prefix='/enrollments') +app.include_router(enrollments.deduplication_window, prefix='/enrollments') app.include_router(orders.router, prefix='/orders') app.include_router(users.router, prefix='/users') app.include_router(users.logs, prefix='/users') diff --git a/http-api/app/routes/enrollments/__init__.py b/http-api/app/routes/enrollments/__init__.py index a9f7661..b4ea344 100644 --- a/http-api/app/routes/enrollments/__init__.py +++ b/http-api/app/routes/enrollments/__init__.py @@ -14,10 +14,11 @@ from config import ENROLLMENT_TABLE, MEILISEARCH_API_KEY, MEILISEARCH_HOST, USER from middlewares import Tenant, TenantMiddleware from .cancel import router as cancel +from .deduplication_window import router as deduplication_window from .enroll import router as enroll from .slots import router as slots -__all__ = ['slots', 'cancel', 'enroll'] +__all__ = ['slots', 'cancel', 'enroll', 'deduplication_window'] router = Router() @@ -76,7 +77,7 @@ def get_enrollment(id: str): + SortKey('archived_date') + SortKey('cancel_policy') + SortKey('parent_vacancy', path_spec='vacancy') - + SortKey('lock', path_spec='hash') + + SortKey('lock') + SortKey('author') + SortKey('tenant') + SortKey('cert') diff --git a/http-api/app/routes/enrollments/cancel.py b/http-api/app/routes/enrollments/cancel.py index d3e4a0c..cd4ecaf 100644 --- a/http-api/app/routes/enrollments/cancel.py +++ b/http-api/app/routes/enrollments/cancel.py @@ -18,7 +18,7 @@ user_layer = DynamoDBPersistenceLayer(USER_TABLE, dynamodb_client) class Cancel(BaseModel): id: UUID4 | str - lock_hash: str + lock_hash: str | None = None course: dict = {} vacancy: dict = {} diff --git a/http-api/app/routes/enrollments/deduplication_window.py b/http-api/app/routes/enrollments/deduplication_window.py new file mode 100644 index 0000000..15008dd --- /dev/null +++ b/http-api/app/routes/enrollments/deduplication_window.py @@ -0,0 +1,29 @@ +from aws_lambda_powertools.event_handler.api_gateway import Router +from layercake.dynamodb import ( + DynamoDBPersistenceLayer, + KeyPair, +) +from pydantic import BaseModel + +from boto3clients import dynamodb_client +from config import ENROLLMENT_TABLE + +router = Router() +enrollment_layer = DynamoDBPersistenceLayer(ENROLLMENT_TABLE, dynamodb_client) + + +class DeduplicationWindow(BaseModel): + lock_hash: str + + +@router.patch( + '//deduplicationwindow', + compress=True, + tags=['Enrollment'], +) +def deduplication_window(id: str, payload: DeduplicationWindow): + with enrollment_layer.transact_writer() as transact: + transact.delete(key=KeyPair(id, 'lock')) + transact.delete(key=KeyPair('lock', payload.lock_hash)) + + return payload diff --git a/http-api/app/rules/enrollment.py b/http-api/app/rules/enrollment.py index e7b2c38..9c1b2c4 100644 --- a/http-api/app/rules/enrollment.py +++ b/http-api/app/rules/enrollment.py @@ -250,7 +250,7 @@ def enroll( def set_status_as_canceled( id: str, *, - lock_hash: str, + lock_hash: str | None = None, author: Author, course: Course | None = None, vacancy_key: KeyPair | None = None, @@ -290,8 +290,10 @@ def set_status_as_canceled( transact.delete(key=KeyPair(id, LifecycleEvents.ACCESS_PERIOD_ENDS)) transact.delete(key=KeyPair(id, LifecycleEvents.DOES_NOT_ACCESS)) transact.delete(key=KeyPair(id, 'parent_vacancy')) - transact.delete(key=KeyPair(id, 'lock')) - transact.delete(key=KeyPair('lock', lock_hash)) + + if lock_hash: + transact.delete(key=KeyPair(id, 'lock')) + transact.delete(key=KeyPair('lock', lock_hash)) if vacancy_key and course: vacancy_pk, vacancy_sk = vacancy_key.values() diff --git a/http-api/seeds/test-enrollments.jsonl b/http-api/seeds/test-enrollments.jsonl index 98d81d7..8223952 100644 --- a/http-api/seeds/test-enrollments.jsonl +++ b/http-api/seeds/test-enrollments.jsonl @@ -9,6 +9,7 @@ {"id": {"S": "70337adf-ddb3-4960-95b7-978cab05dcfe"}, "sk": {"S": "metadata#author"}, "create_date": {"S": "2025-05-20T12:27:09.221021-03:00"},"name": {"S": "Sérgio R Siqueira"},"user_id": {"S": "5OxmMjL-ujoR5IMGegQz"}} {"id": {"S": "70337adf-ddb3-4960-95b7-978cab05dcfe"}, "sk": {"S": "schedules#access_period_reminder_30_days"}, "course": {"S": "Noções em Primeiros Socorros"}, "create_date": {"S": "2025-05-20T12:27:09.221021-03:00"}, "email": {"S": "osergiosiqueira@gmail.com"},"name": {"S": "Sérgio R Siqueira"},"ttl": {"N": "1776266829"} {"id": {"S": "70337adf-ddb3-4960-95b7-978cab05dcfe"}, "sk": {"S": "schedules#reminder_no_access_3_days"}, "course": {"S": "Noções em Primeiros Socorros"}, "create_date": {"S": "2025-05-20T12:27:09.221021-03:00"}, "email": {"S": "osergiosiqueira@gmail.com"},"name": {"S": "Sérgio R Siqueira"},"ttl": {"N": "1748014029"}} +{"id": {"S": "70337adf-ddb3-4960-95b7-978cab05dcfe"}, "sk": {"S": "lock"}, "hash": {"S": "000c8575e1508c2c66c4faa7818b0e77"}, "ttl": {"N": "1779537056"}} {"id": {"S": "WRBj3FV7iGoxRwt63fALYd"}, "sk": {"S": "0"}, "status": {"S": "ARCHIVED"}, "progress": {"S": "100"}, "score": {"S": "100"}, "user": {"M": {"id": {"S": "12047"}, "name": {"S": "Junior Celetino Pires"}, "email": {"S": "juninhocpires@yahoo.com.br"}, "cpf": {"S": "06001201633"}}}, "course": {"M": {"id": {"S": "55"}, "name": {"S": "NR-10 Complementar (SEP)"}, "cert": {"NULL": true}, "access_period": {"N": "360"}}}, "create_date": {"S": "2016-05-16T00:00:00"}, "update_date": {"S": "2019-01-16T10:36:53"}} {"id": {"S": "nshu3G7ndUofcy7TtEvZeM"}, "sk": {"S": "0"}, "status": {"S": "ARCHIVED"}, "progress": {"S": "100"}, "score": {"S": "98"}, "user": {"M": {"id": {"S": "99523191500"}, "name": {"S": "ADELSON DE OLIVEIRA SANTOS"}, "email": {"S": "99523191500@users.noreply.betaeducacao.com.br"}, "cpf": {"S": "99523191500"}}}, "course": {"M": {"id": {"S": "dc1a0428-47bf-4db1-a5da-24be49c9fda6"}, "name": {"S": "NR-11 \u2013 Transporte, movimenta\u00e7\u00e3o, armazenagem e manuseio de materiais"}, "cert": {"NULL": true}, "access_period": {"N": "360"}}}, "create_date": {"S": "2019-09-17T09:42:19"}, "update_date": {"S": "2020-09-03T18:36:44"}} {"id": {"S": "W7Wzqr6jeMBgvmPCUm62UW"}, "sk": {"S": "0"}, "status": {"S": "ARCHIVED"}, "progress": {"S": "100"}, "score": {"S": "88"}, "user": {"M": {"id": {"S": "b70ca900-885e-4aca-a3ef-ceee3b2974d6"}, "name": {"S": "JOICE RIBEIRO ROCHA"}, "email": {"S": "joicerrocha@hotmail.com"}, "cpf": {"S": "12599815762"}}}, "course": {"M": {"id": {"S": "56d1c710-36b1-4db5-8a7a-dacb7098dbad"}, "name": {"S": "NR-11 Seguran\u00e7a na Opera\u00e7\u00e3o de Rebocadores"}, "cert": {"NULL": true}, "access_period": {"N": "360"}}}, "create_date": {"S": "2021-12-14T10:06:10"}, "update_date": {"S": "2021-12-15T09:55:21"}} diff --git a/http-api/uv.lock b/http-api/uv.lock index 5a49186..0248855 100644 --- a/http-api/uv.lock +++ b/http-api/uv.lock @@ -367,6 +367,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/e7/aa315e6a749d9b96c2504a1ba0ba031ba2d0517e972ce22682e3fccecb09/cssselect2-0.8.0-py3-none-any.whl", hash = "sha256:46fc70ebc41ced7a32cd42d58b1884d72ade23d21e5a4eaaf022401c13f0e76e", size = 15454, upload-time = "2025-03-05T14:46:06.463Z" }, ] +[[package]] +name = "dictdiffer" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/7b/35cbccb7effc5d7e40f4c55e2b79399e1853041997fcda15c9ff160abba0/dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578", size = 31513, upload-time = "2021-07-22T13:24:29.276Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" }, +] + [[package]] name = "dnspython" version = "2.7.0" @@ -558,11 +567,12 @@ wheels = [ [[package]] name = "layercake" -version = "0.7.0" +version = "0.7.2" source = { directory = "../layercake" } dependencies = [ { name = "arnparse" }, { name = "aws-lambda-powertools", extra = ["all"] }, + { name = "dictdiffer" }, { name = "ftfy" }, { name = "glom" }, { name = "jinja2" }, @@ -576,6 +586,7 @@ dependencies = [ { name = "requests" }, { name = "smart-open", extra = ["s3"] }, { name = "sqlite-utils" }, + { name = "unidecode" }, { name = "weasyprint" }, ] @@ -583,6 +594,7 @@ dependencies = [ requires-dist = [ { name = "arnparse", specifier = ">=0.0.2" }, { name = "aws-lambda-powertools", extras = ["all"], specifier = ">=3.8.0" }, + { name = "dictdiffer", specifier = ">=0.9.0" }, { name = "ftfy", specifier = ">=6.3.1" }, { name = "glom", specifier = ">=24.11.0" }, { name = "jinja2", specifier = ">=3.1.6" }, @@ -596,6 +608,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.3" }, { name = "smart-open", extras = ["s3"], specifier = ">=7.1.0" }, { name = "sqlite-utils", specifier = ">=3.38" }, + { name = "unidecode", specifier = ">=1.4.0" }, { name = "weasyprint", specifier = ">=65.0" }, ] @@ -1141,6 +1154,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/86/39b65d676ec5732de17b7e3c476e45bb80ec64eb50737a8dce1a4178aba1/typing_extensions-4.13.0-py3-none-any.whl", hash = "sha256:c8dd92cc0d6425a97c18fbb9d1954e5ff92c1ca881a309c45f06ebc0b79058e5", size = 45683, upload-time = "2025-03-26T03:49:40.35Z" }, ] +[[package]] +name = "unidecode" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/7d/a8a765761bbc0c836e397a2e48d498305a865b70a8600fd7a942e85dcf63/Unidecode-1.4.0.tar.gz", hash = "sha256:ce35985008338b676573023acc382d62c264f307c8f7963733405add37ea2b23", size = 200149, upload-time = "2025-04-24T08:45:03.798Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/b7/559f59d57d18b44c6d1250d2eeaa676e028b9c527431f5d0736478a73ba1/Unidecode-1.4.0-py3-none-any.whl", hash = "sha256:c3c7606c27503ad8d501270406e345ddb480a7b5f38827eafe4fa82a137f0021", size = 235837, upload-time = "2025-04-24T08:45:01.609Z" }, +] + [[package]] name = "urllib3" version = "2.3.0" diff --git a/streams-events/app/events/docs_into_eventbus.py b/streams-events/app/events/docs_into_eventbus.py index 3c7f7a3..c2650e2 100644 --- a/streams-events/app/events/docs_into_eventbus.py +++ b/streams-events/app/events/docs_into_eventbus.py @@ -1,4 +1,6 @@ +import decimal import json +import math from typing import TYPE_CHECKING import boto3 @@ -14,7 +16,7 @@ from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import ) from aws_lambda_powertools.utilities.typing import LambdaContext from layercake.dateutils import now, ttl -from utils import JSONEncoder, diff, table_from_arn +from utils import diff, table_from_arn if TYPE_CHECKING: from mypy_boto3_events.client import EventBridgeClient @@ -33,7 +35,7 @@ def record_handler(record: DynamoDBRecord): table_name: str = table_from_arn(record.event_source_arn) # type: ignore new_image: dict = record.dynamodb.new_image # type: ignore old_image: dict = record.dynamodb.old_image # type: ignore - record_ttl: int = old_image.get('ttl') # type: ignore + record_ttl: int | None = old_image.get('ttl') modified = diff(new_image, old_image) now_ = now() @@ -63,10 +65,10 @@ def record_handler(record: DynamoDBRecord): } ] ) + logger.info('Event result', result=result) -@logger.inject_lambda_context @tracer.capture_lambda_handler def lambda_handler(event: dict, context: LambdaContext): return process_partial_response( @@ -75,3 +77,20 @@ def lambda_handler(event: dict, context: LambdaContext): processor=processor, context=context, ) + + +class JSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, decimal.Decimal): + if o.is_nan(): + return math.nan + + if o % 1 != 0: + return float(o.quantize(decimal.Decimal('0.00'))) + + return int(o) + + if isinstance(o, set): + return list(o) + + return super().default(o) diff --git a/streams-events/app/meili.py b/streams-events/app/meili.py index a68d5bc..1143e23 100644 --- a/streams-events/app/meili.py +++ b/streams-events/app/meili.py @@ -1,10 +1,18 @@ from typing import Self +from aws_lambda_powertools.shared.json_encoder import Encoder from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import ( DynamoDBRecordEventName, ) from meilisearch import Client -from utils import JSONEncoder + + +class JSONEncoder(Encoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + + return super().default(obj) class Op: diff --git a/streams-events/app/utils.py b/streams-events/app/utils.py index 7292454..837d541 100644 --- a/streams-events/app/utils.py +++ b/streams-events/app/utils.py @@ -1,6 +1,9 @@ +import decimal +import json +import math + import dictdiffer from arnparse import arnparse -from aws_lambda_powertools.shared.json_encoder import Encoder def table_from_arn(arn: str) -> str: @@ -20,8 +23,18 @@ def diff(first: dict, second: dict) -> list[str]: return changed -class JSONEncoder(Encoder): - def default(self, obj): - if isinstance(obj, set): - return list(obj) - return super(__class__, self).default(obj) +class JSONEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, decimal.Decimal): + if o.is_nan(): + return math.nan + + if o % 1 != 0: + return float(o.quantize(decimal.Decimal('0.00'))) + + return int(o) + + if isinstance(o, set): + return list(o) + + return super().default(o) diff --git a/streams-events/uv.lock b/streams-events/uv.lock index 6f4e26b..6a10deb 100644 --- a/streams-events/uv.lock +++ b/streams-events/uv.lock @@ -575,7 +575,7 @@ wheels = [ [[package]] name = "layercake" -version = "0.7.1" +version = "0.7.2" source = { directory = "../layercake" } dependencies = [ { name = "arnparse" }, @@ -594,6 +594,7 @@ dependencies = [ { name = "requests" }, { name = "smart-open", extra = ["s3"] }, { name = "sqlite-utils" }, + { name = "unidecode" }, { name = "weasyprint" }, ] @@ -615,6 +616,7 @@ requires-dist = [ { name = "requests", specifier = ">=2.32.3" }, { name = "smart-open", extras = ["s3"], specifier = ">=7.1.0" }, { name = "sqlite-utils", specifier = ">=3.38" }, + { name = "unidecode", specifier = ">=1.4.0" }, { name = "weasyprint", specifier = ">=65.0" }, ] @@ -1186,6 +1188,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/31/08/aa4fdfb71f7de5176385bd9e90852eaf6b5d622735020ad600f2bab54385/typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f", size = 14125, upload-time = "2025-02-25T17:27:57.754Z" }, ] +[[package]] +name = "unidecode" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/7d/a8a765761bbc0c836e397a2e48d498305a865b70a8600fd7a942e85dcf63/Unidecode-1.4.0.tar.gz", hash = "sha256:ce35985008338b676573023acc382d62c264f307c8f7963733405add37ea2b23", size = 200149, upload-time = "2025-04-24T08:45:03.798Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/b7/559f59d57d18b44c6d1250d2eeaa676e028b9c527431f5d0736478a73ba1/Unidecode-1.4.0-py3-none-any.whl", hash = "sha256:c3c7606c27503ad8d501270406e345ddb480a7b5f38827eafe4fa82a137f0021", size = 235837, upload-time = "2025-04-24T08:45:01.609Z" }, +] + [[package]] name = "urllib3" version = "2.3.0"