fix

2025-07-14 11:09:16 -03:00
parent cea6b61f51
commit 51cc1bbbb5
11 changed files with 242 additions and 57 deletions
--- a/streams-events/app/events/docs_into_eventbus.py
+++ b/streams-events/app/events/docs_into_eventbus.py
@@ -0,0 +1,74 @@
+import json
+from typing import TYPE_CHECKING
+
+import boto3
+from aws_lambda_powertools import Logger, Tracer
+from aws_lambda_powertools.shared.json_encoder import Encoder
+from aws_lambda_powertools.utilities.batch import (
+    BatchProcessor,
+    EventType,
+    process_partial_response,
+)
+from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import (
+    DynamoDBRecord,
+    DynamoDBRecordEventName,
+)
+from aws_lambda_powertools.utilities.typing import LambdaContext
+from layercake.dateutils import now, ttl
+from utils import diff, table_from_arn
+
+if TYPE_CHECKING:
+    from mypy_boto3_events.client import EventBridgeClient
+else:
+    EventBridgeClient = object
+
+client: EventBridgeClient = boto3.client('events')
+processor = BatchProcessor(event_type=EventType.DynamoDBStreams)
+tracer = Tracer()
+logger = Logger()
+
+
+@tracer.capture_method
+def record_handler(record: DynamoDBRecord):
+    detail_type = record.raw_event['eventName']
+    table_name: str = table_from_arn(record.event_source_arn)  # type: ignore
+    new_image: dict = record.dynamodb.new_image  # type: ignore
+    old_image: dict = record.dynamodb.old_image  # type: ignore
+    record_ttl: int = old_image.get('ttl')  # type: ignore
+    modified = diff(new_image, old_image)
+    now_ = now()
+
+    # Should be EXPIRE if event is REMOVE and TTL has elapsed
+    if record.event_name is DynamoDBRecordEventName.REMOVE and ttl() >= record_ttl:
+        detail_type = 'EXPIRE'
+
+    detail = {
+        'keys': record.dynamodb.keys,  # type: ignore
+        'new_image': new_image,
+        'old_image': old_image,
+        'modified': modified,
+    }
+
+    result = client.put_events(
+        Entries=[
+            {
+                'Source': record.event_source,  # type: ignore
+                'DetailType': detail_type,
+                'Resources': [table_name],
+                'Detail': json.dumps(detail, cls=Encoder),
+                'Time': now_,
+            }
+        ]
+    )
+    logger.info('Event result', result=result)
+
+
+@logger.inject_lambda_context
+@tracer.capture_lambda_handler
+def lambda_handler(event: dict, context: LambdaContext):
+    return process_partial_response(
+        event=event,
+        record_handler=record_handler,
+        processor=processor,
+        context=context,
+    )
--- a/streams-events/app/events/index_docs_into_meili.py
+++ b/streams-events/app/events/index_docs_into_meili.py
@@ -1,5 +1,4 @@
-from arnparse import arnparse
-from aws_lambda_powertools import Logger
+from aws_lambda_powertools import Logger, Tracer
 from aws_lambda_powertools.utilities.data_classes import (
    DynamoDBStreamEvent,
    event_source,
@@ -8,13 +7,16 @@ from aws_lambda_powertools.utilities.typing import LambdaContext
 from config import MEILISEARCH_API_KEY, MEILISEARCH_HOST
 from meili import Op
 from meilisearch import Client as Meilisearch
+from utils import table_from_arn

 logger = Logger(__name__)
+tracer = Tracer()
 meili_client = Meilisearch(MEILISEARCH_HOST, MEILISEARCH_API_KEY)


@event_source(data_class=DynamoDBStreamEvent)
@logger.inject_lambda_context
+@tracer.capture_lambda_handler
 def lambda_handler(event: DynamoDBStreamEvent, context: LambdaContext):
    with Op(meili_client) as op:
        for record in event.records:
@@ -29,12 +31,7 @@ def lambda_handler(event: DynamoDBStreamEvent, context: LambdaContext):
            )


-def table_from_arn(arn: str) -> str:
-    arn_ = arnparse(arn)
-    return arn_.resource.split('/')[0]
-
-
-# Post-migration: remove the following lines
+# Post-migration: remove the following function
 def sanitize(obj):
    if isinstance(obj, dict):
        return {k.replace(':', '__'): sanitize(v) for k, v in obj.items()}
--- a/streams-events/app/utils.py
+++ b/streams-events/app/utils.py
@@ -0,0 +1,19 @@
+import dictdiffer
+from arnparse import arnparse
+
+
+def table_from_arn(arn: str) -> str:
+    arn_ = arnparse(arn)
+    return arn_.resource.split('/')[0]
+
+
+def diff(first: dict, second: dict) -> list[str]:
+    result = dictdiffer.diff(first, second)
+    changed = []
+
+    for record in result:
+        type_, path, _ = record
+        if type_ == 'change':
+            changed.append(path)
+
+    return changed
--- a/streams-events/pyproject.toml
+++ b/streams-events/pyproject.toml
@@ -8,6 +8,7 @@ dependencies = ["layercake"]

 [dependency-groups]
 dev = [
+    "boto3-stubs[events]>=1.39.4",
    "pytest>=8.3.4",
    "pytest-cov>=6.0.0",
    "ruff>=0.9.1",
--- a/streams-events/template.yaml
+++ b/streams-events/template.yaml
@@ -8,7 +8,7 @@ Globals:
    Architectures:
      - x86_64
    Layers:
-      - !Sub arn:aws:lambda:sa-east-1:336641857101:layer:layercake:79
+      - !Sub arn:aws:lambda:sa-east-1:336641857101:layer:layercake:81
    Environment:
      Variables:
        LOG_LEVEL: DEBUG
@@ -24,6 +24,11 @@ Resources:
    Properties:
      RetentionInDays: 90

+  EventBusLog:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      RetentionInDays: 90
+
  EventIndexDocsIntoMeiliFunction:
    Type: AWS::Serverless::Function
    Properties:
@@ -74,3 +79,42 @@ Resources:
            FilterCriteria:
              Filters:
                - Pattern: '{ "dynamodb" : { "Keys" : { "sk" : { "S" : [ "0" ] } } } }'
+
+  EventDocsIntoEventBusFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      Handler: events.docs_into_eventbus.lambda_handler
+      LoggingConfig:
+        LogGroup: !Ref EventBusLog
+      Policies:
+        - EventBridgePutEventsPolicy:
+            EventBusName: default
+      Events:
+        Users:
+          Type: DynamoDB
+          Properties:
+            Stream: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/betaeducacao-prod-users_d2o3r5gmm4it7j/stream/2022-06-12T21:33:25.634
+            StartingPosition: LATEST
+            MaximumRetryAttempts: 5
+            BatchSize: 25
+        Enrolllments:
+          Type: DynamoDB
+          Properties:
+            Stream: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/betaeducacao-prod-enrollments/stream/2023-08-22T22:56:55.612
+            StartingPosition: LATEST
+            MaximumRetryAttempts: 5
+            BatchSize: 25
+        Orders:
+          Type: DynamoDB
+          Properties:
+            Stream: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/betaeducacao-prod-orders/stream/2023-09-15T18:58:50.395
+            StartingPosition: LATEST
+            MaximumRetryAttempts: 5
+            BatchSize: 25
+        Courses:
+          Type: DynamoDB
+          Properties:
+            Stream: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/saladeaula_courses/stream/2025-03-12T20:42:46.706
+            StartingPosition: LATEST
+            MaximumRetryAttempts: 5
+            BatchSize: 25
--- a/streams-events/tests/test_docs_into_eventbus.py
+++ b/streams-events/tests/test_docs_into_eventbus.py
@@ -0,0 +1,5 @@
+import app.events.docs_into_eventbus as app
+
+
+def test_record_handler(monkeypatch, dynamodb_stream_event, lambda_context):
+    app.lambda_handler(dynamodb_stream_event, lambda_context)
--- a/streams-events/uv.lock
+++ b/streams-events/uv.lock
@@ -103,6 +103,24 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/2d/fa/8ea42eff98e02962473f60c11663282cd8b8c04cc66eab954184325516ac/boto3-1.37.24-py3-none-any.whl", hash = "sha256:2f2b8f82a5d7f89283973bf2cab771b90c09348799e78b2a25c60cd22c443514", size = 139561, upload-time = "2025-03-31T19:35:14.96Z" },
 ]

+[[package]]
+name = "boto3-stubs"
+version = "1.39.4"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "botocore-stubs" },
+    { name = "types-s3transfer" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/53/df/f3efdf2ba46f32cf30a42807c8f556b71775ae9f021143a72410361961a7/boto3_stubs-1.39.4.tar.gz", hash = "sha256:29b2a35fdb01f82bd08c667771effcc75df13e0a673f1dbcd2538937005d3f30", size = 99993, upload-time = "2025-07-09T19:28:46.897Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/4b/63/bcbf7b35c03a52372360e84e2c2930395f9bdfbdedbb70e9e9c067577c09/boto3_stubs-1.39.4-py3-none-any.whl", hash = "sha256:fd0b22946dfbb992e51912fe8a7ce6b97f1855211d2b4de24f21af325f94e9c7", size = 69195, upload-time = "2025-07-09T19:28:38.363Z" },
+]
+
+[package.optional-dependencies]
+events = [
+    { name = "mypy-boto3-events" },
+]
+
 [[package]]
 name = "botocore"
 version = "1.37.24"
@@ -117,6 +135,18 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/15/60/85f9dfdb94e58a5aab83e4a29773948b74989ce477de61e946732fb0ed69/botocore-1.37.24-py3-none-any.whl", hash = "sha256:f1a55332cca85a6556af8941cccdaf5d2d00336647d9e89f31174f2361ffb4f2", size = 13462492, upload-time = "2025-03-31T19:34:59.617Z" },
 ]

+[[package]]
+name = "botocore-stubs"
+version = "1.38.46"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "types-awscrt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/05/45/27cabc7c3022dcb12de5098cc646b374065f5e72fae13600ff1756f365ee/botocore_stubs-1.38.46.tar.gz", hash = "sha256:a04e69766ab8bae338911c1897492f88d05cd489cd75f06e6eb4f135f9da8c7b", size = 42299, upload-time = "2025-06-29T22:58:24.765Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/cc/84/06490071e26bab22ac79a684e98445df118adcf80c58c33ba5af184030f2/botocore_stubs-1.38.46-py3-none-any.whl", hash = "sha256:cc21d9a7dd994bdd90872db4664d817c4719b51cda8004fd507a4bf65b085a75", size = 66083, upload-time = "2025-06-29T22:58:22.234Z" },
+]
+
 [[package]]
 name = "brotli"
 version = "1.1.0"
@@ -382,6 +412,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/0f/e7/aa315e6a749d9b96c2504a1ba0ba031ba2d0517e972ce22682e3fccecb09/cssselect2-0.8.0-py3-none-any.whl", hash = "sha256:46fc70ebc41ced7a32cd42d58b1884d72ade23d21e5a4eaaf022401c13f0e76e", size = 15454, upload-time = "2025-03-05T14:46:06.463Z" },
 ]

+[[package]]
+name = "dictdiffer"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/61/7b/35cbccb7effc5d7e40f4c55e2b79399e1853041997fcda15c9ff160abba0/dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578", size = 31513, upload-time = "2021-07-22T13:24:29.276Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" },
+]
+
 [[package]]
 name = "dnspython"
 version = "2.7.0"
@@ -391,46 +430,6 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632, upload-time = "2024-10-05T20:14:57.687Z" },
 ]

-[[package]]
-name = "elastic-transport"
-version = "8.17.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "certifi" },
-    { name = "urllib3" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/d7/82/2a544ac3d9c4ae19acc7f53117251bee20dd65dc3dff01fe55ea45ae9bd9/elastic_transport-8.17.0.tar.gz", hash = "sha256:e755f38f99fa6ec5456e236b8e58f0eb18873ac8fe710f74b91a16dd562de2a5", size = 73304, upload-time = "2025-01-07T08:12:37.534Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2a/0d/2dd25c06078070973164b661e0d79868e434998391f9aed74d4070aab270/elastic_transport-8.17.0-py3-none-any.whl", hash = "sha256:59f553300866750e67a38828fede000576562a0e66930c641adb75249e0c95af", size = 64523, upload-time = "2025-01-07T08:12:34.528Z" },
-]
-
-[[package]]
-name = "elasticsearch"
-version = "8.17.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "elastic-transport" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/5b/3d/f563e58f45d23565c0d0316a565638ce312f536b882a3281b8047fb4a58f/elasticsearch-8.17.2.tar.gz", hash = "sha256:ff7f1db8aeefd87ceba4edce3aa4070994582e6cf029d2e67b74e66d634509db", size = 602691, upload-time = "2025-03-04T12:14:27.382Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c0/50/16306f4722ca2fcb64a5875bc1fa9b4d0bcb08c05967f60c23acd4cbb019/elasticsearch-8.17.2-py3-none-any.whl", hash = "sha256:2d058dcddd8f2686cd431a916cdf983f9fb7d211d902834f564ab7df05ba6478", size = 717971, upload-time = "2025-03-04T12:14:23.843Z" },
-]
-
-[[package]]
-name = "elasticsearch-dsl"
-version = "8.17.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "elastic-transport" },
-    { name = "elasticsearch" },
-    { name = "python-dateutil" },
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/bf/57/d375ce8915f289f1f032e001010cd901c2f1be4a14246e7506af63ba34f7/elasticsearch_dsl-8.17.1.tar.gz", hash = "sha256:d8170699bfdb4fe7fab3854cdac319a2d6dddbaa29c9ea7993d2ec22056db5a0", size = 151630, upload-time = "2025-01-08T12:02:06.801Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ad/b4/5e707bca39062ba0b5227696a767db09767e5f09e869c6cb14aeb36e4b9d/elasticsearch_dsl-8.17.1-py3-none-any.whl", hash = "sha256:49ee12a6a8d43fcfc0af42b49649531a6ef228c9e4795325de27f6b309b62b6d", size = 158294, upload-time = "2025-01-08T12:02:03.951Z" },
-]
-
 [[package]]
 name = "email-validator"
 version = "2.2.0"
@@ -576,13 +575,12 @@ wheels = [

 [[package]]
 name = "layercake"
-version = "0.6.12"
+version = "0.7.1"
 source = { directory = "../layercake" }
 dependencies = [
    { name = "arnparse" },
    { name = "aws-lambda-powertools", extra = ["all"] },
-    { name = "elasticsearch" },
-    { name = "elasticsearch-dsl" },
+    { name = "dictdiffer" },
    { name = "ftfy" },
    { name = "glom" },
    { name = "jinja2" },
@@ -603,8 +601,7 @@ dependencies = [
 requires-dist = [
    { name = "arnparse", specifier = ">=0.0.2" },
    { name = "aws-lambda-powertools", extras = ["all"], specifier = ">=3.8.0" },
-    { name = "elasticsearch", specifier = ">=8.17.2" },
-    { name = "elasticsearch-dsl", specifier = ">=8.17.1" },
+    { name = "dictdiffer", specifier = ">=0.9.0" },
    { name = "ftfy", specifier = ">=6.3.1" },
    { name = "glom", specifier = ">=24.11.0" },
    { name = "jinja2", specifier = ">=3.1.6" },
@@ -682,6 +679,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/e0/2f/264c07a3f488260ea36c78cbc201b76e6baf9ef92e0c7f78657a6a5e5f22/meilisearch-0.34.0-py3-none-any.whl", hash = "sha256:fae8ad2a15d12c27fa0a1fff2ae2e4e3e2e22b869950408d63c87e2c095a9f61", size = 24373, upload-time = "2025-02-18T05:50:32.73Z" },
 ]

+[[package]]
+name = "mypy-boto3-events"
+version = "1.39.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/65/9c/7fc76ab42c5bead0d514d937bb535faea02cd58890a2353c6ade1535ed44/mypy_boto3_events-1.39.0.tar.gz", hash = "sha256:efd52fa8003fee4b22c42eae327e753e2262c38211993d969a3572945be46ebc", size = 33782, upload-time = "2025-06-30T19:38:32.976Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/b8/48/76fe8775022eef841a4256eeba95df4bf9a49d2daea0ec196459e80e9222/mypy_boto3_events-1.39.0-py3-none-any.whl", hash = "sha256:cf2d9d18b6c68a3c6b98e4c2bc6ac1c3b06c14dadf1a6ac81d9f6d63ec3a8592", size = 37377, upload-time = "2025-06-30T19:38:29.401Z" },
+]
+
 [[package]]
 name = "orjson"
 version = "3.10.16"
@@ -1091,6 +1097,7 @@ dependencies = [

 [package.dev-dependencies]
 dev = [
+    { name = "boto3-stubs", extra = ["events"] },
    { name = "pytest" },
    { name = "pytest-cov" },
    { name = "ruff" },
@@ -1101,6 +1108,7 @@ requires-dist = [{ name = "layercake", directory = "../layercake" }]

 [package.metadata.requires-dev]
 dev = [
+    { name = "boto3-stubs", extras = ["events"], specifier = ">=1.39.4" },
    { name = "pytest", specifier = ">=8.3.4" },
    { name = "pytest-cov", specifier = ">=6.0.0" },
    { name = "ruff", specifier = ">=0.9.1" },
@@ -1139,6 +1147,24 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/5c/de/27c57899297163a4a84104d5cec0af3b1ac5faf62f44667e506373c6b8ce/tinyhtml5-2.0.0-py3-none-any.whl", hash = "sha256:13683277c5b176d070f82d099d977194b7a1e26815b016114f581a74bbfbf47e", size = 39793, upload-time = "2024-10-29T15:37:11.743Z" },
 ]

+[[package]]
+name = "types-awscrt"
+version = "0.27.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/94/95/02564024f8668feab6733a2c491005b5281b048b3d0573510622cbcd9fd4/types_awscrt-0.27.4.tar.gz", hash = "sha256:c019ba91a097e8a31d6948f6176ede1312963f41cdcacf82482ac877cbbcf390", size = 16941, upload-time = "2025-06-29T22:58:04.756Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d4/40/cb4d04df4ac3520858f5b397a4ab89f34be2601000002a26edd8ddc0cac5/types_awscrt-0.27.4-py3-none-any.whl", hash = "sha256:a8c4b9d9ae66d616755c322aba75ab9bd793c6fef448917e6de2e8b8cdf66fb4", size = 39626, upload-time = "2025-06-29T22:58:03.157Z" },
+]
+
+[[package]]
+name = "types-s3transfer"
+version = "0.13.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/42/c1/45038f259d6741c252801044e184fec4dbaeff939a58f6160d7c32bf4975/types_s3transfer-0.13.0.tar.gz", hash = "sha256:203dadcb9865c2f68fb44bc0440e1dc05b79197ba4a641c0976c26c9af75ef52", size = 14175, upload-time = "2025-05-28T02:16:07.614Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c8/5d/6bbe4bf6a79fb727945291aef88b5ecbdba857a603f1bbcf1a6be0d3f442/types_s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:79c8375cbf48a64bff7654c02df1ec4b20d74f8c5672fc13e382f593ca5565b3", size = 19588, upload-time = "2025-05-28T02:16:06.709Z" },
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.12.2"