add postgres

2025-08-26 00:00:42 -03:00
parent 1326530991
commit e33eccebb9
43 changed files with 622 additions and 636 deletions
--- a/streams-events/app/config.py
+++ b/streams-events/app/config.py
@@ -2,3 +2,9 @@ import os

 MEILISEARCH_HOST: str = os.getenv('MEILISEARCH_HOST')  # type: ignore
 MEILISEARCH_API_KEY: str = os.getenv('MEILISEARCH_API_KEY')  # type: ignore
+
+POSTGRES_DB: str = os.getenv('POSTGRES_DB')  # type: ignore
+POSTGRES_HOST: str = os.getenv('POSTGRES_HOST')  # type: ignore
+POSTGRES_PORT: str = os.getenv('POSTGRES_PORT')  # type: ignore
+POSTGRES_USER: str = os.getenv('POSTGRES_USER')  # type: ignore
+POSTGRES_PASSWORD: str = os.getenv('POSTGRES_PASSWORD')  # type: ignore
--- a/streams-events/app/events/docs_into_eventbus.py
+++ b/streams-events/app/events/docs_into_eventbus.py
@@ -16,6 +16,7 @@ from aws_lambda_powertools.utilities.data_classes.dynamo_db_stream_event import
 )
 from aws_lambda_powertools.utilities.typing import LambdaContext
 from layercake.dateutils import now, ttl
+
 from utils import diff, table_from_arn

 if TYPE_CHECKING:
--- a/streams-events/app/events/index_docs_into_meili.py
+++ b/streams-events/app/events/index_docs_into_meili.py
@@ -4,9 +4,10 @@ from aws_lambda_powertools.utilities.data_classes import (
    event_source,
 )
 from aws_lambda_powertools.utilities.typing import LambdaContext
+from meilisearch import Client as Meilisearch
+
 from config import MEILISEARCH_API_KEY, MEILISEARCH_HOST
 from meili import Op
-from meilisearch import Client as Meilisearch
 from utils import table_from_arn

 logger = Logger(__name__)
--- a/streams-events/app/events/replicate_into_meili.py
+++ b/streams-events/app/events/replicate_into_meili.py
@@ -1,33 +0,0 @@
-from aws_lambda_powertools import Logger, Tracer
-from aws_lambda_powertools.utilities.data_classes import (
-    DynamoDBStreamEvent,
-    event_source,
-)
-from aws_lambda_powertools.utilities.typing import LambdaContext
-from config import MEILISEARCH_API_KEY, MEILISEARCH_HOST
-from layercake.strutils import md5_hash
-from meili import Op
-from meilisearch import Client as Meilisearch
-from utils import table_from_arn
-
-logger = Logger(__name__)
-tracer = Tracer()
-meili_client = Meilisearch(MEILISEARCH_HOST, MEILISEARCH_API_KEY)
-
-
-@event_source(data_class=DynamoDBStreamEvent)
-@logger.inject_lambda_context
-@tracer.capture_lambda_handler
-def lambda_handler(event: DynamoDBStreamEvent, context: LambdaContext):
-    with Op(meili_client) as op:
-        for record in event.records:
-            new_image = record.dynamodb.new_image  # type: ignore
-            index = table_from_arn(record.event_source_arn)  # type: ignore
-            keys = record.dynamodb.keys  # type: ignore
-            _id = md5_hash(str(keys))
-
-            op.append(
-                f'_{index}_replication',
-                op=record.event_name,  # type: ignore
-                data=({'_id': _id} | new_image) or _id,
-            )
--- a/streams-events/pyproject.toml
+++ b/streams-events/pyproject.toml
@@ -21,8 +21,15 @@ dev = [
 pythonpath = ["app/"]
 addopts = "--cov --cov-report html -v"

+[tool.ruff]
+target-version = "py311"
+src = ["app"]
+
 [tool.ruff.format]
 quote-style = "single"

+[tool.ruff.lint]
+select = ["E", "F", "I"]
+
 [tool.uv.sources]
 layercake = { path = "../layercake" }
--- a/streams-events/template.yaml
+++ b/streams-events/template.yaml
@@ -8,15 +8,20 @@ Globals:
    Architectures:
      - x86_64
    Layers:
-      - !Sub arn:aws:lambda:sa-east-1:336641857101:layer:layercake:94
+      - !Sub arn:aws:lambda:sa-east-1:336641857101:layer:layercake:96
    Environment:
      Variables:
-        LOG_LEVEL: INFO
+        LOG_LEVEL: DEBUG
        TZ: America/Sao_Paulo
        POWERTOOLS_LOGGER_SAMPLE_RATE: 0.1
        POWERTOOLS_LOGGER_LOG_EVENT: true
-        MEILISEARCH_HOST: https://meili.eduseg.com.br
+        MEILISEARCH_HOST: https://meili.saladeaula.digital
        MEILISEARCH_API_KEY: "{{resolve:ssm:/saladeaula/meili_api_key}}"
+        POSTGRES_DB: saladeaula.digital
+        POSTGRES_HOST: sp-node01.saladeaula.digital
+        POSTGRES_PORT: 5432
+        POSTGRES_USER: "{{resolve:ssm:/saladeaula/postgres_user}}"
+        POSTGRES_PASSWORD: "{{resolve:ssm:/saladeaula/postgres_password}}"

 Resources:
  MeilisearchLog:
@@ -29,6 +34,11 @@ Resources:
    Properties:
      RetentionInDays: 90

+  EventPostgresLog:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      RetentionInDays: 90
+
  EventIndexDocsIntoMeiliFunction:
    Type: AWS::Serverless::Function
    Properties:
@@ -46,7 +56,6 @@ Resources:
            FilterCriteria:
              Filters:
                - Pattern: '{ "dynamodb" : { "Keys" : { "sk" : { "S" : [ "0" ] } } } }'
-
        Users:
          Type: DynamoDB
          Properties:
@@ -57,7 +66,6 @@ Resources:
            FilterCriteria:
              Filters:
                - Pattern: '{ "dynamodb" : { "Keys" : { "sk" : { "S" : [ "0" ] } } } }'
-
        Orders:
          Type: DynamoDB
          Properties:
@@ -68,7 +76,6 @@ Resources:
            FilterCriteria:
              Filters:
                - Pattern: '{ "dynamodb" : { "Keys" : { "sk" : { "S" : [ "0" ] } } } }'
-
        Courses:
          Type: DynamoDB
          Properties:
@@ -97,7 +104,7 @@ Resources:
            StartingPosition: LATEST
            MaximumRetryAttempts: 5
            BatchSize: 25
-        Enrolllments:
+        Enrollments:
          Type: DynamoDB
          Properties:
            Stream: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/betaeducacao-prod-enrollments/stream/2023-08-22T22:56:55.612
@@ -119,15 +126,12 @@ Resources:
            MaximumRetryAttempts: 5
            BatchSize: 25

-  EventReplicateIntoMeiliFunction:
+  EventReplicateIntoPostgresFunction:
    Type: AWS::Serverless::Function
    Properties:
-      Handler: events.replicate_into_meili.lambda_handler
+      Handler: events.replicate_into_postgres.lambda_handler
      LoggingConfig:
-        LogGroup: !Ref EventBusLog
-      Policies:
-        - EventBridgePutEventsPolicy:
-            EventBusName: default
+        LogGroup: !Ref EventPostgresLog
      Events:
        Users:
          Type: DynamoDB
@@ -136,7 +140,7 @@ Resources:
            StartingPosition: LATEST
            MaximumRetryAttempts: 5
            BatchSize: 25
-        Enrolllments:
+        Enrollments:
          Type: DynamoDB
          Properties:
            Stream: !Sub arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/betaeducacao-prod-enrollments/stream/2023-08-22T22:56:55.612
--- a/streams-events/tests/conftest.py
+++ b/streams-events/tests/conftest.py
@@ -9,6 +9,11 @@ import pytest
 def pytest_configure():
    os.environ['TZ'] = 'America/Sao_Paulo'
    os.environ['MEILISEARCH_HOST'] = 'http://127.0.0.1:7700'
+    os.environ['POSTGRES_DB'] = 'pytest'
+    os.environ['POSTGRES_HOST'] = 'localhost'
+    os.environ['POSTGRES_PORT'] = '5432'
+    os.environ['POSTGRES_USER'] = 'postgres'
+    os.environ['POSTGRES_PASSWORD'] = 'pgsql@pwd'


 def load_jsonfile(path: str) -> dict:
--- a/streams-events/tests/test_replicate_into_meili.py
+++ b/streams-events/tests/test_replicate_into_meili.py
@@ -1,5 +0,0 @@
-import app.events.replicate_into_meili as app
-
-
-def test_record_handler(dynamodb_stream_event, lambda_context):
-    app.lambda_handler(dynamodb_stream_event, lambda_context)
--- a/streams-events/uv.lock
+++ b/streams-events/uv.lock
@@ -484,7 +484,7 @@ wheels = [

 [[package]]
 name = "layercake"
-version = "0.9.12"
+version = "0.9.14"
 source = { directory = "../layercake" }
 dependencies = [
    { name = "arnparse" },
@@ -496,6 +496,7 @@ dependencies = [
    { name = "meilisearch" },
    { name = "orjson" },
    { name = "passlib" },
+    { name = "psycopg", extra = ["binary"] },
    { name = "pycpfcnpj" },
    { name = "pydantic", extra = ["email"] },
    { name = "pydantic-extra-types" },
@@ -518,6 +519,7 @@ requires-dist = [
    { name = "meilisearch", specifier = ">=0.34.0" },
    { name = "orjson", specifier = ">=3.10.15" },
    { name = "passlib", specifier = ">=1.7.4" },
+    { name = "psycopg", extras = ["binary"], specifier = ">=3.2.9" },
    { name = "pycpfcnpj", specifier = ">=1.8" },
    { name = "pydantic", extras = ["email"], specifier = ">=2.10.6" },
    { name = "pydantic-extra-types", specifier = ">=2.10.3" },
@@ -634,6 +636,53 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/a3/58/35da89ee790598a0700ea49b2a66594140f44dec458c07e8e3d4979137fc/ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce", size = 49567, upload-time = "2018-02-15T19:01:27.172Z" },
 ]

+[[package]]
+name = "psycopg"
+version = "3.2.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+    { name = "tzdata", marker = "sys_platform == 'win32'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/27/4a/93a6ab570a8d1a4ad171a1f4256e205ce48d828781312c0bbaff36380ecb/psycopg-3.2.9.tar.gz", hash = "sha256:2fbb46fcd17bc81f993f28c47f1ebea38d66ae97cc2dbc3cad73b37cefbff700", size = 158122, upload-time = "2025-05-13T16:11:15.533Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/44/b0/a73c195a56eb6b92e937a5ca58521a5c3346fb233345adc80fd3e2f542e2/psycopg-3.2.9-py3-none-any.whl", hash = "sha256:01a8dadccdaac2123c916208c96e06631641c0566b22005493f09663c7a8d3b6", size = 202705, upload-time = "2025-05-13T16:06:26.584Z" },
+]
+
+[package.optional-dependencies]
+binary = [
+    { name = "psycopg-binary", marker = "implementation_name != 'pypy'" },
+]
+
+[[package]]
+name = "psycopg-binary"
+version = "3.2.9"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/29/6f/ec9957e37a606cd7564412e03f41f1b3c3637a5be018d0849914cb06e674/psycopg_binary-3.2.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:be7d650a434921a6b1ebe3fff324dbc2364393eb29d7672e638ce3e21076974e", size = 4022205, upload-time = "2025-05-13T16:07:48.195Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/ba/497b8bea72b20a862ac95a94386967b745a472d9ddc88bc3f32d5d5f0d43/psycopg_binary-3.2.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a76b4722a529390683c0304501f238b365a46b1e5fb6b7249dbc0ad6fea51a0", size = 4083795, upload-time = "2025-05-13T16:07:50.917Z" },
+    { url = "https://files.pythonhosted.org/packages/42/07/af9503e8e8bdad3911fd88e10e6a29240f9feaa99f57d6fac4a18b16f5a0/psycopg_binary-3.2.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96a551e4683f1c307cfc3d9a05fec62c00a7264f320c9962a67a543e3ce0d8ff", size = 4655043, upload-time = "2025-05-13T16:07:54.857Z" },
+    { url = "https://files.pythonhosted.org/packages/28/ed/aff8c9850df1648cc6a5cc7a381f11ee78d98a6b807edd4a5ae276ad60ad/psycopg_binary-3.2.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:61d0a6ceed8f08c75a395bc28cb648a81cf8dee75ba4650093ad1a24a51c8724", size = 4477972, upload-time = "2025-05-13T16:07:57.925Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/bd/8e9d1b77ec1a632818fe2f457c3a65af83c68710c4c162d6866947d08cc5/psycopg_binary-3.2.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad280bbd409bf598683dda82232f5215cfc5f2b1bf0854e409b4d0c44a113b1d", size = 4737516, upload-time = "2025-05-13T16:08:01.616Z" },
+    { url = "https://files.pythonhosted.org/packages/46/ec/222238f774cd5a0881f3f3b18fb86daceae89cc410f91ef6a9fb4556f236/psycopg_binary-3.2.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76eddaf7fef1d0994e3d536ad48aa75034663d3a07f6f7e3e601105ae73aeff6", size = 4436160, upload-time = "2025-05-13T16:08:04.278Z" },
+    { url = "https://files.pythonhosted.org/packages/37/78/af5af2a1b296eeca54ea7592cd19284739a844974c9747e516707e7b3b39/psycopg_binary-3.2.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:52e239cd66c4158e412318fbe028cd94b0ef21b0707f56dcb4bdc250ee58fd40", size = 3753518, upload-time = "2025-05-13T16:08:07.567Z" },
+    { url = "https://files.pythonhosted.org/packages/ec/ac/8a3ed39ea069402e9e6e6a2f79d81a71879708b31cc3454283314994b1ae/psycopg_binary-3.2.9-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:08bf9d5eabba160dd4f6ad247cf12f229cc19d2458511cab2eb9647f42fa6795", size = 3313598, upload-time = "2025-05-13T16:08:09.999Z" },
+    { url = "https://files.pythonhosted.org/packages/da/43/26549af068347c808fbfe5f07d2fa8cef747cfff7c695136172991d2378b/psycopg_binary-3.2.9-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1b2cf018168cad87580e67bdde38ff5e51511112f1ce6ce9a8336871f465c19a", size = 3407289, upload-time = "2025-05-13T16:08:12.66Z" },
+    { url = "https://files.pythonhosted.org/packages/67/55/ea8d227c77df8e8aec880ded398316735add8fda5eb4ff5cc96fac11e964/psycopg_binary-3.2.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:14f64d1ac6942ff089fc7e926440f7a5ced062e2ed0949d7d2d680dc5c00e2d4", size = 3472493, upload-time = "2025-05-13T16:08:15.672Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/02/6ff2a5bc53c3cd653d281666728e29121149179c73fddefb1e437024c192/psycopg_binary-3.2.9-cp312-cp312-win_amd64.whl", hash = "sha256:7a838852e5afb6b4126f93eb409516a8c02a49b788f4df8b6469a40c2157fa21", size = 2927400, upload-time = "2025-05-13T16:08:18.652Z" },
+    { url = "https://files.pythonhosted.org/packages/28/0b/f61ff4e9f23396aca674ed4d5c9a5b7323738021d5d72d36d8b865b3deaf/psycopg_binary-3.2.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:98bbe35b5ad24a782c7bf267596638d78aa0e87abc7837bdac5b2a2ab954179e", size = 4017127, upload-time = "2025-05-13T16:08:21.391Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/00/7e181fb1179fbfc24493738b61efd0453d4b70a0c4b12728e2b82db355fd/psycopg_binary-3.2.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:72691a1615ebb42da8b636c5ca9f2b71f266be9e172f66209a361c175b7842c5", size = 4080322, upload-time = "2025-05-13T16:08:24.049Z" },
+    { url = "https://files.pythonhosted.org/packages/58/fd/94fc267c1d1392c4211e54ccb943be96ea4032e761573cf1047951887494/psycopg_binary-3.2.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25ab464bfba8c401f5536d5aa95f0ca1dd8257b5202eede04019b4415f491351", size = 4655097, upload-time = "2025-05-13T16:08:27.376Z" },
+    { url = "https://files.pythonhosted.org/packages/41/17/31b3acf43de0b2ba83eac5878ff0dea5a608ca2a5c5dd48067999503a9de/psycopg_binary-3.2.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e8aeefebe752f46e3c4b769e53f1d4ad71208fe1150975ef7662c22cca80fab", size = 4482114, upload-time = "2025-05-13T16:08:30.781Z" },
+    { url = "https://files.pythonhosted.org/packages/85/78/b4d75e5fd5a85e17f2beb977abbba3389d11a4536b116205846b0e1cf744/psycopg_binary-3.2.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7e4e4dd177a8665c9ce86bc9caae2ab3aa9360b7ce7ec01827ea1baea9ff748", size = 4737693, upload-time = "2025-05-13T16:08:34.625Z" },
+    { url = "https://files.pythonhosted.org/packages/3b/95/7325a8550e3388b00b5e54f4ced5e7346b531eb4573bf054c3dbbfdc14fe/psycopg_binary-3.2.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fc2915949e5c1ea27a851f7a472a7da7d0a40d679f0a31e42f1022f3c562e87", size = 4437423, upload-time = "2025-05-13T16:08:37.444Z" },
+    { url = "https://files.pythonhosted.org/packages/1a/db/cef77d08e59910d483df4ee6da8af51c03bb597f500f1fe818f0f3b925d3/psycopg_binary-3.2.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a1fa38a4687b14f517f049477178093c39c2a10fdcced21116f47c017516498f", size = 3758667, upload-time = "2025-05-13T16:08:40.116Z" },
+    { url = "https://files.pythonhosted.org/packages/95/3e/252fcbffb47189aa84d723b54682e1bb6d05c8875fa50ce1ada914ae6e28/psycopg_binary-3.2.9-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5be8292d07a3ab828dc95b5ee6b69ca0a5b2e579a577b39671f4f5b47116dfd2", size = 3320576, upload-time = "2025-05-13T16:08:43.243Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/cd/9b5583936515d085a1bec32b45289ceb53b80d9ce1cea0fef4c782dc41a7/psycopg_binary-3.2.9-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:778588ca9897b6c6bab39b0d3034efff4c5438f5e3bd52fda3914175498202f9", size = 3411439, upload-time = "2025-05-13T16:08:47.321Z" },
+    { url = "https://files.pythonhosted.org/packages/45/6b/6f1164ea1634c87956cdb6db759e0b8c5827f989ee3cdff0f5c70e8331f2/psycopg_binary-3.2.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f0d5b3af045a187aedbd7ed5fc513bd933a97aaff78e61c3745b330792c4345b", size = 3477477, upload-time = "2025-05-13T16:08:51.166Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/1d/bf54cfec79377929da600c16114f0da77a5f1670f45e0c3af9fcd36879bc/psycopg_binary-3.2.9-cp313-cp313-win_amd64.whl", hash = "sha256:2290bc146a1b6a9730350f695e8b670e1d1feb8446597bed0bbe7c3c30e0abcb", size = 2928009, upload-time = "2025-05-13T16:08:53.67Z" },
+]
+
 [[package]]
 name = "pycouchdb"
 version = "1.16.0"
@@ -1003,6 +1052,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/31/08/aa4fdfb71f7de5176385bd9e90852eaf6b5d622735020ad600f2bab54385/typing_inspection-0.4.0-py3-none-any.whl", hash = "sha256:50e72559fcd2a6367a19f7a7e610e6afcb9fac940c650290eed893d61386832f", size = 14125, upload-time = "2025-02-25T17:27:57.754Z" },
 ]

+[[package]]
+name = "tzdata"
+version = "2025.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
+]
+
 [[package]]
 name = "unidecode"
 version = "1.4.0"