add batch

batch-jobs/app/events/csv_chunks.py

@@ -1,20 +0,0 @@
-import boto3
-from aws_lambda_powertools.utilities.data_classes import (
-    EventBridgeEvent,
-    event_source,
-)
-from aws_lambda_powertools.utilities.typing import LambdaContext
-
-from csv_utils import byte_ranges
-
-CHUNK_SIZE = 50
-s3_client = boto3.client('s3')
-
-
-@event_source(data_class=EventBridgeEvent)
-def lambda_handler(event: EventBridgeEvent, context: LambdaContext) -> bool:
-    new_image = event.detail['new_image']
-    csvfile = new_image['csv_s3uri']
-    pairs = byte_ranges(csvfile, CHUNK_SIZE)
-
-    return True

batch-jobs/app/events/read_chunk.py

@@ -1,47 +0,0 @@
-import csv
-from io import StringIO
-
-import boto3
-from aws_lambda_powertools.utilities.data_classes import (
-    EventBridgeEvent,
-    event_source,
-)
-from aws_lambda_powertools.utilities.typing import LambdaContext
-
-from csv_utils import byte_ranges
-
-CHUNK_SIZE = 50
-s3_client = boto3.client('s3')
-
-
-@event_source(data_class=EventBridgeEvent)
-def lambda_handler(event: EventBridgeEvent, context: LambdaContext) -> bool:
-    new_image = event.detail['new_image']
-    csvfile = new_image['csv_s3uri']
-    *_, pair = byte_ranges(csvfile, CHUNK_SIZE)
-
-    data = get_object_range(csvfile, pair[0], pair[1], s3_client=s3_client)
-    reader = csv.reader(data)
-
-    for x in reader:
-        print(x)
-    return True
-
-
-def get_object_range(
-    s3_uri: str,
-    start_byte: int,
-    end_byte: int = -1,
-    *,
-    s3_client,
-) -> StringIO:
-    bucket, key = s3_uri.replace('s3://', '').split('/', 1)
-    range_ = f'bytes={start_byte}-{end_byte}' if end_byte else f'bytes={start_byte}-'
-
-    response = s3_client.get_object(
-        Bucket=bucket,
-        Key=key,
-        Range=range_,
-    )
-
-    return StringIO(response['Body'].read().decode('utf-8'))

batch-jobs/pyproject.toml

@@ -8,7 +8,6 @@ dependencies = ["layercake"]
 
 [dependency-groups]
 dev = [
-    "duckdb>=1.2.2",
     "pytest>=8.3.4",
     "pytest-cov>=6.0.0",
     "ruff>=0.9.1",

batch-jobs/tests/events/test_csv_chunks.py

@@ -1,13 +1,13 @@
-import events.csv_chunks as app
+import events.chunk_csv as app
 
 
-def test_csv_chunks(lambda_context):
+def test_chunk_csv(lambda_context):
     event = {
         'detail': {
             'new_image': {
-                'csv_s3uri': 's3://saladeaula.digital/samples/large_users.csv',
+                's3uri': 's3://saladeaula.digital/samples/large_users.csv',
             },
         },
     }
 
-    app.lambda_handler(event, lambda_context)
+    app.lambda_handler(event, lambda_context)  # type: ignore

batch-jobs/tests/test_csv_utils.py

@@ -12,6 +12,8 @@ def test_byte_ranges():
     *_, pair = ranges
     start_byte, end_byte = pair
 
+    assert ranges == [(0, 808), (809, 1655), (1656, 2303)]
+
     expected = """,RICARDO GALLES BONET,ricardo.bonet@fanucamerica.com,424.430.528-93,NR-10 (RECICLAGEM)
 ,RULIO SIEFERT SERA,rulio.sera@fanucamerica.com,063.916.859-08,NR-10 (RECICLAGEM)
 ,MACIEL FERREIRA BOMFIM,maciel.bomfim@fanucamerica.com,334.547.088-85,NR-10 (RECICLAGEM)

batch-jobs/uv.lock

@@ -90,7 +90,6 @@ dependencies = [
 
 [package.dev-dependencies]
 dev = [
-    { name = "duckdb" },
     { name = "pytest" },
     { name = "pytest-cov" },
     { name = "ruff" },
@@ -101,7 +100,6 @@ requires-dist = [{ name = "layercake", directory = "../layercake" }]
 
 [package.metadata.requires-dev]
 dev = [
-    { name = "duckdb", specifier = ">=1.2.2" },
     { name = "pytest", specifier = ">=8.3.4" },
     { name = "pytest-cov", specifier = ">=6.0.0" },
     { name = "ruff", specifier = ">=0.9.1" },
@@ -343,22 +341,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632, upload-time = "2024-10-05T20:14:57.687Z" },
 ]
 
-[[package]]
-name = "duckdb"
-version = "1.2.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/28/b8/0f86278684fb7a1fac7c0c869fc6d68ed005cdc91c963eb4373e0551bc0a/duckdb-1.2.2.tar.gz", hash = "sha256:1e53555dece49201df08645dbfa4510c86440339889667702f936b7d28d39e43", size = 11595514, upload-time = "2025-04-08T08:47:20.234Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/41/31/5e2f68cbd000137f6ed52092ad83a8e9c09eca70c59e0b4c5eb679709997/duckdb-1.2.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:fb9a2c77236fae079185a990434cb9d8432902488ba990235c702fc2692d2dcd", size = 15272507, upload-time = "2025-04-08T08:46:15.605Z" },
-    { url = "https://files.pythonhosted.org/packages/d2/15/aa9078fc897e744e077c0c1510e34db4c809de1d51ddb5cb62e1f9c61312/duckdb-1.2.2-cp313-cp313-macosx_12_0_universal2.whl", hash = "sha256:d8bb89e580cb9a3aaf42e4555bf265d3db9446abfb118e32150e1a5dfa4b5b15", size = 31965548, upload-time = "2025-04-08T08:46:18.593Z" },
-    { url = "https://files.pythonhosted.org/packages/9f/28/943773d44fd97055c59b58dde9182733661c2b6e3b3549f15dc26b2e139e/duckdb-1.2.2-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:88916d7f0532dc926bed84b50408c00dcbe6d2097d0de93c3ff647d8d57b4f83", size = 16800600, upload-time = "2025-04-08T08:46:21.51Z" },
-    { url = "https://files.pythonhosted.org/packages/39/51/2caf01e7791e490290798c8c155d4d702ed61d69e815915b42e72b3e7473/duckdb-1.2.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30bece4f58a6c7bb0944a02dd1dc6de435a9daf8668fa31a9fe3a9923b20bd65", size = 18735886, upload-time = "2025-04-08T08:46:24.26Z" },
-    { url = "https://files.pythonhosted.org/packages/87/0c/48ae1d485725af3a452303af409a9022d751ecab260cb9ca2f8c9fb670bc/duckdb-1.2.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd2c6373b8b54474724c2119f6939c4568c428e1d0be5bcb1f4e3d7f1b7c8bb", size = 20210481, upload-time = "2025-04-08T08:46:26.717Z" },
-    { url = "https://files.pythonhosted.org/packages/69/c7/95fcd7bde0f754ea6700208d36b845379cbd2b28779c0eff4dd4a7396369/duckdb-1.2.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72f688a8b0df7030c5a28ca6072817c1f090979e08d28ee5912dee37c26a7d0c", size = 18756619, upload-time = "2025-04-08T08:46:29.035Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/1b/c9eab9e84d4a70dd5f7e2a93dd6e9d7b4d868d3df755cd58b572d82d6c5d/duckdb-1.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26e9c349f56f7c99341b5c79bbaff5ba12a5414af0261e79bf1a6a2693f152f6", size = 22294667, upload-time = "2025-04-08T08:46:31.295Z" },
-    { url = "https://files.pythonhosted.org/packages/3f/3d/ce68db53084746a4a62695a4cb064e44ce04123f8582bb3afbf6ee944e16/duckdb-1.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:e1aec7102670e59d83512cf47d32a6c77a79df9df0294c5e4d16b6259851e2e9", size = 11370206, upload-time = "2025-04-08T08:46:33.472Z" },
-]
-
 [[package]]
 name = "elastic-transport"
 version = "8.17.1"

certs/pyproject.toml

@@ -12,8 +12,16 @@ dependencies = [
 [tool.uv.sources]
 layercake = { path = "../layercake" }
 
+[tool.ruff]
+target-version = "py311"
+src = ["app"]
+
 [tool.ruff.format]
 quote-style = "single"
 
+[tool.ruff.lint]
+select = ["E", "F", "I"]
 [dependency-groups]
-dev = ["ruff>=0.11.9"]
+dev = [
+    "ruff>=0.11.9",
+]

http-api/app/app.py

@@ -32,6 +32,8 @@ app.use(middlewares=[AuthenticationMiddleware()])
 app.include_router(courses.router, prefix='/courses')
 app.include_router(enrollments.router, prefix='/enrollments')
 app.include_router(enrollments.vacancies, prefix='/enrollments')
+app.include_router(enrollments.enroll, prefix='/enrollments')
+app.include_router(enrollments.cancel, prefix='/enrollments')
 app.include_router(orders.router, prefix='/orders')
 app.include_router(users.router, prefix='/users')
 app.include_router(users.logs, prefix='/users')

http-api/app/models.py

@@ -1,4 +1,4 @@
-from typing import Annotated
+from typing import Annotated, Literal
 from uuid import uuid4
 
 from layercake.extra_types import CnpjStr, CpfStr, NameStr
@@ -37,3 +37,11 @@ class Course(BaseModel):
     name: str
     cert: Cert | None = None
     access_period: int = 90  # 3 months
+
+
+class Enrollment(BaseModel):
+    id: UUID4 = Field(default_factory=uuid4)
+    user: User
+    course: Course
+    progress: int = Field(default=0, ge=0, le=100)
+    status: Literal['PENDING'] = 'PENDING'

http-api/app/routes/enrollments/__init__.py

@@ -5,22 +5,19 @@ from elasticsearch import Elasticsearch
 from layercake.dynamodb import (
     DynamoDBCollection,
     DynamoDBPersistenceLayer,
-    KeyPair,
     SortKey,
     TransactKey,
 )
-from pydantic import UUID4, BaseModel
 
+import elastic
 from boto3clients import dynamodb_client
 from conf import ELASTIC_CONN, ENROLLMENT_TABLE, USER_TABLE
-import elastic
-from middlewares.audit_log_middleware import AuditLogMiddleware
-from middlewares.authentication_middleware import User
-from rules.enrollment import set_status_as_canceled
 
+from .cancel import router as cancel
+from .enroll import router as enroll
 from .vacancies import router as vacancies
 
-__all__ = ['vacancies']
+__all__ = ['vacancies', 'cancel', 'enroll']
 
 
 router = Router()
@@ -62,38 +59,3 @@ def get_enrollment(id: str):
         + SortKey('tenant')
         + SortKey('cert')
     )
-
-
-class Cancel(BaseModel):
-    id: UUID4 | str
-    lock_hash: str
-    course: dict = {}
-    vacancy: dict = {}
-
-
-@router.patch(
-    '/<id>/cancel',
-    compress=True,
-    tags=['Enrollment'],
-    middlewares=[
-        AuditLogMiddleware('ENROLLMENT_CANCEL', user_collect, ('id', 'course'))
-    ],
-)
-def cancel(id: str, payload: Cancel):
-    user: User = router.context['user']
-
-    set_status_as_canceled(
-        id,
-        lock_hash=payload.lock_hash,
-        author=user.model_dump(),  # type: ignore
-        course=payload.course,  # type: ignore
-        vacancy_key=KeyPair.parse_obj(payload.vacancy),
-        persistence_layer=enrollment_layer,
-    )
-
-    return payload
-
-
-@router.post('/', compress=True, tags=['Enrollment'])
-def enroll():
-    return {}

http-api/app/rules/enrollment.py

@@ -1,10 +1,19 @@
-from typing import TypedDict
+from datetime import timedelta
+from enum import Enum
+from typing import Literal, TypedDict
 from uuid import uuid4
 
-from layercake.dateutils import now
+from layercake.dateutils import now, ttl
 from layercake.dynamodb import DynamoDBPersistenceLayer, KeyPair, TransactItems
+from layercake.strutils import md5_hash
 
 from conf import ORDER_TABLE
+from models import Course, Enrollment
+
+
+class Tenant(TypedDict):
+    id: str
+    name: str
 
 
 class Author(TypedDict):
@@ -12,10 +21,170 @@ class Author(TypedDict):
     name: str
 
 
-class Course(TypedDict):
+class Vacancy(TypedDict): ...
+
+
+class Rel(TypedDict):
     id: str
-    name: str
+    scope: Literal['ORG', 'USER', 'ENROLLMENT']
-    time_in_days: int
+
+
+class LifecycleEvents(str, Enum):
+    """Schedules lifecycle events."""
+
+    # Reminder if the user does not access within 3 days
+    REMINDER_NO_ACCESS_3_DAYS = 'schedules#reminder_no_access_3_days'
+
+    # When there is no activity 7 days after the first access
+    NO_ACTIVITY_7_DAYS = 'schedules#no_activity_7_days'
+
+    # When the access period expires
+    ACCESS_PERIOD_EXPIRED = 'schedules#access_period_expired'
+
+    # When the course certificate expires
+    CERTIFICATE_EXPIRATION = 'schedules#certificate_expiration'
+
+    # Archive the course after the certificate expires
+    COURSE_ARCHIVED = 'schedules#course_archived'
+
+
+def enroll(
+    enrollment: Enrollment,
+    *,
+    tenant: Tenant,
+    rel: tuple[Rel, ...] | Rel = (),
+    author: Author | None = None,
+    vacancy: Vacancy | None = None,
+    ensure_vacancy: bool = True,
+    persistence_layer: DynamoDBPersistenceLayer,
+) -> bool:
+    """Enrolls a user into a course and schedules lifecycle events."""
+    now_ = now()
+    user = enrollment.user
+    course = enrollment.course
+    exp_interval = course.exp_interval
+    lock_hash = md5_hash('%s%s' % (user.id, course.id))
+    ttl_date = now_ + timedelta(days=exp_interval - 30)
+
+    transact = TransactItems(persistence_layer.table_name)
+    transact.put(
+        item={
+            'sk': '0',
+            'create_date': now_,
+            'tenant__org_id': tenant['id'],
+            **enrollment.model_dump(),
+        },
+    )
+    transact.put(
+        item={
+            'id': enrollment.id,
+            'sk': 'metadata#tenant',
+            'org_id': tenant['id'],
+            'name': tenant['name'],
+            'create_date': now_,
+        },
+    )
+    transact.put(
+        item={
+            'id': enrollment.id,
+            'sk': LifecycleEvents.COURSE_ARCHIVED,
+            'create_date': now_,
+            'ttl': ttl(days=exp_interval, start_dt=now_),
+        },
+    )
+    transact.put(
+        item={
+            'id': enrollment.id,
+            'sk': LifecycleEvents.REMINDER_NO_ACCESS_3_DAYS,
+            'name': user.name,
+            'email': user.email,
+            'course': course.name,
+            'create_date': now_,
+            'ttl': ttl(days=3, start_dt=now_),
+        },
+    )
+    transact.put(
+        item={
+            'id': enrollment.id,
+            'sk': LifecycleEvents.ACCESS_PERIOD_EXPIRED,
+            'name': user.name,
+            'email': user.email,
+            'course': course.name,
+            'create_date': now_,
+            'ttl': ttl(start_dt=now_ + timedelta(days=course.access_period - 30)),
+        },
+    )
+
+    for r in rel:
+        print(r['id'])
+        transact.put(
+            item={
+                'id': enrollment.id,
+                # 'sk': 'rel#{}' % r['id'],
+                'create_date': now_,
+            },
+        )
+
+    if author:
+        transact.put(
+            item={
+                'id': enrollment.id,
+                'sk': 'metadata#author',
+                'user_id': author['id'],
+                'name': author['name'],
+                'create_date': now_,
+            },
+        )
+
+    if vacancy:
+        transact.put(
+            item={
+                'id': enrollment.id,
+                'sk': 'parent_vacancy',
+                # 'vacancy': vacancy.model_dump(),
+            }
+        )
+
+        if ensure_vacancy:
+            # Ensures that there's a vacancy
+            transact.delete(
+                key=vacancy.model_dump(),
+                cond_expr='attribute_exists(sk)',
+            )
+
+    # Add cancel policy if there is a vacancy
+    if vacancy:
+        transact.put(
+            item={
+                'id': enrollment.id,
+                'sk': 'metadata#cancel_policy',
+                'create_date': now_,
+            }
+        )
+
+    # To ensure that the user does not enroll in the same course again until
+    # the certificate expires.
+    transact.put(
+        item={
+            'id': 'metadata#lock',
+            'sk': lock_hash,
+            'enrollment_id': enrollment.id,
+            'create_date': vacancy,
+            'ttl': ttl(start_dt=ttl_date),
+        },
+        cond_expr='attribute_not_exists(sk)',
+    )
+    transact.put(
+        item={
+            'id': enrollment.id,
+            'sk': 'lock',
+            'hash': lock_hash,
+            'create_date': vacancy,
+            'ttl': ttl(start_dt=ttl_date),
+        },
+    )
+
+    return persistence_layer.transact_write_items(transact)
 
 
 def set_status_as_canceled(

http-api/tests/routes/test_enrollments.py

@@ -11,6 +11,42 @@ from layercake.dynamodb import (
 from ..conftest import HttpApiProxy, LambdaContext
 
 
+def test_enroll(
+    mock_app,
+    dynamodb_seeds,
+    dynamodb_persistence_layer: DynamoDBPersistenceLayer,
+    http_api_proxy: HttpApiProxy,
+    lambda_context: LambdaContext,
+):
+    r = mock_app.lambda_handler(
+        http_api_proxy(
+            raw_path='/enrollments',
+            method=HTTPMethod.POST,
+            headers={'X-Tenant': 'cJtK9SsnJhKPyxESe7g3DG'},
+            body={
+                'items': [
+                    {
+                        'user': {
+                            'id': '5OxmMjL-ujoR5IMGegQz',
+                            'name': 'Sérgio R Siqueira',
+                            'email': 'sergio@somosbeta.com.br',
+                            'cpf': '07879819908',
+                        },
+                        'course': {
+                            'id': '6d69a34a-cefd-40aa-a89b-dceb694c3e61',
+                            'name': 'pytest',
+                        },
+                    },
+                ],
+            },
+        ),
+        lambda_context,
+    )
+
+    assert r['statusCode'] == HTTPStatus.OK
+    print(r)
+
+
 def test_vacancies(
     mock_app,
     dynamodb_seeds,
@@ -28,7 +64,6 @@ def test_vacancies(
     )
 
     assert r['statusCode'] == HTTPStatus.OK
-    print(r)
 
 
 def test_cancel_enrollment(

http-api/tests/routes/test_users.py

@@ -138,7 +138,6 @@ def test_get_orgs(
     )
 
     assert r['statusCode'] == HTTPStatus.OK
-    print(r)
 
 
 def test_get_logs(

http-api/uv.lock

@@ -259,36 +259,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/52/26/9f53293ff4cc1d47d98367ce045ca2e62746d6be74a5c6851a474eabf59b/coverage-7.7.1-py3-none-any.whl", hash = "sha256:822fa99dd1ac686061e1219b67868e25d9757989cf2259f735a4802497d6da31", size = 203006, upload-time = "2025-03-21T17:23:56.378Z" },
 ]
 
-[[package]]
-name = "crc32c"
-version = "2.7.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/7f/4c/4e40cc26347ac8254d3f25b9f94710b8e8df24ee4dddc1ba41907a88a94d/crc32c-2.7.1.tar.gz", hash = "sha256:f91b144a21eef834d64178e01982bb9179c354b3e9e5f4c803b0e5096384968c", size = 45712, upload-time = "2024-09-24T06:20:17.553Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/bf/98/1a6d60d5b3b5edc8382777b64100343cb4aa6a7e172fae4a6cfcb8ebbbd9/crc32c-2.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:24949bffb06fc411cc18188d33357923cb935273642164d0bb37a5f375654169", size = 49567, upload-time = "2024-09-24T06:18:44.485Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/56/0dd652d4e950e6348bbf16b964b3325e4ad8220470774128fc0b0dd069cb/crc32c-2.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d5d326e7e118d4fa60187770d86b66af2fdfc63ce9eeb265f0d3e7d49bebe0b", size = 37018, upload-time = "2024-09-24T06:18:45.434Z" },
-    { url = "https://files.pythonhosted.org/packages/47/02/2bd65fdef10139b6a802d83a7f966b7750fe5ffb1042f7cbe5dbb6403869/crc32c-2.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ba110df60c64c8e2d77a9425b982a520ccdb7abe42f06604f4d98a45bb1fff62", size = 35374, upload-time = "2024-09-24T06:18:46.304Z" },
-    { url = "https://files.pythonhosted.org/packages/a9/0d/3e797d1ed92d357a6a4c5b41cea15a538b27a8fdf18c7863747eb50b73ad/crc32c-2.7.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c277f9d16a3283e064d54854af0976b72abaa89824955579b2b3f37444f89aae", size = 54641, upload-time = "2024-09-24T06:18:47.207Z" },
-    { url = "https://files.pythonhosted.org/packages/a7/d3/4ddeef755caaa75680c559562b6c71f5910fee4c4f3a2eb5ea8b57f0e48c/crc32c-2.7.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:881af0478a01331244e27197356929edbdeaef6a9f81b5c6bacfea18d2139289", size = 52338, upload-time = "2024-09-24T06:18:49.31Z" },
-    { url = "https://files.pythonhosted.org/packages/01/cf/32f019be5de9f6e180926a50ee5f08648e686c7d9a59f2c5d0806a77b1c7/crc32c-2.7.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:724d5ff4d29ff093a983ae656be3307093706d850ea2a233bf29fcacc335d945", size = 53447, upload-time = "2024-09-24T06:18:50.296Z" },
-    { url = "https://files.pythonhosted.org/packages/b2/8b/92f3f62f3bafe8f7ab4af7bfb7246dc683fd11ec0d6dfb73f91e09079f69/crc32c-2.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b2416c4d88696ac322632555c0f81ab35e15f154bc96055da6cf110d642dbc10", size = 54484, upload-time = "2024-09-24T06:18:51.311Z" },
-    { url = "https://files.pythonhosted.org/packages/98/b2/113a50f8781f76af5ac65ffdb907e72bddbe974de8e02247f0d58bc48040/crc32c-2.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:60254251b88ec9b9795215f0f9ec015a6b5eef8b2c5fba1267c672d83c78fc02", size = 52703, upload-time = "2024-09-24T06:18:52.488Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/6c/309229e9acda8cf36a8ff4061d70b54d905f79b7037e16883ce6590a24ab/crc32c-2.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:edefc0e46f3c37372183f70338e5bdee42f6789b62fcd36ec53aa933e9dfbeaf", size = 53367, upload-time = "2024-09-24T06:18:53.49Z" },
-    { url = "https://files.pythonhosted.org/packages/b5/2a/6c6324d920396e1bd9f3efbe8753da071be0ca52bd22d6c82d446b8d6975/crc32c-2.7.1-cp313-cp313-win32.whl", hash = "sha256:813af8111218970fe2adb833c5e5239f091b9c9e76f03b4dd91aaba86e99b499", size = 38377, upload-time = "2024-09-24T06:18:54.487Z" },
-    { url = "https://files.pythonhosted.org/packages/db/a0/f01ccfab538db07ef3f6b4ede46357ff147a81dd4f3c59ca6a34c791a549/crc32c-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:7d9ede7be8e4ec1c9e90aaf6884decbeef10e3473e6ddac032706d710cab5888", size = 39803, upload-time = "2024-09-24T06:18:55.419Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/80/61dcae7568b33acfde70c9d651c7d891c0c578c39cc049107c1cf61f1367/crc32c-2.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db9ac92294284b22521356715784b91cc9094eee42a5282ab281b872510d1831", size = 49386, upload-time = "2024-09-24T06:18:56.813Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/f1/80f17c089799ab2b4c247443bdd101d6ceda30c46d7f193e16b5ca29c5a0/crc32c-2.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:8fcd7f2f29a30dc92af64a9ee3d38bde0c82bd20ad939999427aac94bbd87373", size = 36937, upload-time = "2024-09-24T06:18:57.77Z" },
-    { url = "https://files.pythonhosted.org/packages/63/42/5fcfc71a3de493d920fd2590843762a2749981ea56b802b380e5df82309d/crc32c-2.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5c056ef043393085523e149276a7ce0cb534b872e04f3e20d74d9a94a75c0ad7", size = 35292, upload-time = "2024-09-24T06:18:58.676Z" },
-    { url = "https://files.pythonhosted.org/packages/03/de/fef962e898a953558fe1c55141644553e84ef4190693a31244c59a0856c7/crc32c-2.7.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03a92551a343702629af91f78d205801219692b6909f8fa126b830e332bfb0e0", size = 54223, upload-time = "2024-09-24T06:18:59.675Z" },
-    { url = "https://files.pythonhosted.org/packages/21/14/fceca1a6f45c0a1814fe8602a65657b75c27425162445925ba87438cad6b/crc32c-2.7.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb9424ec1a8ca54763155a703e763bcede82e6569fe94762614bb2de1412d4e1", size = 51588, upload-time = "2024-09-24T06:19:00.938Z" },
-    { url = "https://files.pythonhosted.org/packages/13/3b/13d40a7dfbf9ef05c84a0da45544ee72080dca4ce090679e5105689984bd/crc32c-2.7.1-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88732070f6175530db04e0bb36880ac45c33d49f8ac43fa0e50cfb1830049d23", size = 52678, upload-time = "2024-09-24T06:19:02.661Z" },
-    { url = "https://files.pythonhosted.org/packages/36/09/65ffc4fb9fa60ff6714eeb50a92284a4525e5943f0b040b572c0c76368c1/crc32c-2.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:57a20dfc27995f568f64775eea2bbb58ae269f1a1144561df5e4a4955f79db32", size = 53847, upload-time = "2024-09-24T06:19:03.705Z" },
-    { url = "https://files.pythonhosted.org/packages/24/71/938e926085b7288da052db7c84416f3ce25e71baf7ab5b63824c7bcb6f22/crc32c-2.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:f7186d098bfd2cff25eac6880b7c7ad80431b90610036131c1c7dd0eab42a332", size = 51860, upload-time = "2024-09-24T06:19:04.726Z" },
-    { url = "https://files.pythonhosted.org/packages/3c/d8/4526d5380189d6f2fa27256c204100f30214fe402f47cf6e9fb9a91ab890/crc32c-2.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:55a77e29a265418fa34bef15bd0f2c60afae5348988aaf35ed163b4bbf93cf37", size = 52508, upload-time = "2024-09-24T06:19:05.731Z" },
-    { url = "https://files.pythonhosted.org/packages/19/30/15f7e35176488b77e5b88751947d321d603fccac273099ace27c7b2d50a6/crc32c-2.7.1-cp313-cp313t-win32.whl", hash = "sha256:ae38a4b6aa361595d81cab441405fbee905c72273e80a1c010fb878ae77ac769", size = 38319, upload-time = "2024-09-24T06:19:07.233Z" },
-    { url = "https://files.pythonhosted.org/packages/19/c4/0b3eee04dac195f4730d102d7a9fbea894ae7a32ce075f84336df96a385d/crc32c-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:eee2a43b663feb6c79a6c1c6e5eae339c2b72cfac31ee54ec0209fa736cf7ee5", size = 39781, upload-time = "2024-09-24T06:19:08.182Z" },
-]
-
 [[package]]
 name = "cryptography"
 version = "44.0.2"
@@ -346,18 +316,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632, upload-time = "2024-10-05T20:14:57.687Z" },
 ]
 
-[[package]]
-name = "donfig"
-version = "0.8.1.post1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "pyyaml" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz", hash = "sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", size = 19506, upload-time = "2024-05-23T14:14:31.513Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl", hash = "sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", size = 21592, upload-time = "2024-05-23T14:13:55.283Z" },
-]
-
 [[package]]
 name = "elastic-transport"
 version = "8.17.1"
@@ -564,56 +522,50 @@ wheels = [
 
 [[package]]
 name = "layercake"
-version = "0.2.11"
+version = "0.2.16"
 source = { directory = "../layercake" }
 dependencies = [
     { name = "arnparse" },
     { name = "aws-lambda-powertools", extra = ["all"] },
-    { name = "boto3" },
     { name = "elasticsearch" },
     { name = "elasticsearch-dsl" },
     { name = "ftfy" },
     { name = "glom" },
     { name = "meilisearch" },
-    { name = "numpy" },
     { name = "orjson" },
-    { name = "pandas" },
     { name = "pycpfcnpj" },
     { name = "pydantic", extra = ["email"] },
     { name = "pydantic-extra-types" },
     { name = "pytz" },
     { name = "requests" },
+    { name = "smart-open", extra = ["s3"] },
     { name = "weasyprint" },
-    { name = "zarr" },
 ]
 
 [package.metadata]
 requires-dist = [
     { name = "arnparse", specifier = ">=0.0.2" },
     { name = "aws-lambda-powertools", extras = ["all"], specifier = ">=3.8.0" },
-    { name = "boto3", specifier = ">=1.37.16" },
     { name = "elasticsearch", specifier = ">=8.17.2" },
     { name = "elasticsearch-dsl", specifier = ">=8.17.1" },
     { name = "ftfy", specifier = ">=6.3.1" },
     { name = "glom", specifier = ">=24.11.0" },
     { name = "meilisearch", specifier = ">=0.34.0" },
-    { name = "numpy", specifier = ">=2.2.5" },
     { name = "orjson", specifier = ">=3.10.15" },
-    { name = "pandas", specifier = ">=2.2.3" },
     { name = "pycpfcnpj", specifier = ">=1.8" },
     { name = "pydantic", extras = ["email"], specifier = ">=2.10.6" },
     { name = "pydantic-extra-types", specifier = ">=2.10.3" },
     { name = "pytz", specifier = ">=2025.1" },
     { name = "requests", specifier = ">=2.32.3" },
+    { name = "smart-open", extras = ["s3"], specifier = ">=7.1.0" },
     { name = "weasyprint", specifier = ">=65.0" },
-    { name = "zarr", specifier = ">=3.0.7" },
 ]
 
 [package.metadata.requires-dev]
 dev = [
+    { name = "boto3", specifier = ">=1.37.16" },
     { name = "boto3-stubs", extras = ["essential"], specifier = ">=1.37.33" },
     { name = "jsonlines", specifier = ">=4.0.0" },
-    { name = "mkdocstrings", extras = ["python"], specifier = ">=0.29.0" },
     { name = "pytest", specifier = ">=8.3.5" },
     { name = "pytest-cov", specifier = ">=6.0.0" },
     { name = "pytest-env", specifier = ">=1.1.5" },
@@ -633,56 +585,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/2f/264c07a3f488260ea36c78cbc201b76e6baf9ef92e0c7f78657a6a5e5f22/meilisearch-0.34.0-py3-none-any.whl", hash = "sha256:fae8ad2a15d12c27fa0a1fff2ae2e4e3e2e22b869950408d63c87e2c095a9f61", size = 24373, upload-time = "2025-02-18T05:50:32.73Z" },
 ]
 
-[[package]]
-name = "numcodecs"
-version = "0.16.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy" },
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/bd/3d/54b07213c1c3b489f3be23e11f4d2c81d2e682852fc93b2bf37d5efd5664/numcodecs-0.16.0.tar.gz", hash = "sha256:f71229a951c4991ab226df4147f3780c988d1cecf1309e388d9e57bcba827387", size = 6268216, upload-time = "2025-04-07T15:00:11.168Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e6/81/eaf0eb5f60123840e31081ed4226d326eb035d0284e49596331b9f76995f/numcodecs-0.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a28f3f2c1aab21417bec52932d453eedb8323a07088be72edc2a440426d823ae", size = 1672497, upload-time = "2025-04-07T15:00:04.027Z" },
-    { url = "https://files.pythonhosted.org/packages/da/b8/e15f3dc5c5763efbfceed34100bc045d397dc02f277bc128845ac2e751f2/numcodecs-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0f5203b86e57bc6d0dad8f0719f10404fcadeabb5f112205d4e926075a4e1388", size = 1170472, upload-time = "2025-04-07T15:00:05.305Z" },
-    { url = "https://files.pythonhosted.org/packages/e3/16/773ae1edb0e63e16c539a06b30d3d117fa0de475400576da2b0cfa682831/numcodecs-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:881b1650938feb395b7157fd46bde683aa6cfde0616a31fe47972f1d0ea8ba37", size = 8188333, upload-time = "2025-04-07T15:00:06.478Z" },
-    { url = "https://files.pythonhosted.org/packages/25/df/6cbdd9842f9da811367773bdcbc12b196aaf4ad84bf3395b366931b32130/numcodecs-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271d07bfa230e02dec7ba93d5776b5032aecf1d4c20d52a93858e957d44ae722", size = 8752531, upload-time = "2025-04-07T15:00:07.982Z" },
-    { url = "https://files.pythonhosted.org/packages/52/e7/a8df8fc7e7ee574a66ca706916e486448c1f1f90cc3dcc660b3d6d846e72/numcodecs-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:7e6f501d7ac78107d80a2f4798e81b38584929cab1d2c7c03789610b9a39826a", size = 811488, upload-time = "2025-04-07T15:00:09.649Z" },
-]
-
-[package.optional-dependencies]
-crc32c = [
-    { name = "crc32c" },
-]
-
-[[package]]
-name = "numpy"
-version = "2.2.5"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/dc/b2/ce4b867d8cd9c0ee84938ae1e6a6f7926ebf928c9090d036fc3c6a04f946/numpy-2.2.5.tar.gz", hash = "sha256:a9c0d994680cd991b1cb772e8b297340085466a6fe964bc9d4e80f5e2f43c291", size = 20273920, upload-time = "2025-04-19T23:27:42.561Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e2/a0/0aa7f0f4509a2e07bd7a509042967c2fab635690d4f48c6c7b3afd4f448c/numpy-2.2.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:059b51b658f4414fff78c6d7b1b4e18283ab5fa56d270ff212d5ba0c561846f4", size = 20935102, upload-time = "2025-04-19T22:41:16.234Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/e4/a6a9f4537542912ec513185396fce52cdd45bdcf3e9d921ab02a93ca5aa9/numpy-2.2.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:47f9ed103af0bc63182609044b0490747e03bd20a67e391192dde119bf43d52f", size = 14191709, upload-time = "2025-04-19T22:41:38.472Z" },
-    { url = "https://files.pythonhosted.org/packages/be/65/72f3186b6050bbfe9c43cb81f9df59ae63603491d36179cf7a7c8d216758/numpy-2.2.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:261a1ef047751bb02f29dfe337230b5882b54521ca121fc7f62668133cb119c9", size = 5149173, upload-time = "2025-04-19T22:41:47.823Z" },
-    { url = "https://files.pythonhosted.org/packages/e5/e9/83e7a9432378dde5802651307ae5e9ea07bb72b416728202218cd4da2801/numpy-2.2.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4520caa3807c1ceb005d125a75e715567806fed67e315cea619d5ec6e75a4191", size = 6684502, upload-time = "2025-04-19T22:41:58.689Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/27/b80da6c762394c8ee516b74c1f686fcd16c8f23b14de57ba0cad7349d1d2/numpy-2.2.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d14b17b9be5f9c9301f43d2e2a4886a33b53f4e6fdf9ca2f4cc60aeeee76372", size = 14084417, upload-time = "2025-04-19T22:42:19.897Z" },
-    { url = "https://files.pythonhosted.org/packages/aa/fc/ebfd32c3e124e6a1043e19c0ab0769818aa69050ce5589b63d05ff185526/numpy-2.2.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ba321813a00e508d5421104464510cc962a6f791aa2fca1c97b1e65027da80d", size = 16133807, upload-time = "2025-04-19T22:42:44.433Z" },
-    { url = "https://files.pythonhosted.org/packages/bf/9b/4cc171a0acbe4666f7775cfd21d4eb6bb1d36d3a0431f48a73e9212d2278/numpy-2.2.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4cbdef3ddf777423060c6f81b5694bad2dc9675f110c4b2a60dc0181543fac7", size = 15575611, upload-time = "2025-04-19T22:43:09.928Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/45/40f4135341850df48f8edcf949cf47b523c404b712774f8855a64c96ef29/numpy-2.2.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:54088a5a147ab71a8e7fdfd8c3601972751ded0739c6b696ad9cb0343e21ab73", size = 17895747, upload-time = "2025-04-19T22:43:36.983Z" },
-    { url = "https://files.pythonhosted.org/packages/f8/4c/b32a17a46f0ffbde8cc82df6d3daeaf4f552e346df143e1b188a701a8f09/numpy-2.2.5-cp313-cp313-win32.whl", hash = "sha256:c8b82a55ef86a2d8e81b63da85e55f5537d2157165be1cb2ce7cfa57b6aef38b", size = 6309594, upload-time = "2025-04-19T22:47:10.523Z" },
-    { url = "https://files.pythonhosted.org/packages/13/ae/72e6276feb9ef06787365b05915bfdb057d01fceb4a43cb80978e518d79b/numpy-2.2.5-cp313-cp313-win_amd64.whl", hash = "sha256:d8882a829fd779f0f43998e931c466802a77ca1ee0fe25a3abe50278616b1471", size = 12638356, upload-time = "2025-04-19T22:47:30.253Z" },
-    { url = "https://files.pythonhosted.org/packages/79/56/be8b85a9f2adb688e7ded6324e20149a03541d2b3297c3ffc1a73f46dedb/numpy-2.2.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e8b025c351b9f0e8b5436cf28a07fa4ac0204d67b38f01433ac7f9b870fa38c6", size = 20963778, upload-time = "2025-04-19T22:44:09.251Z" },
-    { url = "https://files.pythonhosted.org/packages/ff/77/19c5e62d55bff507a18c3cdff82e94fe174957bad25860a991cac719d3ab/numpy-2.2.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8dfa94b6a4374e7851bbb6f35e6ded2120b752b063e6acdd3157e4d2bb922eba", size = 14207279, upload-time = "2025-04-19T22:44:31.383Z" },
-    { url = "https://files.pythonhosted.org/packages/75/22/aa11f22dc11ff4ffe4e849d9b63bbe8d4ac6d5fae85ddaa67dfe43be3e76/numpy-2.2.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:97c8425d4e26437e65e1d189d22dff4a079b747ff9c2788057bfb8114ce1e133", size = 5199247, upload-time = "2025-04-19T22:44:40.361Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/6c/12d5e760fc62c08eded0394f62039f5a9857f758312bf01632a81d841459/numpy-2.2.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:352d330048c055ea6db701130abc48a21bec690a8d38f8284e00fab256dc1376", size = 6711087, upload-time = "2025-04-19T22:44:51.188Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/94/ece8280cf4218b2bee5cec9567629e61e51b4be501e5c6840ceb593db945/numpy-2.2.5-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b4c0773b6ada798f51f0f8e30c054d32304ccc6e9c5d93d46cb26f3d385ab19", size = 14059964, upload-time = "2025-04-19T22:45:12.451Z" },
-    { url = "https://files.pythonhosted.org/packages/39/41/c5377dac0514aaeec69115830a39d905b1882819c8e65d97fc60e177e19e/numpy-2.2.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55f09e00d4dccd76b179c0f18a44f041e5332fd0e022886ba1c0bbf3ea4a18d0", size = 16121214, upload-time = "2025-04-19T22:45:37.734Z" },
-    { url = "https://files.pythonhosted.org/packages/db/54/3b9f89a943257bc8e187145c6bc0eb8e3d615655f7b14e9b490b053e8149/numpy-2.2.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:02f226baeefa68f7d579e213d0f3493496397d8f1cff5e2b222af274c86a552a", size = 15575788, upload-time = "2025-04-19T22:46:01.908Z" },
-    { url = "https://files.pythonhosted.org/packages/b1/c4/2e407e85df35b29f79945751b8f8e671057a13a376497d7fb2151ba0d290/numpy-2.2.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c26843fd58f65da9491165072da2cccc372530681de481ef670dcc8e27cfb066", size = 17893672, upload-time = "2025-04-19T22:46:28.585Z" },
-    { url = "https://files.pythonhosted.org/packages/29/7e/d0b44e129d038dba453f00d0e29ebd6eaf2f06055d72b95b9947998aca14/numpy-2.2.5-cp313-cp313t-win32.whl", hash = "sha256:1a161c2c79ab30fe4501d5a2bbfe8b162490757cf90b7f05be8b80bc02f7bb8e", size = 6377102, upload-time = "2025-04-19T22:46:39.949Z" },
-    { url = "https://files.pythonhosted.org/packages/63/be/b85e4aa4bf42c6502851b971f1c326d583fcc68227385f92089cf50a7b45/numpy-2.2.5-cp313-cp313t-win_amd64.whl", hash = "sha256:d403c84991b5ad291d3809bace5e85f4bbf44a04bdc9a88ed2bb1807b3360bb8", size = 12750096, upload-time = "2025-04-19T22:47:00.147Z" },
-]
-
 [[package]]
 name = "orjson"
 version = "3.10.16"
@@ -714,33 +616,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451, upload-time = "2024-11-08T09:47:44.722Z" },
 ]
 
-[[package]]
-name = "pandas"
-version = "2.2.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "numpy" },
-    { name = "python-dateutil" },
-    { name = "pytz" },
-    { name = "tzdata" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213, upload-time = "2024-09-20T13:10:04.827Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/64/22/3b8f4e0ed70644e85cfdcd57454686b9057c6c38d2f74fe4b8bc2527214a/pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015", size = 12477643, upload-time = "2024-09-20T13:09:25.522Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/93/b3f5d1838500e22c8d793625da672f3eec046b1a99257666c94446969282/pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28", size = 11281573, upload-time = "2024-09-20T13:09:28.012Z" },
-    { url = "https://files.pythonhosted.org/packages/f5/94/6c79b07f0e5aab1dcfa35a75f4817f5c4f677931d4234afcd75f0e6a66ca/pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0", size = 15196085, upload-time = "2024-09-20T19:02:10.451Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/31/aa8da88ca0eadbabd0a639788a6da13bb2ff6edbbb9f29aa786450a30a91/pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24", size = 12711809, upload-time = "2024-09-20T13:09:30.814Z" },
-    { url = "https://files.pythonhosted.org/packages/ee/7c/c6dbdb0cb2a4344cacfb8de1c5808ca885b2e4dcfde8008266608f9372af/pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659", size = 16356316, upload-time = "2024-09-20T19:02:13.825Z" },
-    { url = "https://files.pythonhosted.org/packages/57/b7/8b757e7d92023b832869fa8881a992696a0bfe2e26f72c9ae9f255988d42/pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb", size = 14022055, upload-time = "2024-09-20T13:09:33.462Z" },
-    { url = "https://files.pythonhosted.org/packages/3b/bc/4b18e2b8c002572c5a441a64826252ce5da2aa738855747247a971988043/pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d", size = 11481175, upload-time = "2024-09-20T13:09:35.871Z" },
-    { url = "https://files.pythonhosted.org/packages/76/a3/a5d88146815e972d40d19247b2c162e88213ef51c7c25993942c39dbf41d/pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468", size = 12615650, upload-time = "2024-09-20T13:09:38.685Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/8c/f0fd18f6140ddafc0c24122c8a964e48294acc579d47def376fef12bcb4a/pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18", size = 11290177, upload-time = "2024-09-20T13:09:41.141Z" },
-    { url = "https://files.pythonhosted.org/packages/ed/f9/e995754eab9c0f14c6777401f7eece0943840b7a9fc932221c19d1abee9f/pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2", size = 14651526, upload-time = "2024-09-20T19:02:16.905Z" },
-    { url = "https://files.pythonhosted.org/packages/25/b0/98d6ae2e1abac4f35230aa756005e8654649d305df9a28b16b9ae4353bff/pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4", size = 11871013, upload-time = "2024-09-20T13:09:44.39Z" },
-    { url = "https://files.pythonhosted.org/packages/cc/57/0f72a10f9db6a4628744c8e8f0df4e6e21de01212c7c981d31e50ffc8328/pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d", size = 15711620, upload-time = "2024-09-20T19:02:20.639Z" },
-    { url = "https://files.pythonhosted.org/packages/ab/5f/b38085618b950b79d2d9164a711c52b10aefc0ae6833b96f626b7021b2ed/pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a", size = 13098436, upload-time = "2024-09-20T13:09:48.112Z" },
-]
-
 [[package]]
 name = "pillow"
 version = "11.1.0"
@@ -950,23 +825,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" },
 ]
 
-[[package]]
-name = "pyyaml"
-version = "6.0.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631, upload-time = "2024-08-06T20:33:50.674Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309, upload-time = "2024-08-06T20:32:43.4Z" },
-    { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679, upload-time = "2024-08-06T20:32:44.801Z" },
-    { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428, upload-time = "2024-08-06T20:32:46.432Z" },
-    { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361, upload-time = "2024-08-06T20:32:51.188Z" },
-    { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523, upload-time = "2024-08-06T20:32:53.019Z" },
-    { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660, upload-time = "2024-08-06T20:32:54.708Z" },
-    { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597, upload-time = "2024-08-06T20:32:56.985Z" },
-    { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527, upload-time = "2024-08-06T20:33:03.001Z" },
-    { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" },
-]
-
 [[package]]
 name = "requests"
 version = "2.32.3"
@@ -1028,6 +886,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
 ]
 
+[[package]]
+name = "smart-open"
+version = "7.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "wrapt" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/21/30/1f41c3d3b8cec82024b4b277bfd4e5b18b765ae7279eb9871fa25c503778/smart_open-7.1.0.tar.gz", hash = "sha256:a4f09f84f0f6d3637c6543aca7b5487438877a21360e7368ccf1f704789752ba", size = 72044, upload-time = "2024-12-17T13:19:17.71Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7a/18/9a8d9f01957aa1f8bbc5676d54c2e33102d247e146c1a3679d3bd5cc2e3a/smart_open-7.1.0-py3-none-any.whl", hash = "sha256:4b8489bb6058196258bafe901730c7db0dcf4f083f316e97269c66f45502055b", size = 61746, upload-time = "2024-12-17T13:19:21.076Z" },
+]
+
+[package.optional-dependencies]
+s3 = [
+    { name = "boto3" },
+]
+
 [[package]]
 name = "tinycss2"
 version = "1.4.0"
@@ -1073,15 +948,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/86/39b65d676ec5732de17b7e3c476e45bb80ec64eb50737a8dce1a4178aba1/typing_extensions-4.13.0-py3-none-any.whl", hash = "sha256:c8dd92cc0d6425a97c18fbb9d1954e5ff92c1ca881a309c45f06ebc0b79058e5", size = 45683, upload-time = "2025-03-26T03:49:40.35Z" },
 ]
 
-[[package]]
-name = "tzdata"
-version = "2025.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" },
-]
-
 [[package]]
 name = "urllib3"
 version = "2.3.0"
@@ -1159,22 +1025,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594, upload-time = "2025-01-14T10:35:44.018Z" },
 ]
 
-[[package]]
-name = "zarr"
-version = "3.0.7"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "donfig" },
-    { name = "numcodecs", extra = ["crc32c"] },
-    { name = "numpy" },
-    { name = "packaging" },
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/e3/83/ab85d1517a5a08693751fb04919f270757219c6071ae12a8c4588c7e815e/zarr-3.0.7.tar.gz", hash = "sha256:c06da406e08da9565ba368ac0337a4466d7743f643acac2873beee4458b6e5e1", size = 253194, upload-time = "2025-04-21T13:43:24.733Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/0d/67/971d9a661b66cc1c2fe2472e38085f5025a393ee1c1822b0155cc3f357b7/zarr-3.0.7-py3-none-any.whl", hash = "sha256:db0ca28f12a59be681813ac719e4d5436591e2b315e89c5ae6e2d255715be960", size = 203937, upload-time = "2025-04-21T13:43:23.214Z" },
-]
-
 [[package]]
 name = "zopfli"
 version = "0.2.3.post1"

layercake/layercake/batch.py

@@ -0,0 +1,118 @@
+import inspect
+import logging
+from contextlib import AbstractContextManager
+from enum import Enum
+from typing import Any, Callable, NamedTuple, Self, Sequence
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+class Status(Enum):
+    SUCCESS = 'success'
+    FAIL = 'fail'
+
+
+class Result(NamedTuple):
+    status: Status
+    cause: Any
+    record: Any
+
+
+class BatchProcessor(AbstractContextManager):
+    """
+    This class provides a structured way to process a sequence of records,
+    capturing both successful and failed executions along with exception details.
+
+    Batch processing utility inspired by AWS Lambda Powertools for Python:
+    https://github.com/aws-powertools/powertools-lambda-python/blob/develop/aws_lambda_powertools/utilities/batch/base.py
+
+    Attributes
+    ----------
+    successes : list
+        List of successfully processed records.
+    failures : list
+        List of records that failed to process.
+    exceptions : list
+        List of exceptions raised during processing.
+
+    Examples
+    --------
+    >>> def handler(record, context):
+    ...     if record % 2 == 0:
+    ...         return record * 2
+    ...     else:
+    ...         raise ValueError("Odd number not allowed")
+    ...
+    >>> records = [1, 2, 3, 4]
+    >>> processor = BatchProcessor()
+    >>> with processor(records, handler) as p:
+    ...     results = p.process()
+    ...
+    >>> for result in results:
+    ...     print(result.status, result.cause, result.record)
+    Status.FAIL ValueError: Odd number not allowed 1
+    Status.SUCCESS 4 2
+    Status.FAIL ValueError: Odd number not allowed 3
+    Status.SUCCESS 8 4
+    """
+
+    def __init__(self) -> None:
+        self.successes: list[Any] = []
+        self.failures: list[Any] = []
+        self.exceptions: list[BaseException] = []
+
+    def __call__(
+        self,
+        records: Sequence[Any],
+        handler: Callable[..., Any],
+        context: dict[str, Any] | None = None,
+    ) -> Self:
+        self.records = records
+        self.handler = handler
+
+        if context is None:
+            self._handler_accepts_context = False
+        else:
+            self.context = context
+            self._handler_accepts_context = (
+                'context' in inspect.signature(self.handler).parameters
+            )
+
+        return self
+
+    def __enter__(self):
+        """Remove results from previous execution."""
+        self.successes.clear()
+        self.failures.clear()
+        self.exceptions.clear()
+
+        return self
+
+    def __exit__(self, *exc_details) -> None:
+        pass
+
+    def process(self) -> Sequence[Result]:
+        return tuple(self._process_record(record) for record in self.records)
+
+    def _process_record(self, record: Any) -> Result:
+        """
+        Processes a single record using the handler function with optional context.
+        Returns a Result indicating success or failure.
+        """
+        try:
+            if self._handler_accepts_context:
+                result = self.handler(record, self.context)
+            else:
+                result = self.handler(record)
+
+            self.successes.append(record)
+            return Result(Status.SUCCESS, result, record)
+        except Exception as exc:
+            exc_str = f'{type(exc).__name__}: {exc}'
+            logger.debug(f'Record processing exception: {exc_str}')
+
+            self.exceptions.append(exc)
+            self.failures.append(record)
+
+            return Result(Status.FAIL, exc_str, record)

layercake/layercake/strutils.py

@@ -0,0 +1,18 @@
+import hashlib
+
+
+def first_word(s: str) -> str:
+    """Returns the first word from a string."""
+    return s.split(' ', 1)[0]
+
+
+def truncate_str(s: str, maxlen: int = 30) -> str:
+    """Truncates a string to fit within a maximum length, adding '...' if truncated."""
+    if len(s) <= maxlen:
+        return s
+    return s[: maxlen - 3] + '...'
+
+
+def md5_hash(s: str) -> str:
+    """Computes the MD5 hash of a string."""
+    return hashlib.md5(s.encode()).hexdigest()

layercake/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "layercake"
-version = "0.2.15"
+version = "0.2.17"
 description = "Packages shared dependencies to optimize deployment and ensure consistency across functions."
 readme = "README.md"
 authors = [

layercake/tests/test_batch.py

@@ -0,0 +1,56 @@
+from layercake.batch import BatchProcessor, Result, Status
+
+processor = BatchProcessor()
+
+
+def test_batch():
+    def record_handler(record: bool):
+        if record:
+            return True
+        raise ValueError('Invalid record')
+
+    records = (
+        True,
+        True,
+        False,
+    )
+
+    with processor(records=records, handler=record_handler) as p:
+        processed_messages = p.process()
+
+        assert processed_messages == (
+            Result(Status.SUCCESS, True, True),
+            Result(Status.SUCCESS, True, True),
+            Result(Status.FAIL, 'ValueError: Invalid record', False),
+        )
+
+    assert processor.successes == [True, True]
+    assert processor.failures == [False]
+
+    with processor(records=(False,), handler=record_handler):
+        processed_messages = processor.process()
+
+        assert processed_messages == (
+            Result(Status.FAIL, 'ValueError: Invalid record', False),
+        )
+
+    assert processor.successes == []
+    assert processor.failures == [False]
+
+
+def test_batch_context():
+    def record_handler(val: int, context: dict):
+        return val * context['multiplier']
+
+    with processor(
+        records=(2, 3, 4),
+        handler=record_handler,
+        context={'multiplier': 2},
+    ):
+        processed_messages = processor.process()
+
+        assert processed_messages == (
+            Result(Status.SUCCESS, 4, 2),
+            Result(Status.SUCCESS, 6, 3),
+            Result(Status.SUCCESS, 8, 4),
+        )

layercake/uv.lock

@@ -589,7 +589,7 @@ wheels = [
 
 [[package]]
 name = "layercake"
-version = "0.2.14"
+version = "0.2.17"
 source = { editable = "." }
 dependencies = [
     { name = "arnparse" },