sergio/saladeaula.digital
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add datetable cnfcnpj

Browse Source
This commit is contained in:
Sérgio Rafael Siqueira
2025-11-24 11:45:53 -03:00
parent 5b1ba9e9c7
commit 78ad183e61
29 changed files with 828 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
users-events/tests/events/batch/test_chunks_into_users.py Normal file
View File

@@ -0,0 +1,47 @@
import pprint
from layercake.dynamodb import DynamoDBPersistenceLayer, KeyPair
import events.batch.chunks_into_users as app
def test_chunk_csv(
dynamodb_persistence_layer: DynamoDBPersistenceLayer,
lambda_context,
):
pk = 'BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461'
file_sk = 'FILE#2025-11-13T16:04:53.024743'
event = {
'detail': {
'new_image': {
'id': pk,
'sk': 'CHUNK#START#0#END#4885',
'weight': 100,
'created_at': '2025-11-20T19:00:41.896001-03:00',
'file_sk': file_sk,
's3_uri': 's3://saladeaula.digital/samples/users.csv',
'columns': {
'1:name',
'2:email',
'3:cpf',
},
'org': {
'id': '1411844c-10d6-456e-959d-e91775145461',
'name': 'EDUSEG',
'cnpj': '15608435000190',
},
},
},
}
assert app.lambda_handler(event, lambda_context) # type: ignore
r = dynamodb_persistence_layer.collection.query(
KeyPair(
pk=pk,
sk=f'REPORTING#{file_sk}',
),
limit=100,
)
pprint.pp(r['items'])
assert 26 == len(r['items'])

26
users-events/tests/events/batch/test_csv_into_chunks.py
View File

@@ -1,13 +1,37 @@
from layercake.dateutils import now
from layercake.dynamodb import DynamoDBPersistenceLayer, PartitionKey
import events.batch.csv_into_chunks as app
def test_chunk_csv(lambda_context):
def test_chunk_csv(
dynamodb_persistence_layer: DynamoDBPersistenceLayer,
lambda_context,
):
pk = 'BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461'
sk = 'FILE#2025-11-13T16:04:53.024743'
event = {
'detail': {
'new_image': {
'id': pk,
'sk': sk,
's3_uri': 's3://saladeaula.digital/samples/large_users.csv',
'columns': {
'1:email',
'2:cpf',
'3:name',
},
'org': {
'id': '1411844c-10d6-456e-959d-e91775145461',
'name': 'EDUSEG',
'cnpj': '15608435000190',
},
'created_at': now(),
},
},
}
app.lambda_handler(event, lambda_context) # type: ignore
r = dynamodb_persistence_layer.collection.query(PartitionKey(pk), limit=100)
assert len(r['items']) == 67

13
users-events/tests/events/batch/test_excel_to_csv.py Normal file
View File

@@ -0,0 +1,13 @@
import events.batch.excel_to_csv as app
def test_excel_to_csv(lambda_context):
event = {
'detail': {
'new_image': {
's3_uri': 's3://saladeaula.digital/samples/large_users.csv',
},
},
}
assert app.lambda_handler(event, lambda_context) # type: ignore

9
users-events/tests/seeds.jsonl
View File

@@ -1,4 +1,11 @@
{"id": {"S": "cJtK9SsnJhKPyxESe7g3DG"}, "sk": {"S": "0"}, "name": {"S": "EDUSEG"}}
{"id": {"S": "cJtK9SsnJhKPyxESe7g3DG"}, "sk": {"S": "admins#5OxmMjL-ujoR5IMGegQz"}, "name": {"S": "Sérgio R Siqueira"}, "email": {"S": "sergio@somosbeta.com.br"}}
{"id": {"S": "cnpj"}, "sk": {"S": "15608435000190"}, "user_id": {"S": "cJtK9SsnJhKPyxESe7g3DG"}}
{"id": {"S": "email"}, "sk": {"S": "org+15608435000190@users.noreply.saladeaula.digital"}, "user_id": {"S": "cJtK9SsnJhKPyxESe7g3DG"}}
{"id": {"S": "email"}, "sk": {"S": "org+15608435000190@users.noreply.saladeaula.digital"}, "user_id": {"S": "cJtK9SsnJhKPyxESe7g3DG"}}
{"id": "BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461", "sk": "FILE#2025-11-13T16:04:53.024743", "progress": 0, "s3_uri": "s3://saladeaula.digital/samples/large_users.csv"}
{"id": "BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461", "sk": "CHUNK#START#0#END#3847", "weight": 25, "file_sk": "FILE#2025-11-13T16:04:53.024743", "s3_uri": "s3://saladeaula.digital/samples/large_users.csv"}
{"id": "BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461", "sk": "CHUNK#START#3848#END#7925", "weight": 25, "file_sk": "FILE#2025-11-13T16:04:53.024743", "s3_uri": "s3://saladeaula.digital/samples/large_users.csv"}
{"id": "BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461", "sk": "CHUNK#START#7926#END#11866", "weight": 25, "file_sk": "FILE#2025-11-13T16:04:53.024743", "s3_uri": "s3://saladeaula.digital/samples/large_users.csv"}
{"id": "BATCH_JOB#ORG#1411844c-10d6-456e-959d-e91775145461", "sk": "CHUNK#START#11867#END#15913", "weight": 25, "file_sk": "FILE#2025-11-13T16:04:53.024743", "s3_uri": "s3://saladeaula.digital/samples/large_users.csv"}

4
users-events/tests/test_csv_utils.py
View File

@@ -9,8 +9,8 @@ def test_detect_delimiter():
def test_byte_ranges():
csvpath = 'tests/samples/users.csv'
ranges = byte_ranges(csvpath, 10)
*_, pair = ranges
start_byte, end_byte = pair
*_, chunk = ranges
start_byte, end_byte = chunk
assert ranges == [(0, 808), (809, 1655), (1656, 2303)]