add

users-events/app/events/batch/csv_into_chunks.py

@@ -0,0 +1,20 @@
+from aws_lambda_powertools.utilities.data_classes import (
+    EventBridgeEvent,
+    event_source,
+)
+from aws_lambda_powertools.utilities.typing import LambdaContext
+
+from boto3clients import s3_client
+from config import CHUNK_SIZE
+from csv_utils import byte_ranges
+
+transport_params = {'client': s3_client}
+
+
+@event_source(data_class=EventBridgeEvent)
+def lambda_handler(event: EventBridgeEvent, context: LambdaContext) -> bool:
+    new_image = event.detail['new_image']
+    csvfile = new_image['s3uri']
+    pairs = byte_ranges(csvfile, CHUNK_SIZE, transport_params=transport_params)
+
+    return True

users-events/app/events/batch/excel_to_csv.py

@@ -0,0 +1,14 @@
+from aws_lambda_powertools.utilities.data_classes import (
+    EventBridgeEvent,
+    event_source,
+)
+from aws_lambda_powertools.utilities.typing import LambdaContext
+
+from boto3clients import s3_client
+
+transport_params = {'client': s3_client}
+
+
+@event_source(data_class=EventBridgeEvent)
+def lambda_handler(event: EventBridgeEvent, context: LambdaContext) -> bool:
+    return True

users-events/app/events/batch/read_csv_chunk.py

@@ -0,0 +1,55 @@
+import csv
+from io import StringIO
+from typing import TYPE_CHECKING
+
+from aws_lambda_powertools.utilities.data_classes import (
+    EventBridgeEvent,
+    event_source,
+)
+from aws_lambda_powertools.utilities.typing import LambdaContext
+
+from boto3clients import s3_client
+
+if TYPE_CHECKING:
+    from mypy_boto3_s3.client import S3Client
+else:
+    S3Client = object
+
+transport_params = {'client': s3_client}
+
+
+@event_source(data_class=EventBridgeEvent)
+def lambda_handler(event: EventBridgeEvent, context: LambdaContext) -> bool:
+    new_image = event.detail['new_image']
+    csvfile = new_image['s3_uri']
+
+    data = _get_s3_object_range(
+        csvfile,
+        start_byte=new_image['start_byte'],
+        end_byte=new_image['end_byte'],
+        s3_client=s3_client,
+    )
+    reader = csv.reader(data)
+
+    for x in reader:
+        print(x)
+
+    return True
+
+
+def _get_s3_object_range(
+    s3_uri: str,
+    *,
+    start_byte: int,
+    end_byte: int,
+    s3_client: S3Client,
+) -> StringIO:
+    bucket, key = s3_uri.replace('s3://', '').split('/', 1)
+
+    response = s3_client.get_object(
+        Bucket=bucket,
+        Key=key,
+        Range=f'bytes={start_byte}-{end_byte}',
+    )
+
+    return StringIO(response['Body'].read().decode('utf-8'))