This commit is contained in:
2025-07-22 14:26:05 -03:00
parent 2f7b880f48
commit 1a1d545544
12 changed files with 298 additions and 200 deletions

View File

@@ -3,12 +3,10 @@ from typing import Any, Generator
import boto3
import jsonlines
from aws_lambda_powertools.shared.json_encoder import Encoder
from elasticsearch import Elasticsearch
from layercake.dynamodb import deserialize
from meilisearch import Client as Meilisearch
from tqdm import tqdm
elastic_client = Elasticsearch('http://127.0.0.1:9200')
dynamodb_client = boto3.client('dynamodb', endpoint_url='http://127.0.0.1:8000')
meili_client = Meilisearch('http://127.0.0.1:7700')
@@ -57,33 +55,6 @@ def scan_table(table_name: str, /, dynamodb_client, **kwargs) -> Generator:
)
class Elastic:
def __init__(self, client: Elasticsearch) -> None:
self.client = client
def index_item(
self,
/,
id: str,
index: str,
doc: dict,
):
return self.client.update(
index=index,
id=id,
doc=_serialize_to_basic_types(doc),
doc_as_upsert=True,
)
def delete_index(self, index: str) -> bool:
try:
self.client.indices.delete(index=index)
except Exception:
return False
else:
return True
def _serialize_to_basic_types(value: Any) -> Any:
if isinstance(value, dict):
return {k: _serialize_to_basic_types(v) for k, v in value.items()}
@@ -98,8 +69,6 @@ def _serialize_to_basic_types(value: Any) -> Any:
if __name__ == '__main__':
elastic = Elastic(elastic_client)
# Populate DynamoDB tables with data from JSONL files
for file in tqdm(jsonl_files, desc='Processing files'):
with open(f'seeds/{file}') as fp:
@@ -128,7 +97,7 @@ if __name__ == '__main__':
index = meili_client.index(table_name)
index.update_settings(
{
'sortableAttributes': ['create_date', 'createDate'],
'sortableAttributes': ['create_date', 'createDate', 'created_at'],
'filterableAttributes': ['tenant_id', 'status'],
}
)