wio

http-api/cli/seeds.py

@@ -2,12 +2,23 @@ from typing import Any, Generator
 
 import boto3
 import jsonlines
+from aws_lambda_powertools.shared.json_encoder import Encoder
 from elasticsearch import Elasticsearch
 from layercake.dynamodb import deserialize
+from meilisearch import Client as Meilisearch
 from tqdm import tqdm
 
 elastic_client = Elasticsearch('http://127.0.0.1:9200')
 dynamodb_client = boto3.client('dynamodb', endpoint_url='http://127.0.0.1:8000')
+meili_client = Meilisearch('http://127.0.0.1:7700')
+
+
+class JSONEncoder(Encoder):
+    def default(self, obj):
+        if isinstance(obj, set):
+            return list(obj)
+        return super(__class__, self).default(obj)
+
 
 jsonl_files = (
     'test-orders.jsonl',
@@ -99,9 +110,24 @@ if __name__ == '__main__':
                 put_item(line, table_name, dynamodb_client)  # type: ignore
 
     # Scan DynamoDB tables and index the data into Elasticsearch
+    # for file in tqdm(jsonl_files, desc='Scanning tables'):
+    #     table_name = file.removesuffix('.jsonl')
+    #     elastic.delete_index(table_name)
+
+    #     for doc in tqdm(
+    #         scan_table(
+    #             table_name,
+    #             dynamodb_client,
+    #             FilterExpression='sk = :sk',
+    #             ExpressionAttributeValues={':sk': {'S': '0'}},
+    #         ),
+    #         desc=f'Indexing {table_name}',
+    #     ):
+    #         elastic.index_item(id=doc['id'], index=table_name, doc=doc)
+
+    # Scan DynamoDB tables and index the data into Meilisearch
     for file in tqdm(jsonl_files, desc='Scanning tables'):
         table_name = file.removesuffix('.jsonl')
-        elastic.delete_index(table_name)
 
         for doc in tqdm(
             scan_table(
@@ -112,4 +138,16 @@ if __name__ == '__main__':
             ),
             desc=f'Indexing {table_name}',
         ):
-            elastic.index_item(id=doc['id'], index=table_name, doc=doc)
+            meili_client.index(table_name).add_documents([doc], serializer=JSONEncoder)
+
+            if table_name == 'test-enrollments':
+                print('a')
+                print(doc)
+
+        index = meili_client.index(table_name)
+        index.update_settings(
+            {
+                'sortableAttributes': ['create_date'],
+                'filterableAttributes': ['metadata__tenant_id'],
+            }
+        )