From c77d415893bdb227224bb21e6e145f5c2418b93e Mon Sep 17 00:00:00 2001 From: JeevaRamanathan M Date: Thu, 24 Oct 2024 20:36:47 +0000 Subject: [PATCH] feat: JSON parser implementation Signed-off-by: JeevaRamanathan M --- application/api/user/routes.py | 1 + application/parser/file/bulk.py | 2 + application/parser/file/json_parser.py | 57 ++++++++++++++++++++++++++ frontend/src/index.css | 2 - frontend/src/locale/en.json | 2 +- frontend/src/locale/es.json | 2 +- frontend/src/locale/jp.json | 2 +- frontend/src/locale/zh-TW.json | 2 +- frontend/src/locale/zh.json | 2 +- frontend/src/upload/Upload.tsx | 1 + 10 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 application/parser/file/json_parser.py diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 2ead8ef10..b476345de 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -340,6 +340,7 @@ def post(self): ".epub", ".html", ".mdx", + ".json" ], job_name, final_filename, diff --git a/application/parser/file/bulk.py b/application/parser/file/bulk.py index 79fc2c452..bb63aa61b 100644 --- a/application/parser/file/bulk.py +++ b/application/parser/file/bulk.py @@ -11,6 +11,7 @@ from application.parser.file.markdown_parser import MarkdownParser from application.parser.file.rst_parser import RstParser from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser +from application.parser.file.json_parser import JSONParser from application.parser.schema.base import Document DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = { @@ -23,6 +24,7 @@ ".rst": RstParser(), ".html": HTMLParser(), ".mdx": MarkdownParser(), + ".json":JSONParser(), } diff --git a/application/parser/file/json_parser.py b/application/parser/file/json_parser.py new file mode 100644 index 000000000..0201b4206 --- /dev/null +++ b/application/parser/file/json_parser.py @@ -0,0 +1,57 @@ +import json +from typing import Any, Dict, List, Union +from pathlib import Path + +from application.parser.file.base_parser import BaseParser + +class JSONParser(BaseParser): + r"""JSON (.json) parser. + + Parses JSON files into a list of strings or a concatenated document. + It handles both JSON objects (dictionaries) and arrays (lists). + + Args: + concat_rows (bool): Whether to concatenate all rows into one document. + If set to False, a Document will be created for each item in the JSON. + True by default. + + row_joiner (str): Separator to use for joining each row. + Only used when `concat_rows=True`. + Set to "\n" by default. + + json_config (dict): Options for parsing JSON. Can be used to specify options like + custom decoding or formatting. Set to empty dict by default. + + """ + + def __init__( + self, + *args: Any, + concat_rows: bool = True, + row_joiner: str = "\n", + json_config: dict = {}, + **kwargs: Any + ) -> None: + """Init params.""" + super().__init__(*args, **kwargs) + self._concat_rows = concat_rows + self._row_joiner = row_joiner + self._json_config = json_config + + def _init_parser(self) -> Dict: + """Init parser.""" + return {} + + def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]: + """Parse JSON file.""" + + with open(file, 'r', encoding='utf-8') as f: + data = json.load(f, **self._json_config) + + if isinstance(data, dict): + data = [data] + + if self._concat_rows: + return self._row_joiner.join([str(item) for item in data]) + else: + return data diff --git a/frontend/src/index.css b/frontend/src/index.css index 1eca983c6..9b87724a6 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -67,10 +67,8 @@ body.dark { .table-default td:last-child { @apply border-r-0; /* Ensure no right border on the last column */ } - } - /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */ /* Document diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index cefb99b7b..ec77b7e09 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -86,7 +86,7 @@ "start": "Start Chatting", "name": "Name", "choose": "Choose Files", - "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip limited to 25mb", + "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .json, .zip limited to 25mb", "uploadedFiles": "Uploaded Files", "cancel": "Cancel", "train": "Train", diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 66b457e8a..9ee7d5665 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -86,7 +86,7 @@ "start": "Empezar a chatear", "name": "Nombre", "choose": "Seleccionar Archivos", - "info": "Por favor, suba archivos .pdf, .txt, .rst, .docx, .md, .zip limitados a 25 MB", + "info": "Por favor, suba archivos .pdf, .txt, .rst, .docx, .md, .json, .zip limitados a 25 MB", "uploadedFiles": "Archivos Subidos", "cancel": "Cancelar", "train": "Entrenar", diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 53f1da145..841a477b8 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -86,7 +86,7 @@ "start": "チャットを開始する", "name": "名前", "choose": "ファイルを選択", - "info": ".pdf, .txt, .rst, .docx, .md, .zipファイルを25MBまでアップロードしてください", + "info": ".pdf, .txt, .rst, .docx, .md, .json, .zipファイルを25MBまでアップロードしてください", "uploadedFiles": "アップロードされたファイル", "cancel": "キャンセル", "train": "トレーニング", diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index afcef769d..35df818b0 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -80,7 +80,7 @@ "remote": "遠端", "name": "名稱", "choose": "選擇檔案", - "info": "請上傳 .pdf, .txt, .rst, .docx, .md, .zip 檔案，大小限制為 25MB", + "info": "請上傳 .pdf, .txt, .rst, .docx, .md, .json, .zip 檔案，大小限制為 25MB", "uploadedFiles": "已上傳的檔案", "cancel": "取消", "train": "訓練", diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 0d60a701d..0abe7a590 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -86,7 +86,7 @@ "start": "开始聊天", "name": "名称", "choose": "选择文件", - "info": "请上传 .pdf, .txt, .rst, .docx, .md, .zip 文件，限 25MB", + "info": "请上传 .pdf, .txt, .rst, .docx, .md, .json, .zip 文件，限 25MB", "uploadedFiles": "已上传文件", "cancel": "取消", "train": "训练", diff --git a/frontend/src/upload/Upload.tsx b/frontend/src/upload/Upload.tsx index 37a1fc0cf..140fc688c 100644 --- a/frontend/src/upload/Upload.tsx +++ b/frontend/src/upload/Upload.tsx @@ -314,6 +314,7 @@ function Upload({ 'application/zip': ['.zip'], 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'], + 'application/json': ['.json'], 'text/csv': ['.csv'], 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [ '.xlsx',