diff --git a/dedoc/api/api_args.py b/dedoc/api/api_args.py index d1f7d5cf..f2b9e7c4 100644 --- a/dedoc/api/api_args.py +++ b/dedoc/api/api_args.py @@ -22,9 +22,6 @@ class QueryParameters: # tables handling need_pdf_table_analysis: str = Form("true", enum=["true", "false"], description="Enable table recognition for pdf") table_type: str = Form("", description="Pipeline mode for table recognition") - orient_analysis_cells: str = Form("false", enum=["true", "false"], description="Enable analysis of rotated cells in table headers") - orient_cell_angle: str = Form("90", enum=["90", "270"], - description='Set cells orientation in table headers, "90" means 90 degrees counterclockwise cells rotation') # pdf handling pdf_with_text_layer: str = Form("auto_tabby", enum=["true", "false", "auto", "auto_tabby", "tabby"], diff --git a/dedoc/api/web/index.html b/dedoc/api/web/index.html index ede62117..5538878a 100644 --- a/dedoc/api/web/index.html +++ b/dedoc/api/web/index.html @@ -98,31 +98,9 @@

Attachments handling

- -
-

Tables handling

-
need_pdf_table_analysis, orient_analysis_cells, orient_cell_angle -
-

- -

- -

- -

- -

- -

-
-
- -

PDF handling

-
pdf_with_text_layer, fast_textual_layer_detection, language, pages, is_one_column_document, document_orientation, need_header_footer_analysis, need_binarization, need_gost_frame_analysis +
pdf_with_text_layer, need_pdf_table_analysis, fast_textual_layer_detection, language, pages, is_one_column_document, document_orientation, need_header_footer_analysis, need_binarization, need_gost_frame_analysis

PDF handling

+
need_pdf_table_analysis +
+

+ +

+
+

diff --git a/dedoc/data_structures/cell_with_meta.py b/dedoc/data_structures/cell_with_meta.py index d23cad1c..03ee0c67 100644 --- a/dedoc/data_structures/cell_with_meta.py +++ b/dedoc/data_structures/cell_with_meta.py @@ -47,9 +47,8 @@ def get_annotations(self) -> List[Annotation]: """ return LineWithMeta.join(lines=self.lines, delimiter="\n").annotations - @staticmethod - def create_from_cell(cell: "CellWithMeta") -> "CellWithMeta": - return CellWithMeta(lines=cell.lines, colspan=cell.colspan, rowspan=cell.rowspan, invisible=cell.invisible) + def __str__(self) -> str: + return f"CellWithMeta(cs={self.colspan}, rs={self.rowspan}, {self.get_text()})" def to_api_schema(self) -> ApiCellWithMeta: import numpy as np diff --git a/dedoc/readers/pdf_reader/data_classes/tables/cell.py b/dedoc/readers/pdf_reader/data_classes/tables/cell.py index 0d42dc37..d83e2b6c 100644 --- a/dedoc/readers/pdf_reader/data_classes/tables/cell.py +++ b/dedoc/readers/pdf_reader/data_classes/tables/cell.py @@ -1,8 +1,8 @@ +import copy from typing import List, Optional from dedocutils.data_structures import BBox -from dedoc.data_structures.annotation import Annotation from dedoc.data_structures.cell_with_meta import CellWithMeta from dedoc.data_structures.line_with_meta import LineWithMeta @@ -10,69 +10,37 @@ class Cell(CellWithMeta): @staticmethod - def copy_from(cell: "Cell", - x_top_left: Optional[int] = None, - x_bottom_right: Optional[int] = None, - y_top_left: Optional[int] = None, - y_bottom_right: Optional[int] = None) -> "Cell": - x_top_left = cell.x_top_left if x_top_left is None else x_top_left - x_bottom_right = cell.x_bottom_right if x_bottom_right is None else x_bottom_right - y_top_left = cell.y_top_left if y_top_left is None else y_top_left - y_bottom_right = cell.y_bottom_right if y_bottom_right is None else y_bottom_right - return Cell(x_top_left=x_top_left, - x_bottom_right=x_bottom_right, - y_top_left=y_top_left, - y_bottom_right=y_bottom_right, - id_con=cell.id_con, - lines=cell.lines, - is_attribute=cell.is_attribute, - is_attribute_required=cell.is_attribute_required, - rotated_angle=cell.rotated_angle, - uid=cell.cell_uid, - contour_coord=cell.con_coord) + def copy_from(cell: "Cell", bbox: Optional[BBox] = None) -> "Cell": + copy_cell = copy.deepcopy(cell) + if bbox: + copy_cell.bbox = bbox + + return copy_cell def shift(self, shift_x: int, shift_y: int, image_width: int, image_height: int) -> None: if self.lines: for line in self.lines: line.shift(shift_x=shift_x, shift_y=shift_y, image_width=image_width, image_height=image_height) - self.x_top_left += shift_x - self.x_bottom_right += shift_x - self.y_top_left += shift_y - self.y_bottom_right += shift_y - if self.con_coord: - self.con_coord.shift(shift_x=shift_x, shift_y=shift_y) - def __init__(self, x_top_left: int, x_bottom_right: int, y_top_left: int, y_bottom_right: int, id_con: int = -1, lines: Optional[List[LineWithMeta]] = None, - is_attribute: bool = False, is_attribute_required: bool = False, rotated_angle: int = 0, uid: str = None, - contour_coord: Optional[BBox] = None) -> None: + self.bbox.shift(shift_x=shift_x, shift_y=shift_y) + if self.contour_coord: + self.contour_coord.shift(shift_x=shift_x, shift_y=shift_y) - import uuid + def __init__(self, bbox: BBox, id_con: int = -1, lines: Optional[List[LineWithMeta]] = None, + is_attribute: bool = False, is_attribute_required: bool = False, rotated_angle: int = 0, uid: Optional[str] = None, + contour_coord: Optional[BBox] = None, colspan: int = 1, rowspan: int = 1, invisible: bool = False) -> None: - assert x_top_left <= x_bottom_right - assert y_top_left <= y_bottom_right + import uuid - self.lines = [] if lines is None else lines - super().__init__(lines) + super().__init__(lines=lines, colspan=colspan, rowspan=rowspan, invisible=invisible) - self.x_top_left = x_top_left - self.x_bottom_right = x_bottom_right - self.y_top_left = y_top_left - self.y_bottom_right = y_bottom_right + self.bbox = bbox self.id_con = id_con self.is_attribute = is_attribute self.is_attribute_required = is_attribute_required self.rotated_angle = rotated_angle - self.cell_uid = f"cell_{uuid.uuid1()}" if uid is None else uid - self.con_coord = contour_coord or BBox(0, 0, 0, 0) - - def __str__(self) -> str: - return f"Cell((cs={self.colspan}, rs={self.rowspan}, {self.get_text()})" - - def get_text(self) -> str: - return "\n".join([line.line for line in self.lines]) - - def get_annotations(self) -> List[Annotation]: - return LineWithMeta.join(self.lines, delimiter="\n").annotations + self.uuid = uuid.uuid4() if uuid is None else uid + self.contour_coord = contour_coord or BBox(0, 0, 0, 0) def change_lines_boxes_page_width_height(self, new_page_width: int, new_page_height: int) -> None: from dedoc.data_structures.concrete_annotations.bbox_annotation import BBoxAnnotation @@ -96,11 +64,3 @@ def change_lines_boxes_page_width_height(self, new_page_width: int, new_page_hei def __repr__(self) -> str: return self.__str__() - - @property - def width(self) -> int: - return self.x_bottom_right - self.x_top_left - - @property - def height(self) -> int: - return self.y_bottom_right - self.y_top_left diff --git a/dedoc/readers/pdf_reader/data_classes/tables/scantable.py b/dedoc/readers/pdf_reader/data_classes/tables/scantable.py index be812630..9ae91c18 100644 --- a/dedoc/readers/pdf_reader/data_classes/tables/scantable.py +++ b/dedoc/readers/pdf_reader/data_classes/tables/scantable.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional +from typing import List from dedocutils.data_structures import BBox @@ -9,93 +9,32 @@ from dedoc.readers.pdf_reader.data_classes.tables.location import Location -class ScanTable: - def __init__(self, page_number: int, matrix_cells: Optional[List[List[CellWithMeta]]] = None, bbox: Optional[BBox] = None, - name: str = "", order: int = -1) -> None: - self.matrix_cells = matrix_cells - self.page_number = page_number - self.locations = [] - self.name = name +class ScanTable(Table): + def __init__(self, page_number: int, cells: List[List[CellWithMeta]], bbox: BBox, order: int = -1) -> None: + + super().__init__(cells, TableMetadata(page_id=page_number)) self.order = order - if bbox is not None: - self.locations.append(Location(page_number, bbox)) + self.locations = [Location(page_number, bbox)] def extended(self, table: "ScanTable") -> None: # extend locations self.locations.extend(table.locations) # extend values - self.matrix_cells.extend(table.matrix_cells) + self.cells.extend(table.cells) # extend order self.order = max(self.order, table.order) def check_on_cell_instance(self) -> bool: - if len(self.matrix_cells) == 0: + if len(self.cells) == 0: return False - if len(self.matrix_cells[0]) == 0: + if len(self.cells[0]) == 0: return False - if not isinstance(self.matrix_cells[0][0], Cell): + if not isinstance(self.cells[0][0], Cell): return False return True - def to_table(self) -> Table: - metadata = TableMetadata(page_id=self.page_number, uid=self.name, rotated_angle=self.location.rotated_angle) - cells_with_meta = [[CellWithMeta.create_from_cell(cell) for cell in row] for row in self.matrix_cells] - return Table(metadata=metadata, cells=cells_with_meta) - - @staticmethod - def get_cells_text(attr_cells: List[List[Cell]]) -> List[List[str]]: - attrs = [] - for i in range(0, len(attr_cells)): - attrs.append([a.get_text() for a in attr_cells[i]]) - - return attrs - - @staticmethod - def get_key_value_attrs(attrs: List, val: Any) -> dict: # noqa - res_attrs = [] - for i in range(0, len(attrs)): - res_attrs.append({"attr": attrs[i]}) - res = { - "attrs": res_attrs, - "val": val - } - return res - - @staticmethod - def get_index_of_end_string_attr(matrix_cells: List[List[Cell]]) -> int: - end_attr_string = 0 - for i in range(0, len(matrix_cells)): - if matrix_cells[i][0].is_attribute: - end_attr_string = i - - return end_attr_string - - @staticmethod - def get_attributes_cell(matrix_cells: List[List[Cell]]) -> (List[int], List[List[Cell]], int): - import copy - import numpy as np - - required_columns = [] - for j in range(0, len(matrix_cells[0])): - if matrix_cells[0][j].is_attribute_required: - required_columns.append(j) - - end_attr_string = ScanTable.get_index_of_end_string_attr(matrix_cells) - - attrs = copy.deepcopy(np.array(matrix_cells[0:end_attr_string + 1])) - attrs = attrs.transpose().tolist() - - return [required_columns, attrs, end_attr_string] - - @staticmethod - def get_matrix_attrs_and_data(matrix_cells: List[List[Cell]]) -> (List[List[Cell]], List[List[str]], List[List[str]]): - required_columns, attrs, end_attr_string = ScanTable.get_attributes_cell(matrix_cells) - attrs_text = ScanTable.get_cells_text(attrs) - - data = matrix_cells[(end_attr_string + 1):] - data_text = ScanTable.get_cells_text(data) - - return [attrs, attrs_text, data_text] + def __get_cells_text(self, cells: List[List[CellWithMeta]]) -> List[List[str]]: + return [[cell.get_text() for cell in row] for row in cells] @property def location(self) -> Location: @@ -103,12 +42,12 @@ def location(self) -> Location: @property def uid(self) -> str: - return self.name + return self.metadata.uid def to_dict(self) -> dict: from collections import OrderedDict - data_text = ScanTable.get_cells_text(self.matrix_cells) + data_text = self.__get_cells_text(self.cells) res = OrderedDict() res["locations"] = [location.to_dict() for location in self.locations] diff --git a/dedoc/readers/pdf_reader/pdf_base_reader.py b/dedoc/readers/pdf_reader/pdf_base_reader.py index 4fd9fdec..3a6e29ef 100644 --- a/dedoc/readers/pdf_reader/pdf_base_reader.py +++ b/dedoc/readers/pdf_reader/pdf_base_reader.py @@ -15,8 +15,6 @@ ParametersForParseDoc = namedtuple("ParametersForParseDoc", [ - "orient_analysis_cells", - "orient_cell_angle", "is_one_column_document", "document_orientation", "language", @@ -73,8 +71,6 @@ def read(self, file_path: str, parameters: Optional[dict] = None) -> Unstructure params_for_parse = ParametersForParseDoc( language=param_utils.get_param_language(parameters), - orient_analysis_cells=param_utils.get_param_orient_analysis_cells(parameters), - orient_cell_angle=param_utils.get_param_orient_cell_angle(parameters), is_one_column_document=param_utils.get_param_is_one_column_document(parameters), document_orientation=param_utils.get_param_document_orientation(parameters), need_header_footers_analysis=param_utils.get_param_need_header_footers_analysis(parameters), @@ -91,12 +87,11 @@ def read(self, file_path: str, parameters: Optional[dict] = None) -> Unstructure ) lines, scan_tables, attachments, warnings, metadata = self._parse_document(file_path, params_for_parse) - tables = [scan_table.to_table() for scan_table in scan_tables] if params_for_parse.with_attachments and self.attachment_extractor.can_extract(file_path): attachments += self.attachment_extractor.extract(file_path=file_path, parameters=parameters) - result = UnstructuredDocument(lines=lines, tables=tables, attachments=attachments, warnings=warnings, metadata=metadata) + result = UnstructuredDocument(lines=lines, tables=scan_tables, attachments=attachments, warnings=warnings, metadata=metadata) return self._postprocess(result) def _parse_document(self, path: str, parameters: ParametersForParseDoc) -> ( @@ -177,7 +172,7 @@ def _shift_all_contents(self, lines: List[LineWithMeta], unref_tables: List[Scan table_page_number = location.page_number location.shift(shift_x=gost_analyzed_images[table_page_number][1].x_top_left, shift_y=gost_analyzed_images[table_page_number][1].y_top_left) page_number = scan_table.locations[0].page_number - for row in scan_table.matrix_cells: + for row in scan_table.cells: for cell in row: image_width, image_height = gost_analyzed_images[page_number][2][1], gost_analyzed_images[page_number][2][0] shift_x, shift_y = (gost_analyzed_images[page_number][1].x_top_left, gost_analyzed_images[page_number][1].y_top_left) @@ -275,16 +270,3 @@ def _binarization(self, gray_image: ndarray) -> ndarray: binary_mask = gray_image >= np.quantile(gray_image, 0.05) gray_image[binary_mask] = 255 return gray_image - - def eval_tables_by_batch(self, - batch: Iterator[ndarray], - page_number_begin: int, - language: str, - orient_analysis_cells: bool = False, - orient_cell_angle: int = 270, - table_type: str = "") -> Tuple[List[ndarray], List[ScanTable]]: - from joblib import Parallel, delayed - - result_batch = Parallel(n_jobs=self.config["n_jobs"])(delayed(self.table_recognizer.recognize_tables_from_image)( - image, page_number_begin + i, language, orient_analysis_cells, orient_cell_angle, table_type) for i, image in enumerate(batch)) - return result_batch diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/pdf_image_reader.py b/dedoc/readers/pdf_reader/pdf_image_reader/pdf_image_reader.py index 64d96fe6..e53ba9e3 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/pdf_image_reader.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/pdf_image_reader.py @@ -85,8 +85,6 @@ def _process_one_page(self, image=rotated_image, page_number=page_number, language=parameters.language, - orient_analysis_cells=parameters.orient_analysis_cells, - orient_cell_angle=parameters.orient_cell_angle, table_type=parameters.table_type ) else: diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/cell_splitter.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/cell_splitter.py index 0e72128c..ab1c355d 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/cell_splitter.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/cell_splitter.py @@ -1,6 +1,7 @@ from typing import Dict, List, Optional, Tuple import numpy as np +from dedocutils.data_structures import BBox from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell from dedoc.utils.utils import flatten @@ -55,25 +56,26 @@ def split(self, cells: List[List[Cell]]) -> List[List[Cell]]: for row_id, row in enumerate(result_matrix): for col_id, cell in enumerate(row): if cell is None: - result_matrix[row_id][col_id] = Cell(x_top_left=horizontal_borders[row_id], - x_bottom_right=horizontal_borders[row_id + 1], - y_top_left=vertical_borders[col_id], - y_bottom_right=vertical_borders[col_id + 1]) + bbox = BBox(x_top_left=int(horizontal_borders[row_id]), + y_top_left=int(vertical_borders[col_id]), + width=int(horizontal_borders[row_id + 1] - horizontal_borders[row_id]), + height=int(vertical_borders[col_id + 1] - vertical_borders[col_id])) + result_matrix[row_id][col_id] = Cell(bbox=bbox) return result_matrix @staticmethod def __split_one_cell(cell: Cell, horizontal_borders: np.ndarray, vertical_borders: np.ndarray, result_matrix: List[List[Cell]]) -> None: - left_id, right_id = np.searchsorted(vertical_borders, [cell.x_top_left, cell.x_bottom_right]) - top_id, bottom_id = np.searchsorted(horizontal_borders, [cell.y_top_left, cell.y_bottom_right]) + left_id, right_id = np.searchsorted(vertical_borders, [cell.bbox.x_top_left, cell.bbox.x_bottom_right]) + top_id, bottom_id = np.searchsorted(horizontal_borders, [cell.bbox.y_top_left, cell.bbox.y_bottom_right]) colspan = right_id - left_id rowspan = bottom_id - top_id for row_id in range(top_id, bottom_id): for column_id in range(left_id, right_id): - new_cell = Cell.copy_from(cell, - x_top_left=vertical_borders[column_id], - x_bottom_right=vertical_borders[column_id + 1], - y_top_left=horizontal_borders[row_id], - y_bottom_right=horizontal_borders[row_id + 1]) + bbox = BBox(x_top_left=int(vertical_borders[column_id]), + y_top_left=int(horizontal_borders[row_id]), + width=int(vertical_borders[column_id + 1] - vertical_borders[column_id]), + height=int(horizontal_borders[row_id + 1] - horizontal_borders[row_id])) + new_cell = Cell.copy_from(cell, bbox) new_cell.invisible = True result_matrix[row_id][column_id] = new_cell @@ -106,20 +108,21 @@ def _merge_close_borders(self, cells: List[List[Cell]]) -> List[List[Cell]]: @return: cells with merged borders """ horizontal_borders, vertical_borders = self.__get_borders(cells) - eps_vertical = self.eps * min((cell.width for cell in flatten(cells)), default=0) - eps_horizontal = self.eps * min((cell.height for cell in flatten(cells)), default=0) + eps_vertical = self.eps * min((cell.bbox.width for cell in flatten(cells)), default=0) + eps_horizontal = self.eps * min((cell.bbox.height for cell in flatten(cells)), default=0) horizontal_dict = self.__get_border_dict(borders=horizontal_borders, threshold=eps_horizontal) vertical_dict = self.__get_border_dict(borders=vertical_borders, threshold=eps_vertical) result = [] for row in cells: new_row = [] for cell in row: - x_top_left = vertical_dict[cell.x_top_left] - x_bottom_right = vertical_dict[cell.x_bottom_right] - y_top_left = horizontal_dict[cell.y_top_left] - y_bottom_right = horizontal_dict[cell.y_bottom_right] + x_top_left = vertical_dict[cell.bbox.x_top_left] + x_bottom_right = vertical_dict[cell.bbox.x_bottom_right] + y_top_left = horizontal_dict[cell.bbox.y_top_left] + y_bottom_right = horizontal_dict[cell.bbox.y_bottom_right] if y_top_left < y_bottom_right and x_top_left < x_bottom_right: - new_cell = Cell.copy_from(cell, x_top_left=x_top_left, x_bottom_right=x_bottom_right, y_top_left=y_top_left, y_bottom_right=y_bottom_right) + bbox = BBox(x_top_left=x_top_left, y_top_left=y_top_left, width=x_bottom_right - x_top_left, height=y_bottom_right - y_top_left) + new_cell = Cell.copy_from(cell, bbox) new_row.append(new_cell) result.append(new_row) return result @@ -130,8 +133,8 @@ def __get_borders(cells: List[List[Cell]]) -> Tuple[List[int], List[int]]: vertical_borders = [] for row in cells: for cell in row: - horizontal_borders.append(cell.y_top_left) - horizontal_borders.append(cell.y_bottom_right) - vertical_borders.append(cell.x_top_left) - vertical_borders.append(cell.x_bottom_right) + horizontal_borders.append(cell.bbox.y_top_left) + horizontal_borders.append(cell.bbox.y_bottom_right) + vertical_borders.append(cell.bbox.x_top_left) + vertical_borders.append(cell.bbox.x_bottom_right) return horizontal_borders, vertical_borders diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/split_last_hor_union_cells.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/split_last_hor_union_cells.py index 0b14f034..8dd0bbac 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/split_last_hor_union_cells.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/split_last_hor_union_cells.py @@ -127,11 +127,11 @@ def _split_row(cell_splitter: Cell, union_cell: List[Cell], language: str, image # Get width of all union cell eps = len(union_cell) - x_left = union_cell[0].x_top_left + eps - x_right = union_cell[-1].x_bottom_right + x_left = union_cell[0].bbox.x_top_left + eps + x_right = union_cell[-1].bbox.x_bottom_right # get y coordinate from cell before union cell - y_top_split = cell_splitter.con_coord.y_top_left - y_bottom_split = cell_splitter.con_coord.y_top_left + cell_splitter.con_coord.height + y_top_split = cell_splitter.contour_coord.y_top_left + y_bottom_split = cell_splitter.contour_coord.y_top_left + cell_splitter.contour_coord.height if abs(y_bottom_split - y_top_split) < 10: for cell in union_cell: cell.lines = [] @@ -141,8 +141,8 @@ def _split_row(cell_splitter: Cell, union_cell: List[Cell], language: str, image col_id = len(union_cell) - 1 result_row = copy.deepcopy(union_cell) while col_id >= 0: - union_cell[col_id].y_top_left = y_top_split - union_cell[col_id].y_bottom_right = y_bottom_split + union_cell[col_id].bbox.y_top_left = y_top_split + union_cell[col_id].bbox.height = y_bottom_split - union_cell[col_id].bbox.y_top_left cell_image, padding_value = OCRCellExtractor.upscale(image[y_top_split:y_bottom_split, x_left:x_right]) result_row[col_id].lines = __get_ocr_lines(cell_image, language, page_image=image, @@ -162,11 +162,8 @@ def __get_ocr_lines(cell_image: np.ndarray, language: str, page_image: np.ndarra for line in list(ocr_result.lines): text_line = OCRCellExtractor.get_line_with_meta("") for word in line.words: - # do absolute coordinate on src_image (inside src_image) - word.bbox.y_top_left -= padding_cell_value - word.bbox.x_top_left -= padding_cell_value - word.bbox.y_top_left += cell_bbox.y_top_left - word.bbox.x_top_left += cell_bbox.x_top_left + # do absolute coordinates on src_image (inside src_image) + word.bbox.shift(shift_x=cell_bbox.x_top_left - padding_cell_value, shift_y=cell_bbox.y_top_left - padding_cell_value) # add space between words if len(text_line) != 0: diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/multipage_table_extractor.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/multipage_table_extractor.py index 06abe0c2..8d74829d 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/multipage_table_extractor.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/multipage_table_extractor.py @@ -7,7 +7,7 @@ from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell from dedoc.readers.pdf_reader.data_classes.tables.scantable import ScanTable from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.base_table_extractor import BaseTableExtractor -from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.concrete_extractors.table_attribute_extractor import TableAttributeExtractor +from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.concrete_extractors.table_attribute_extractor import TableHeaderExtractor from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_utils.utils import equal_with_eps @@ -21,11 +21,11 @@ def extract_multipage_tables(self, single_tables: List[ScanTable], lines_with_me self.single_tables = single_tables multipages_tables = [] list_page_with_tables = [] - total_pages = max((table.page_number + 1 for table in single_tables), default=0) + total_pages = max((table.location.page_number + 1 for table in single_tables), default=0) for cur_page in range(total_pages): # 1. get possible diapason of neighbors pages with tables # pages distribution - list_mp_table = [t for t in self.single_tables if t.page_number == cur_page] + list_mp_table = [t for t in self.single_tables if t.location.page_number == cur_page] list_page_with_tables.append(list_mp_table) total_cur_page = 0 @@ -86,7 +86,7 @@ def __handle_multipage_table(self, # t2 is merged with t1 t1.extended(t2) list_page_with_tables[cur_page].pop(0) - self.__delete_ref_table(lines=lines_with_meta, table_name=t2.name) + self.__delete_ref_table(lines=lines_with_meta, table_name=t2.uid) else: if len(list_page_with_tables[cur_page]) > 0: cur_page -= 1 # analysis from the current page, not the next one @@ -117,12 +117,12 @@ def __get_width_cell_wo_separating(row: List[Cell]) -> List[int]: end = None for cell_id, cell in enumerate(row): if prev_uid is None: - start = cell.x_top_left - prev_uid = cell.cell_uid - elif prev_uid != cell.cell_uid: + start = cell.bbox.x_top_left + prev_uid = cell.uuid + elif prev_uid != cell.uuid: widths.append(end - start) - start = cell.x_top_left - end = cell.x_bottom_right + start = cell.bbox.x_top_left + end = cell.bbox.x_bottom_right if cell_id == len(row) - 1: widths.append(end - start) return widths @@ -154,28 +154,28 @@ def __is_one_table(self, t1: ScanTable, t2: ScanTable) -> bool: return False # condition 2. Exclusion of the duplicated header (if any) - attr1 = TableAttributeExtractor.get_header_table(t1.matrix_cells) - attr2 = TableAttributeExtractor.get_header_table(t2.matrix_cells) + attr1 = TableHeaderExtractor.get_header_table(t1.cells) + attr2 = TableHeaderExtractor.get_header_table(t2.cells) t2_update = copy.deepcopy(t2) - if TableAttributeExtractor.is_equal_attributes(attr1, attr2): - t2_update.matrix_cells = t2_update.matrix_cells[len(attr2):] + if TableHeaderExtractor.is_equal_header(attr1, attr2): + t2_update.cells = t2_update.cells[len(attr2):] - if len(t2_update.matrix_cells) == 0 or len(t1.matrix_cells) == 0: + if len(t2_update.cells) == 0 or len(t1.cells) == 0: return False - TableAttributeExtractor.clear_attributes(t2_update.matrix_cells) + TableHeaderExtractor.clear_attributes(t2_update.cells) # condition 3. Number of columns should be equal - if len(t1.matrix_cells[-1]) != len(t2_update.matrix_cells[0]): + if len(t1.cells[-1]) != len(t2_update.cells[0]): if self.config.get("debug_mode", False): self.logger.debug("Different count column") return False # condition 4. Comparison of the widths of last and first rows - if t1.check_on_cell_instance() and t2_update.check_on_cell_instance() and not self.__is_equal_width_cells(t1.matrix_cells, t2_update.matrix_cells): + if t1.check_on_cell_instance() and t2_update.check_on_cell_instance() and not self.__is_equal_width_cells(t1.cells, t2_update.cells): if self.config.get("debug_mode", False): self.logger.debug("Different width columns") return False - t2.matrix_cells = copy.deepcopy(t2_update.matrix_cells) # save changes + t2.cells = copy.deepcopy(t2_update.cells) # save changes return True diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/onepage_table_extractor.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/onepage_table_extractor.py index c946cccf..c676b3da 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/onepage_table_extractor.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/onepage_table_extractor.py @@ -1,10 +1,10 @@ import copy import logging -import uuid from typing import List import numpy as np +from dedoc.common.exceptions.recognize_error import RecognizeError from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell from dedoc.readers.pdf_reader.data_classes.tables.scantable import ScanTable from dedoc.readers.pdf_reader.data_classes.tables.table_tree import TableTree @@ -12,7 +12,7 @@ from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.cell_splitter import CellSplitter from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.split_last_hor_union_cells import split_last_column from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.base_table_extractor import BaseTableExtractor -from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.concrete_extractors.table_attribute_extractor import TableAttributeExtractor +from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.concrete_extractors.table_attribute_extractor import TableHeaderExtractor from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_utils.img_processing import detect_tables_by_contours @@ -23,26 +23,18 @@ def __init__(self, *, config: dict, logger: logging.Logger) -> None: self.image = None self.page_number = 0 - self.attribute_selector = TableAttributeExtractor(logger=self.logger) + self.table_header_extractor = TableHeaderExtractor(logger=self.logger) self.count_vertical_extended = 0 self.splitter = CellSplitter() self.table_options = TableTypeAdditionalOptions() self.language = "rus" - def extract_onepage_tables_from_image(self, - image: np.ndarray, - page_number: int, - language: str, - orient_analysis_cells: bool, - orient_cell_angle: int, # TODO remove - table_type: str) -> List[ScanTable]: + def extract_onepage_tables_from_image(self, image: np.ndarray, page_number: int, language: str, table_type: str) -> List[ScanTable]: """ extracts tables from input image :param image: input gray image :param page_number: :param language: language for Tesseract - :param orient_analysis_cells: need or not analyse orientations of cells - :param orient_cell_angle: angle of cells (needs if orient_analysis_cells==True) :return: List[ScanTable] """ self.image = image @@ -50,73 +42,14 @@ def extract_onepage_tables_from_image(self, self.language = language # Read the image - tables_tree, contours, angle_rotate = detect_tables_by_contours(image, - language=language, - config=self.config, - orient_analysis_cells=orient_analysis_cells, - table_type=table_type) - + tables_tree, contours, angle_rotate = detect_tables_by_contours(image, language=language, config=self.config, table_type=table_type) tables = self.__build_structure_table_from_tree(tables_tree=tables_tree, table_type=table_type) - for matrix in tables: - for location in matrix.locations: + for table in tables: + for location in table.locations: location.bbox.rotate_coordinates(angle_rotate=-angle_rotate, image_shape=image.shape) location.rotated_angle = angle_rotate - tables = self.__select_attributes_matrix_tables(tables=tables) - - return tables - - """ TODO fix in the future (REMOVE) - def __detect_diff_orient(self, cell_text: str) -> bool: - # 1 - разбиваем на строки длины которых состоят хотя бы из одного символа - parts = cell_text.split("\n") - parts = [p for p in parts if len(p) > 0] - - # 2 - подсчитываем среднюю длину строк ячейки - len_parts = [len(p) for p in parts] - avg_len_part = np.average(len_parts) - - # Эвристика: считаем что ячейка повернута, если у нас большое количество строк и строки короткие - if len(parts) > TableTree.minimal_cell_cnt_line \ - and avg_len_part < TableTree.minimal_cell_avg_length_line: - return True - return False - - def __correct_orient_cell(self, cell: Cell, language: str, rotated_angle: int) -> [Cell, np.ndarray]: - img_cell = self.image[cell.y_top_left: cell.y_bottom_right, cell.x_top_left: cell.x_bottom_right] - rotated_image_cell = rotate_image(img_cell, -rotated_angle) - - output_dict = get_text_with_bbox_from_cells(img_cell, language=language) - line_boxes = [ - TextWithBBox(text=line.text, page_num=page_num, bbox=line.bbox, line_num=line_num, annotations=line.get_annotations(width, height)) - for line_num, line in enumerate(output_dict.lines)] - # get_cell_text_by_ocr(rotated_image_cell, language=language) - cell.set_rotated_angle(rotated_angle=-rotated_angle) - return cell, rotated_image_cell - - - def __analyze_header_cell_with_diff_orient(self, tables: List[ScanTable], language: str, - rotated_angle: int) -> List[ScanTable]: - - for table in tables: - attrs = TableAttributeExtractor.get_header_table(table.matrix_cells) - for i, row in enumerate(attrs): - for j, attr in enumerate(row): - if self.__detect_diff_orient(attr.text): - rotated_cell, rotated_image = self.__correct_orient_cell(attr, language=language, rotated_angle=rotated_angle) - table.matrix_cells[i][j] = rotated_cell - - return tables - """ - - def __select_attributes_matrix_tables(self, tables: List[ScanTable]) -> List[ScanTable]: - for matrix in tables: - matrix = self.attribute_selector.select_attributes(matrix) - - if self.config.get("debug_mode", False): - self._print_table_attr(matrix.matrix_cells) - return tables def __get_matrix_table_from_tree(self, table_tree: TableTree) -> ScanTable: @@ -127,26 +60,20 @@ def __get_matrix_table_from_tree(self, table_tree: TableTree) -> ScanTable: matrix = [] line = [] for cell in table_tree.children: - if len(line) != 0 and abs(cell.cell_box.y_top_left - line[-1].y_top_left) > 15: # add eps + if len(line) != 0 and abs(cell.cell_box.y_top_left - line[-1].bbox.y_top_left) > 15: # add eps cpy_line = copy.deepcopy(line) matrix.append(cpy_line) line.clear() - cell_ = Cell(x_top_left=cell.cell_box.x_top_left, - x_bottom_right=cell.cell_box.x_bottom_right, - y_top_left=cell.cell_box.y_top_left, - y_bottom_right=cell.cell_box.y_bottom_right, - id_con=cell.id_contours, - lines=cell.lines, - contour_coord=cell.cell_box) + cell_ = Cell(bbox=cell.cell_box, id_con=cell.id_contours, lines=cell.lines, contour_coord=cell.cell_box) line.append(cell_) matrix.append(line) # sorting column in each row - for i in range(0, len(matrix)): - matrix[i] = sorted(matrix[i], key=lambda cell: cell.x_top_left, reverse=False) + for i, row in enumerate(matrix): + matrix[i] = sorted(row, key=lambda cell: cell.bbox.x_top_left, reverse=False) - matrix_table = ScanTable(matrix_cells=matrix, bbox=table_tree.cell_box, page_number=self.page_number, name=str(uuid.uuid4())) + matrix_table = ScanTable(cells=matrix, bbox=table_tree.cell_box, page_number=self.page_number) return matrix_table @@ -157,19 +84,33 @@ def __build_structure_table_from_tree(self, tables_tree: TableTree, table_type: tables = [] for table_tree in tables_tree.children: try: - cur_table = self.__get_matrix_table_from_tree(table_tree) - # Эвристика 1: Таблица должна состоять из 1 строк и более - if len(cur_table.matrix_cells) > 0: - cur_table.matrix_cells = self.splitter.split(cells=cur_table.matrix_cells) - - # Эвристика 2: таблица должна иметь больше одного столбца - if len(cur_table.matrix_cells[0]) > 1 or (self.table_options.detect_one_cell_table in table_type and cur_table.matrix_cells[0] != []): - tables.append(cur_table) - - if self.table_options.split_last_column in table_type: - cur_table.matrix_cells = split_last_column(cur_table.matrix_cells, language=self.language, image=self.image) + table = self.__get_matrix_table_from_tree(table_tree) + table.cells = self.handle_cells(table.cells, table_type) + tables.append(table) except Exception as ex: self.logger.warning(f"Warning: unrecognized table into page {self.page_number}. {ex}") if self.config.get("debug_mode", False): raise ex return tables + + def handle_cells(self, cells: List[List[Cell]], table_type: str = "") -> List[List[Cell]]: + # Эвристика 1: Таблица должна состоять из 1 строк и более + if len(cells) < 1: + raise RecognizeError("Invalid recognized table") + + cells = self.splitter.split(cells=cells) + + # Эвристика 2: таблица должна иметь больше одного столбца + if cells[0] == [] or (len(cells[0]) <= 1 and self.table_options.detect_one_cell_table not in table_type): + raise RecognizeError("Invalid recognized table") + + # Postprocess table + if self.table_options.split_last_column in table_type: + cells = split_last_column(cells, language=self.language, image=self.image) + + self.table_header_extractor.set_header_cells(cells) + + if self.config.get("debug_mode", False): + self._print_table_attr(cells) + + return cells diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/table_attribute_extractor.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/table_attribute_extractor.py index f13f0eec..3dfca0e1 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/table_attribute_extractor.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_extractors/concrete_extractors/table_attribute_extractor.py @@ -2,31 +2,31 @@ from typing import List from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell -from dedoc.readers.pdf_reader.data_classes.tables.scantable import ScanTable from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_utils.utils import similarity -class TableAttributeExtractor(object): +class TableHeaderExtractor: """ - Class finds and labels "is_attributes=True" attribute cells into ScanTable + Class finds and labels "is_attributes=True" attribute (header) cells into ScanTable + """ def __init__(self, logger: logging.Logger) -> None: self.logger = logger - def select_attributes(self, scan_table: ScanTable) -> ScanTable: - return self.__set_attributes_for_type_top(scan_table) + def set_header_cells(self, cells: List[List[Cell]]) -> None: + self.__set_attributes_for_type_top(cells) @staticmethod - def is_equal_attributes(attr1: List[List[Cell]], attr2: List[List[Cell]], thr_similarity: int = 0.8) -> bool: - if len(attr1) != len(attr2): + def is_equal_header(header_1: List[List[Cell]], header_2: List[List[Cell]], thr_similarity: int = 0.8) -> bool: + if len(header_1) != len(header_2): return False - for i in range(len(attr1)): - if len(attr1[i]) != len(attr2[i]): + for i in range(len(header_1)): + if len(header_1[i]) != len(header_2[i]): return False - for j in range(len(attr1[i])): - if similarity(attr1[i][j].get_text(), attr2[i][j].get_text()) < thr_similarity: + for j in range(len(header_1[i])): + if similarity(header_1[i][j].get_text(), header_2[i][j].get_text()) < thr_similarity: return False return True @@ -44,7 +44,7 @@ def check_have_attributes(matrix_table: List[List[Cell]]) -> bool: @staticmethod def get_header_table(matrix_table: List[List[Cell]]) -> List[List[Cell]]: - if not TableAttributeExtractor.check_have_attributes(matrix_table): + if not TableHeaderExtractor.check_have_attributes(matrix_table): return matrix_table[:1] header_rows = len(matrix_table) @@ -58,7 +58,7 @@ def get_header_table(matrix_table: List[List[Cell]]) -> List[List[Cell]]: @staticmethod def clear_attributes(matrix_table: List[List[Cell]]) -> None: - if not TableAttributeExtractor.check_have_attributes(matrix_table): + if not TableHeaderExtractor.check_have_attributes(matrix_table): return for row in matrix_table: @@ -66,114 +66,87 @@ def clear_attributes(matrix_table: List[List[Cell]]) -> None: cell.is_attribute = False cell.is_attribute_required = False - def __is_indexable_column(self, matrix_table: List[List[Cell]], column_id: int, max_raw_of_search: int) -> bool: + def __is_indexable_column(self, matrix_table: List[List[Cell]], column_id: int, max_row_of_search: int) -> bool: # № п/п - for i in range(0, max_raw_of_search + 1): - if column_id < len(matrix_table[i]) and "№" in matrix_table[i][column_id].get_text() and len( - matrix_table[i][column_id].get_text()) < len("№ п/п\n"): + for row in matrix_table[:max_row_of_search + 1]: + if column_id < len(row) and "№" in row[column_id].get_text() and len(row[column_id].get_text()) < len("№ п/п\n"): return True return False - def __set_attributes_for_type_top(self, scan_table: ScanTable) -> ScanTable: - vertical_union_columns = self.__analyze_attr_for_vertical_union_columns(scan_table) - horizontal_union_rows = self.__analyze_attr_for_horizontal_union_raws(scan_table) + def __set_attributes_for_type_top(self, cells: List[List[Cell]]) -> List[List[Cell]]: + horizontal_union_rows = self.__analyze_attr_for_horizontal_union_raws(cells) - # simple table - if (0 not in horizontal_union_rows) and len(vertical_union_columns) == 0: - self.__analyze_attr_for_simple_table(scan_table) + if 0 not in horizontal_union_rows: + self.__analyze_attr_for_simple_table(cells) - return scan_table + return cells def __is_empty_column(self, matrix_table: List[List[Cell]], column_id: int) -> bool: - all_empty = True - for i in range(0, len(matrix_table)): - if len(matrix_table[i]) <= column_id: - break - if matrix_table[i][column_id].get_text() != "": - all_empty = False - break - return all_empty + for row in matrix_table: + if len(row) <= column_id: + return True + if row[column_id].get_text() != "": + return False + return True def __is_empty_row(self, matrix_table: List[List[Cell]], row_index: int) -> bool: - all_empty = True - for j in range(0, len(matrix_table[row_index])): - if matrix_table[row_index][j].get_text() != "": - all_empty = False - break - return all_empty - - def __analyze_attr_for_vertical_union_columns(self, scan_table: ScanTable) -> List[int]: - vertical_union_columns = [] - if len(vertical_union_columns) != 0 and len(scan_table.matrix_cells) > 1: - self.logger.debug("ATTR_TYPE: vertical union table") - row_max_attr = 1 - i = 1 - - # Установка атрибутов таблицы - for i in range(0, row_max_attr): - for j in range(0, len(scan_table.matrix_cells[i])): - scan_table.matrix_cells[i][j].is_attribute = True - # Установка обязательных атрибутов - scan_table.matrix_cells[0][0].is_attribute_required = True - for j in range(1, len(scan_table.matrix_cells[0])): - is_attribute_required = True - if is_attribute_required: - scan_table.matrix_cells[0][j].is_attribute_required = True - - return vertical_union_columns - - def __analyze_attr_for_horizontal_union_raws(self, scan_table: ScanTable) -> List[int]: + + for cell in matrix_table[row_index]: + if cell.get_text() != "": + return False + return True + + def __analyze_attr_for_horizontal_union_raws(self, cells: List[List[Cell]]) -> List[int]: horizontal_union_rows = [] union_first = False - for i in range(0, len(scan_table.matrix_cells)): + for i in range(len(cells)): if len(horizontal_union_rows) > 0 and i not in horizontal_union_rows: horizontal_union_rows.append(i) - if not self.__is_empty_row(scan_table.matrix_cells, i): + if not self.__is_empty_row(cells, i): break if union_first and len(horizontal_union_rows) != 0: self.logger.debug("ATTR_TYPE: horizontal_union_rows") - for i in range(0, len(horizontal_union_rows)): - for j in range(0, len(scan_table.matrix_cells[i])): - scan_table.matrix_cells[i][j].is_attribute = True - scan_table.matrix_cells[0][0].is_attribute_required = True + for i in range(len(horizontal_union_rows)): + for j in range(len(cells[i])): + cells[i][j].is_attribute = True + cells[0][0].is_attribute_required = True first_required_column = 0 # search indexable_column # один один столбец должен быть (0) - нумерованным, # один (1) - с обязательными поляями, один (2) - с необязательными # поэтому len(matrix_table) > first_required_column + 2 if len(horizontal_union_rows) > 0 and \ - self.__is_indexable_column(scan_table.matrix_cells, first_required_column, max_raw_of_search=horizontal_union_rows[-1]) \ - and len(scan_table.matrix_cells) > first_required_column + 2: - scan_table.matrix_cells[0][first_required_column + 1].is_attribute_required = True + self.__is_indexable_column(cells, first_required_column, max_raw_of_search=horizontal_union_rows[-1]) \ + and len(cells) > first_required_column + 2: + cells[0][first_required_column + 1].is_attribute_required = True # Полностью пустые строки не могут быть атрибутами (не информативны) # Перенос атрибутов на след строку таблицы index_empty_rows = horizontal_union_rows[-1] - if self.__is_empty_row(scan_table.matrix_cells, index_empty_rows) and len(scan_table.matrix_cells) != index_empty_rows + 1: + if self.__is_empty_row(cells, index_empty_rows) and len(cells) != index_empty_rows + 1: horizontal_union_rows.append(index_empty_rows + 1) - for j in range(0, len(scan_table.matrix_cells[index_empty_rows + 1])): - scan_table.matrix_cells[index_empty_rows + 1][j].is_attribute = True + for j in range(0, len(cells[index_empty_rows + 1])): + cells[index_empty_rows + 1][j].is_attribute = True self.logger.debug("detect empty attributes row") return horizontal_union_rows - def __analyze_attr_for_simple_table(self, scan_table: ScanTable) -> None: + def __analyze_attr_for_simple_table(self, cells: List[List[Cell]]) -> None: self.logger.debug("ATTR_TYPE: simple table") - for j in range(0, len(scan_table.matrix_cells[0])): - scan_table.matrix_cells[0][j].is_attribute = True + for cell in cells[0]: + cell.is_attribute = True + # set first required column - j = 0 - first_required_column = j - while j < len(scan_table.matrix_cells[0]): - if not self.__is_empty_column(scan_table.matrix_cells, j): - scan_table.matrix_cells[0][j].is_attribute_required = True + first_required_column = 0 + for j in range(len(cells[0])): + if not self.__is_empty_column(cells, j): + cells[0][j].is_attribute_required = True first_required_column = j break - j += 1 # search indexable_column - # один один столбец должен быть (0) - нумерованным, - # один (1) - с обязательными поляями, один (2) - с необязательными + # один столбец должен быть (0) - нумерованным, + # один (1) - с обязательными полями, один (2) - с необязательными # поэтому len(matrix_table) > first_required_column + 2 - if self.__is_indexable_column(scan_table.matrix_cells, first_required_column, 0) and len(scan_table.matrix_cells) > first_required_column + 2: - scan_table.matrix_cells[0][first_required_column + 1].is_attribute_required = True + if self.__is_indexable_column(cells, first_required_column, 0) and len(cells) > first_required_column + 2: + cells[0][first_required_column + 1].is_attribute_required = True diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_recognizer.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_recognizer.py index c1124ca4..11c30cab 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_recognizer.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_recognizer.py @@ -21,57 +21,36 @@ class TableRecognizer(object): def __init__(self, *, config: dict = None) -> None: - self.logger = config.get("logger", logging.getLogger()) - self.onepage_tables_extractor = OnePageTableExtractor(config=config, logger=self.logger) self.multipage_tables_extractor = MultiPageTableExtractor(config=config, logger=self.logger) self.config = config self.table_type = TableTypeAdditionalOptions() def convert_to_multipages_tables(self, all_single_tables: List[ScanTable], lines_with_meta: List[LineWithMeta]) -> List[ScanTable]: - multipage_tables = self.multipage_tables_extractor.extract_multipage_tables(single_tables=all_single_tables, lines_with_meta=lines_with_meta) return multipage_tables - def recognize_tables_from_image(self, - image: np.ndarray, - page_number: int, - language: str, - orient_analysis_cells: bool, - orient_cell_angle: int, - table_type: str = "") -> Tuple[np.ndarray, List[ScanTable]]: + def recognize_tables_from_image(self, image: np.ndarray, page_number: int, language: str, table_type: str = "") -> Tuple[np.ndarray, List[ScanTable]]: self.logger.debug(f"Page {page_number}") try: - cleaned_image, matrix_tables = self.__rec_tables_from_img(image, - page_num=page_number, - language=language, - orient_analysis_cells=orient_analysis_cells, - orient_cell_angle=orient_cell_angle, - table_type=table_type) - return cleaned_image, matrix_tables + cleaned_image, scan_tables = self.__rec_tables_from_img(image, page_num=page_number, language=language, table_type=table_type) + return cleaned_image, scan_tables except Exception as ex: logging.warning(ex) if self.config.get("debug_mode", False): raise ex return image, [] - def __rec_tables_from_img(self, - src_image: np.ndarray, - page_num: int, - language: str, - orient_analysis_cells: bool, - orient_cell_angle: int, - table_type: str) -> Tuple[np.ndarray, List[ScanTable]]: + def __rec_tables_from_img(self, src_image: np.ndarray, page_num: int, language: str, table_type: str) -> Tuple[np.ndarray, List[ScanTable]]: gray_image = cv2.cvtColor(src_image, cv2.COLOR_BGR2GRAY) if len(src_image.shape) == 3 else src_image single_page_tables = self.onepage_tables_extractor.extract_onepage_tables_from_image( image=gray_image, page_number=page_num, language=language, - orient_analysis_cells=orient_analysis_cells, - orient_cell_angle=orient_cell_angle, table_type=table_type) + if self.config.get("labeling_mode", False): self.__save_tables(tables=single_page_tables, image=src_image, table_path=self.config.get("table_path", "/tmp/tables")) if self.table_type.detect_one_cell_table in table_type: @@ -128,11 +107,8 @@ def __if_not_table(self, table: ScanTable, image: np.ndarray) -> bool: std = table_image.std() white_mean = (table_image > 225).mean() black_mean = (table_image < 225).mean() - table_area = bbox.width * bbox.height - cells_area = 0 - for row in table.matrix_cells: - for cell in row: - cells_area += cell.width * cell.height + table_area = bbox.square + cells_area = sum([cell.bbox.square for row in table.cells for cell in row]) ratio = cells_area / table_area res = (white_mean < 0.5) or (black_mean > 0.3) or (std < 30) or (mean < 150) or (mean < 200 and std < 80) or ratio < 0.65 diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/accuracy_table_rec.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/accuracy_table_rec.py deleted file mode 100644 index f18b7505..00000000 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/accuracy_table_rec.py +++ /dev/null @@ -1,140 +0,0 @@ -import csv -import json -import os -from typing import List, Tuple - -import cv2 - -from dedoc.config import get_config -from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell -from dedoc.readers.pdf_reader.data_classes.tables.scantable import ScanTable -from dedoc.readers.pdf_reader.pdf_image_reader.pdf_image_reader import PdfImageReader - - -def _create_cell(c: str, text_cells: list) -> Cell: - cell = Cell(x_bottom_right=-1, x_top_left=-1, y_top_left=-1, y_bottom_right=-1) - if "a" in c: - cell.is_attribute = True - # loading cell text - if len(text_cells) != 0: - cell_text = [r for r in text_cells if r[0] == c] - if len(cell_text) != 0: - cell.text = cell_text[0][-1] - return cell - - -def load_from_csv(path_csv: str, path_class_2_csv: str = "") -> List[List[Cell]]: - text_cells = [] - if path_class_2_csv != "": - csv_file_class_2 = open(path_class_2_csv, "r", newline="") - reader_class_2 = csv.reader(csv_file_class_2) - text_cells = [r for r in reader_class_2] - - matrix = [] - with open(path_csv, "r", newline="") as csv_file: - reader = csv.reader(csv_file) - - for raw in reader: - if len(raw) >= 5 and raw[0] == "bbox": - pass - else: - line = [_create_cell(c, text_cells) for c in raw if c != ""] - if len(line) != 0: - matrix.append(line) - return matrix - - -def get_quantitative_parameters(matrix: List[List[Cell]]) -> Tuple[int, int, int, int]: - cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = 0, 0, 0, 0 - - # calculating data - if len(matrix) > 0: - cnt_columns = len(matrix[0]) - cnt_rows = len(matrix) - - for i in range(0, len(matrix)): - for j in range(0, len(matrix[i])): - if matrix[i][j].is_attribute: - cnt_a_cell += 1 - - cnt_cell += 1 - - return cnt_a_cell, cnt_cell, cnt_columns, cnt_rows - - -def calc_agreement(matrix_gt: List[List[Cell]], matrix: List[List[Cell]]) -> float: - q_params = get_quantitative_parameters(matrix) - q_params_gt = get_quantitative_parameters(matrix_gt) - - equal_indexes = [i for i in range(0, len(q_params)) if q_params[i] == q_params_gt[i]] - - agreement = 1.0 * len(equal_indexes) / len(q_params_gt) - return agreement - - -def draw_recognized_cell(tables: List[ScanTable], path_image: str, path_save: str) -> None: - img = cv2.imread(path_image) - for t_index in range(0, len(tables)): - table = tables[t_index].matrix_cells - bbox = tables[t_index].locations.location - blue_color, green_color, red_color = (255, 0, 0), (0, 255, 0), (0, 0, 255) - cv2.rectangle(img, (bbox.x_top_left, bbox.y_top_left), (bbox.width, bbox.height), blue_color, 6) - for i in range(0, len(table)): - for j in range(0, len(table[i])): - cv2.rectangle(img, (table[i][j].x_top_left, table[i][j].y_top_left), (table[i][j].x_bottom_right, table[i][j].y_bottom_right), red_color, 4) - cv2.putText(img, str(table[i][j].id_con), (table[i][j].x_top_left, table[i][j].y_bottom_right), cv2.FONT_HERSHEY_PLAIN, 4, green_color) - cv2.imwrite(path_save, img) - - -def save_json(tables: List[ScanTable], number_test_string: str, path_output: str) -> None: - for i in range(0, len(tables)): - with open(f"{path_output}{number_test_string}_table_{i}.json", "w") as out: - json.dump(tables[i].to_dict(), out, ensure_ascii=False, indent=2) - - -def calc_accuracy(path_image: str, path_gt_struct: str, path_gt_text: str, path_save_image: str, path_save_json: str) -> None: - from os import listdir - from os.path import isfile, join - - os.makedirs(path_save_image, exist_ok=True) - os.makedirs(path_save_json, exist_ok=True) - - image_files = [f for f in listdir(path_image) if isfile(join(path_image, f))] - agreements = [] - - for image_file in image_files: - name_example = image_file.split(".")[0].split("_")[0] - # predict tables - image = cv2.imread(path_image + image_file, 0) - # TODO fix this - clean_images, tables = PdfImageReader(config=get_config()).get_tables([image]) - draw_recognized_cell(tables, path_image + image_file, path_save_image + image_file) - save_json(tables, name_example, path_save_json) - - gt_files = [f for f in listdir(path_gt_struct) if isfile(join(path_gt_struct, f)) and name_example + "_" in f] - for index_table in range(0, len(gt_files)): - - csv_filename = path_gt_struct + name_example + "_" + str(index_table + 1) + ".csv" - csv_text_filename = path_gt_text + name_example + "_" + str(index_table + 1) + "_text.csv" - if os.path.exists(csv_filename): - if not os.path.exists(csv_text_filename): - csv_text_filename = "" - # load_GT - matrix_cell_gt = load_from_csv(csv_filename, csv_text_filename) - # calc agreement - if len(tables) == 0 and matrix_cell_gt == []: - agreements.append(1.0) - elif len(tables) <= index_table: - agreements.append(0) - else: - agreement = calc_agreement(matrix_cell_gt, tables[index_table].matrix_cells) - agreements.append(agreement) - - -if __name__ == "__main__": - current_path = os.path.dirname(__file__) + "/" - calc_accuracy(current_path + "../../backend/test_dataset_table/images/", - current_path + "../../backend/test_dataset_table/GT_struct/", - current_path + "../../backend/test_dataset_table/GT_text/", - "/tmp/backend_claw/out_tables/acc/draw_tables/", - "/tmp/backend_claw/out_tables/acc/json_tables/") diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/img_processing.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/img_processing.py index c060d9d6..6bc12eab 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/img_processing.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/img_processing.py @@ -246,15 +246,9 @@ def __paint_bounds(image: np.ndarray) -> np.ndarray: return image -def detect_tables_by_contours(img: np.ndarray, - language: str = "rus", - orient_analysis_cells: bool = False, - table_type: str = "", - *, - config: dict) -> [TableTree, List[np.ndarray], float]: +def detect_tables_by_contours(img: np.ndarray, language: str = "rus", table_type: str = "", *, config: dict) -> [TableTree, List[np.ndarray], float]: """ detecting contours and TreeTable with help contour analysis. TreeTable is - :param orient_analysis_cells: :param img: input image :param language: parameter language for Tesseract :param config: dict from config.py diff --git a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/utils.py b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/utils.py index 19674772..693b8417 100644 --- a/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/utils.py +++ b/dedoc/readers/pdf_reader/pdf_image_reader/table_recognizer/table_utils/utils.py @@ -1,7 +1,9 @@ -import difflib +from typing import List, Tuple import numpy as np +from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell + def equal_with_eps(x: int, y: int, eps: int = 10) -> bool: return y + eps >= x >= y - eps @@ -20,24 +22,19 @@ def get_highest_pixel_frequency(image: np.ndarray) -> int: def similarity(s1: str, s2: str) -> float: """string similarity""" + import difflib + normalized1 = s1.lower() normalized2 = s2.lower() matcher = difflib.SequenceMatcher(None, normalized1, normalized2) return matcher.ratio() -MINIMAL_CELL_CNT_LINE = 7 -MINIMAL_CELL_AVG_LENGTH_LINE = 10 - - -def detect_diff_orient(cell_text: str) -> bool: - # 1 - разбиваем на строки длины которых состоят хотя бы из одного символа - parts = cell_text.split("\n") - parts = [p for p in parts if len(p) > 0] +def get_statistic_values(cells: List[List[Cell]]) -> Tuple[int, int, int, int]: - # 2 - подсчитываем среднюю длину строк ячейки - len_parts = [len(p) for p in parts] - avg_len_part = np.average(len_parts) + cnt_rows = len(cells) + cnt_columns = len(cells[0]) if cnt_rows else 0 + cnt_cell = cnt_columns * cnt_rows + cnt_attr_cell = len([cell for row in cells for cell in row if cell.is_attribute]) - # Эвристика: считаем сто ячейка повернута если у нас большое количество строк и строки короткие - return len(parts) > MINIMAL_CELL_CNT_LINE and avg_len_part < MINIMAL_CELL_AVG_LENGTH_LINE + return cnt_attr_cell, cnt_cell, cnt_columns, cnt_rows diff --git a/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_tabby_reader.py b/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_tabby_reader.py index 1d0d594d..b60cbed7 100644 --- a/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_tabby_reader.py +++ b/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_tabby_reader.py @@ -29,6 +29,10 @@ class PdfTabbyReader(PdfBaseReader): def __init__(self, *, config: Optional[dict] = None) -> None: import os from dedoc.extensions import recognized_extensions, recognized_mimes + from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.concrete_extractors.onepage_table_extractor import \ + OnePageTableExtractor + from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_extractors.concrete_extractors.table_attribute_extractor import \ + TableHeaderExtractor super().__init__(config=config, recognized_extensions=recognized_extensions.pdf_like_format, recognized_mimes=recognized_mimes.pdf_like_format) self.tabby_java_version = "2.0.0" @@ -36,6 +40,8 @@ def __init__(self, *, config: Optional[dict] = None) -> None: self.jar_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tabbypdf", "jars")) self.java_not_found_error = "`java` command is not found from this Python process. Please ensure Java is installed and PATH is set for `java`" self.default_config = {"JAR_PATH": os.path.join(self.jar_dir, self.jar_name)} + self.table_header_selector = TableHeaderExtractor(logger=self.logger) + self.table_extractor = OnePageTableExtractor(config=config, logger=self.logger) def can_read(self, file_path: Optional[str] = None, mime: Optional[str] = None, extension: Optional[str] = None, parameters: Optional[dict] = None) -> bool: """ @@ -132,9 +138,7 @@ def __extract(self, path: str, parameters: dict, warnings: List[str], tmp_dir: s mp_tables = self.table_recognizer.convert_to_multipages_tables(all_scan_tables, lines_with_meta=all_lines) all_lines = self.linker.link_objects(lines=all_lines, tables=mp_tables, images=all_attached_images) - tables = [scan_table.to_table() for scan_table in mp_tables] - - return all_lines, tables, all_attached_images, document_metadata + return all_lines, mp_tables, all_attached_images, document_metadata def __save_gost_frame_boxes_to_json(self, first_page: Optional[int], last_page: Optional[int], page_count: int, path: str, tmp_dir: str) -> str: from joblib import Parallel, delayed @@ -158,8 +162,7 @@ def __save_gost_frame_boxes_to_json(self, first_page: Optional[int], last_page: return result_json_path def __get_tables(self, page: dict) -> List[ScanTable]: - import uuid - from dedoc.data_structures.cell_with_meta import CellWithMeta + from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell from dedoc.data_structures.concrete_annotations.bbox_annotation import BBoxAnnotation from dedoc.data_structures.line_metadata import LineMetadata @@ -170,7 +173,7 @@ def __get_tables(self, page: dict) -> List[ScanTable]: for table in page["tables"]: table_bbox = BBox(x_top_left=table["x_top_left"], y_top_left=table["y_top_left"], width=table["width"], height=table["height"]) - order = table["order"] # TODO add table order into TableMetadata + order = table["order"] rows = table["rows"] cell_properties = table["cell_properties"] assert len(rows) == len(cell_properties) @@ -187,20 +190,29 @@ def __get_tables(self, page: dict) -> List[ScanTable]: for c in cell_blocks: cell_bbox = BBox(x_top_left=int(c["x_top_left"]), y_top_left=int(c["y_top_left"]), width=int(c["width"]), height=int(c["height"])) annotations.append(BBoxAnnotation(c["start"], c["end"], cell_bbox, page_width=page_width, page_height=page_height)) - """ - TODO: change to Cell class after tabby can return cell coordinates. Then set type Cell in class "ScanTable" - https://jira.intra.ispras.ru/browse/TLDR-851 - """ - result_row.append(CellWithMeta( + current_cell_properties = cell_properties[num_row][num_col] + bbox = BBox(x_top_left=int(current_cell_properties["x_top_left"]), + y_top_left=int(current_cell_properties["y_top_left"]), + width=int(current_cell_properties["width"]), + height=int(current_cell_properties["height"])) + + result_row.append(Cell( + bbox=bbox, lines=[LineWithMeta(line=cell["text"], metadata=LineMetadata(page_id=page_number, line_id=0), annotations=annotations)], - colspan=cell_properties[num_row][num_col]["col_span"], - rowspan=cell_properties[num_row][num_col]["row_span"], - invisible=bool(cell_properties[num_row][num_col]["invisible"]) + colspan=current_cell_properties["col_span"], + rowspan=current_cell_properties["row_span"], + invisible=bool(current_cell_properties["invisible"]) )) cells.append(result_row) - scan_tables.append(ScanTable(page_number=page_number, matrix_cells=cells, bbox=table_bbox, name=str(uuid.uuid4()), order=order)) + try: + cells = self.table_extractor.handle_cells(cells) + scan_tables.append(ScanTable(page_number=page_number, cells=cells, bbox=table_bbox, order=order)) + except Exception as ex: + self.logger.warning(f"Warning: unrecognized table on page {self.page_number}. {ex}") + if self.config.get("debug_mode", False): + raise ex return scan_tables diff --git a/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_txtlayer_reader.py b/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_txtlayer_reader.py index 4cebbaf4..385f02a8 100644 --- a/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_txtlayer_reader.py +++ b/dedoc/readers/pdf_reader/pdf_txtlayer_reader/pdf_txtlayer_reader.py @@ -52,8 +52,6 @@ def _process_one_page(self, image=gray_image, page_number=page_number, language=parameters.language, - orient_analysis_cells=parameters.orient_analysis_cells, - orient_cell_angle=parameters.orient_cell_angle, table_type=parameters.table_type ) else: @@ -87,7 +85,7 @@ def _move_table_cells(self, tables: List[ScanTable], page_shift: BBox, page: Tup shift_x, shift_y = page_shift.x_top_left, page_shift.y_top_left # shift tables to original coordinates for location in table.locations: location.bbox.shift(shift_x=shift_x, shift_y=shift_y) - for row in table.matrix_cells: + for row in table.cells: for cell in row: cell.shift(shift_x=shift_x, shift_y=shift_y, image_width=image_width, image_height=image_height) @@ -97,7 +95,7 @@ def __change_table_boxes_page_width_heigth(self, pdf_width: int, pdf_height: int """ for table in tables: - for row in table.matrix_cells: + for row in table.cells: for cell in row: cell.change_lines_boxes_page_width_height(new_page_width=pdf_width, new_page_height=pdf_height) diff --git a/dedoc/readers/pdf_reader/pdf_txtlayer_reader/tabbypdf/jars/ispras_tbl_extr.jar b/dedoc/readers/pdf_reader/pdf_txtlayer_reader/tabbypdf/jars/ispras_tbl_extr.jar index 2b3c916f..8a737c79 100644 Binary files a/dedoc/readers/pdf_reader/pdf_txtlayer_reader/tabbypdf/jars/ispras_tbl_extr.jar and b/dedoc/readers/pdf_reader/pdf_txtlayer_reader/tabbypdf/jars/ispras_tbl_extr.jar differ diff --git a/dedoc/utils/parameter_utils.py b/dedoc/utils/parameter_utils.py index 3df9f6ca..993b6b8a 100644 --- a/dedoc/utils/parameter_utils.py +++ b/dedoc/utils/parameter_utils.py @@ -33,13 +33,6 @@ def get_param_document_type(parameters: Optional[dict]) -> str: return document_type -def get_param_orient_analysis_cells(parameters: Optional[dict]) -> bool: - if parameters is None: - return False - orient_analysis_cells = str(parameters.get("orient_analysis_cells", "False")).lower() == "true" - return orient_analysis_cells - - def get_param_with_attachments(parameters: Optional[dict]) -> bool: if parameters is None: return False @@ -80,16 +73,6 @@ def get_param_need_binarization(parameters: Optional[dict]) -> bool: return need_binarization -def get_param_orient_cell_angle(parameters: Optional[dict]) -> int: - if parameters is None: - return 90 - - orient_cell_angle = str(parameters.get("orient_cell_angle", "90")) - if orient_cell_angle == "": - orient_cell_angle = "90" - return int(orient_cell_angle) - - def get_param_is_one_column_document(parameters: Optional[dict]) -> Optional[bool]: if parameters is None: return None diff --git a/docs/source/dedoc_api_usage/api.rst b/docs/source/dedoc_api_usage/api.rst index c357ac78..c61a6e01 100644 --- a/docs/source/dedoc_api_usage/api.rst +++ b/docs/source/dedoc_api_usage/api.rst @@ -150,7 +150,7 @@ Api parameters description The encoded contents will be saved in the attachment's metadata in the ``base64_encode`` field. Use ``true`` value to enable this behaviour. - * - :cspan:`3` **Tables handling** + * - :cspan:`3` **PDF handling** * - need_pdf_table_analysis - true, false @@ -162,26 +162,6 @@ Api parameters description If the document has a textual layer, it is recommended to use ``pdf_with_text_layer=tabby``, in this case tables will be parsed much easier and faster. - * - orient_analysis_cells - - true, false - - false - - This option is used for a table recognition in case of PDF documents without a textual layer - (images, scanned documents or when ``pdf_with_text_layer`` is ``true``, ``false`` or ``auto``). - When set to ``true``, it enables analysis of rotated cells in table headers. - Use this option if you are sure that the cells of the table header are rotated. - - * - orient_cell_angle - - 90, 270 - - 90 - - This option is used for a table recognition in case of PDF documents without a textual layer - (images, scanned documents or when ``pdf_with_text_layer`` is ``true``, ``false`` or ``auto``). - It is ignored when ``orient_analysis_cells=false``. - The option is used to set orientation of cells in table headers: - - * **270** -- cells are rotated 90 degrees clockwise; - * **90** -- cells are rotated 90 degrees counterclockwise (or 270 clockwise). - - * - :cspan:`3` **PDF handling** * - pdf_with_text_layer - true, false, tabby, auto, auto_tabby diff --git a/docs/source/parameters/pdf_handling.rst b/docs/source/parameters/pdf_handling.rst index 20fabec9..46c03416 100644 --- a/docs/source/parameters/pdf_handling.rst +++ b/docs/source/parameters/pdf_handling.rst @@ -161,30 +161,6 @@ PDF and images handling It allows :class:`dedoc.readers.PdfImageReader`, :class:`dedoc.readers.PdfTxtlayerReader` and :class:`dedoc.readers.PdfTabbyReader` to properly process the content of the document containing GOST frame, see :ref:`gost_frame_handling` for more details. - * - orient_analysis_cells - - True, False - - False - - * :meth:`dedoc.DedocManager.parse` - * :meth:`dedoc.readers.PdfAutoReader.read`, :meth:`dedoc.readers.PdfTxtlayerReader.read`, :meth:`dedoc.readers.PdfImageReader.read` - * :meth:`dedoc.readers.ReaderComposition.read` - - This option is used for a table recognition for PDF documents or images. - It is ignored when ``need_pdf_table_analysis=False``. - When set to ``True``, it enables analysis of rotated cells in table headers. - Use this option if you are sure that the cells of the table header are rotated. - - * - orient_cell_angle - - 90, 270 - - 90 - - * :meth:`dedoc.DedocManager.parse` - * :meth:`dedoc.readers.PdfAutoReader.read`, :meth:`dedoc.readers.PdfTxtlayerReader.read`, :meth:`dedoc.readers.PdfImageReader.read` - * :meth:`dedoc.readers.ReaderComposition.read` - - This option is used for a table recognition for PDF documents or images. - It is ignored when ``need_pdf_table_analysis=False`` or ``orient_analysis_cells=False``. - The option is used to set orientation of cells in table headers: - - * **270** -- cells are rotated 90 degrees clockwise; - * **90** -- cells are rotated 90 degrees counterclockwise (or 270 clockwise). - .. toctree:: :maxdepth: 1 diff --git a/tests/api_tests/test_api_format_pdf_tabby_reader.py b/tests/api_tests/test_api_format_pdf_tabby_reader.py index b2ff91a6..959e15ca 100644 --- a/tests/api_tests/test_api_format_pdf_tabby_reader.py +++ b/tests/api_tests/test_api_format_pdf_tabby_reader.py @@ -182,7 +182,7 @@ def test_pdf_with_tables(self) -> None: table = tables[3]["cells"] self.assertListEqual(["", "2016", "2017", "2018", "2019"], self._get_text_of_row(table[0])) - self.assertListEqual(["", "Прогноз", "Прогноз бюджета"], self._get_text_of_row(table[1])) + self.assertListEqual(["", "Прогноз", "Прогноз бюджета", "Прогноз бюджета", "Прогноз бюджета"], self._get_text_of_row(table[1])) self.assertListEqual(["Ненефтегазов\nые доходы", "10,4", "9,6", "9,6", "9,6"], self._get_text_of_row(table[21])) self.assertListEqual(["Сальдо\nбюджета", "-3,7", "-3,2", "-2,2", "-1,2"], self._get_text_of_row(table[22])) @@ -227,7 +227,7 @@ def test_tables_with_merged_cells(self) -> None: result = self._send_request(file_name, data=dict(pdf_with_text_layer="tabby")) table = result["content"]["tables"][0]["cells"] - hidden_cells_big_table_with_colspan = [[(1, 0), 10], [(5, 1), 5]] + hidden_cells_big_table_with_colspan = [[(1, 0), 10], [(5, 5), 5]] for (i, j), k in hidden_cells_big_table_with_colspan: self.assertFalse(table[i][j]["invisible"]) diff --git a/tests/api_tests/test_api_misc_multipage_table.py b/tests/api_tests/test_api_misc_multipage_table.py index 5c3c0d2e..ef64fb09 100644 --- a/tests/api_tests/test_api_misc_multipage_table.py +++ b/tests/api_tests/test_api_misc_multipage_table.py @@ -1,4 +1,5 @@ import os +import unittest from typing import List from tests.api_tests.abstract_api_test import AbstractTestApiDocReader @@ -45,14 +46,13 @@ def test_api_ml_table_recognition_synthetic_data_1(self) -> None: tables = self._get_tables(file_name, pdf_with_text_layer=pdf_param) self.assertEqual(len(tables), 1) + @unittest.skip("TLDR-886 подправить координаты ячеек таблиц табби") def test_api_ml_table_recognition_synthetic_data_3(self) -> None: file_name = "example_mp_table_with_repeate_header_2.pdf" - for pdf_param in ["false", "true"]: - # for "tabby" doesn't work because need to unify the output of table in matrix form and set attribute cells, - # without this tables won't be merge. + for pdf_param in ["false", "true", "tabby"]: tables = self._get_tables(file_name, pdf_with_text_layer=pdf_param) - self.assertEqual(len(tables), 1) + self.assertEqual(len(tables), 1, f"Error when pdf_with_text_layer={pdf_param}") table = tables[0]["cells"] self.assertListEqual( @@ -67,8 +67,5 @@ def test_api_ml_table_recognition_synthetic_data_3(self) -> None: self.assertListEqual(["Данные 3", "Данные 3", "Данные 3", "Данные 3", "Данные 3"], self._get_text_of_row(table[5])) self.assertListEqual(["Данные 4", "Данные 4", "Данные 4", "Данные 4", "Данные 4"], self._get_text_of_row(table[6])) self.assertListEqual(["Данные 5", "Данные 5", "Данные 5", "Данные 5", "Данные 5"], self._get_text_of_row(table[7])) - self.assertListEqual(["Заголовок\nБольшой", "Заголовок поменьше 1", "Заголовок поменьше 1", "Заголовок поменьше 2", "Заголовок поменьше 2"], - self._get_text_of_row(table[8])) - self.assertListEqual(["Заголовок\nБольшой", "Заголовочек 1", "Заголовочек 2", "Заголовочек 3", "Заголовочек 4"], self._get_text_of_row(table[9])) - self.assertListEqual(["Данные 6", "Данные 6", "Данные 6", "Данные 6", "Данные 6"], self._get_text_of_row(table[10])) - self.assertListEqual(["Данные 7", "Данные 7", "Данные 7", "Данные 7", "Данные 7"], self._get_text_of_row(table[11])) + self.assertListEqual(["Данные 6", "Данные 6", "Данные 6", "Данные 6", "Данные 6"], self._get_text_of_row(table[8])) + self.assertListEqual(["Данные 7", "Данные 7", "Данные 7", "Данные 7", "Данные 7"], self._get_text_of_row(table[9])) diff --git a/tests/api_tests/test_api_module_table_recognizer.py b/tests/api_tests/test_api_module_table_recognizer.py index a73b4ee5..a6f48a70 100644 --- a/tests/api_tests/test_api_module_table_recognizer.py +++ b/tests/api_tests/test_api_module_table_recognizer.py @@ -1,6 +1,5 @@ import json import os -import unittest from typing import List from tests.api_tests.abstract_api_test import AbstractTestApiDocReader @@ -98,21 +97,6 @@ def _check_header_table(self, cells: List[dict]) -> None: self._check_similarity(row0[9], "Систетематический\nконтроль") self._check_similarity(row0[10], "Экспертная оценка") - @unittest.skip("TODO") - def test_api_table_recognition_with_diff_orient_cells_90(self) -> None: - file_name = "example_table_with_90_orient_cells.pdf" - response = self._send_request(file_name, dict(orient_analysis_cells=True, orient_cell_angle="90")) - table = response["content"]["tables"][0] - - self._check_header_table(table["cells"]) - - @unittest.skip - def test_api_table_recognition_with_diff_orient_cells_270(self) -> None: - file_name = "example_table_with_270_orient_cells.pdf" - response = self._send_request(file_name, dict(orient_analysis_cells=True, orient_cell_angle="270")) - table = response["content"]["tables"][0] - self._check_header_table(table["cells"]) - def test_pdf_table(self) -> None: file_name = "example_with_table1.pdf" result = self._send_request(file_name) diff --git a/tests/unit_tests/test_module_cell_splitter.py b/tests/unit_tests/test_module_cell_splitter.py index ad48952a..36113dbc 100644 --- a/tests/unit_tests/test_module_cell_splitter.py +++ b/tests/unit_tests/test_module_cell_splitter.py @@ -1,5 +1,7 @@ import unittest +from dedocutils.data_structures import BBox + from dedoc.readers.pdf_reader.data_classes.tables.cell import Cell from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.cell_splitter import CellSplitter @@ -10,42 +12,42 @@ class TestCellSplitter(unittest.TestCase): def test_merge_close_borders(self) -> None: cells = [ [ - Cell(x_top_left=0, y_top_left=0, x_bottom_right=50, y_bottom_right=30), - Cell(x_top_left=51, y_top_left=2, x_bottom_right=90, y_bottom_right=29) + Cell(BBox(x_top_left=0, y_top_left=0, width=50, height=30)), + Cell(BBox(x_top_left=51, y_top_left=2, width=39, height=27)) ], [ - Cell(x_top_left=0, y_top_left=31, x_bottom_right=50, y_bottom_right=50), - Cell(x_top_left=51, y_top_left=31, x_bottom_right=91, y_bottom_right=50) + Cell(BBox(x_top_left=0, y_top_left=31, width=50, height=19)), + Cell(BBox(x_top_left=51, y_top_left=31, width=40, height=19)) ] ] cells_merged = self.splitter._merge_close_borders(cells) - self.assertEqual(0, cells_merged[0][0].x_top_left) - self.assertEqual(0, cells_merged[0][0].y_top_left) - self.assertEqual(50, cells_merged[0][0].x_bottom_right) - self.assertEqual(29, cells_merged[0][0].y_bottom_right) - - self.assertEqual(50, cells_merged[0][1].x_top_left) - self.assertEqual(0, cells_merged[0][1].y_top_left) - self.assertEqual(90, cells_merged[0][1].x_bottom_right) - self.assertEqual(29, cells_merged[0][1].y_bottom_right) - - self.assertEqual(0, cells_merged[1][0].x_top_left) - self.assertEqual(29, cells_merged[1][0].y_top_left) - self.assertEqual(50, cells_merged[1][0].x_bottom_right) - self.assertEqual(50, cells_merged[1][0].y_bottom_right) - - self.assertEqual(50, cells_merged[1][1].x_top_left) - self.assertEqual(29, cells_merged[1][1].y_top_left) - self.assertEqual(90, cells_merged[1][1].x_bottom_right) - self.assertEqual(50, cells_merged[1][1].y_bottom_right) + self.assertEqual(0, cells_merged[0][0].bbox.x_top_left) + self.assertEqual(0, cells_merged[0][0].bbox.y_top_left) + self.assertEqual(50, cells_merged[0][0].bbox.x_bottom_right) + self.assertEqual(29, cells_merged[0][0].bbox.y_bottom_right) + + self.assertEqual(50, cells_merged[0][1].bbox.x_top_left) + self.assertEqual(0, cells_merged[0][1].bbox.y_top_left) + self.assertEqual(90, cells_merged[0][1].bbox.x_bottom_right) + self.assertEqual(29, cells_merged[0][1].bbox.y_bottom_right) + + self.assertEqual(0, cells_merged[1][0].bbox.x_top_left) + self.assertEqual(29, cells_merged[1][0].bbox.y_top_left) + self.assertEqual(50, cells_merged[1][0].bbox.x_bottom_right) + self.assertEqual(50, cells_merged[1][0].bbox.y_bottom_right) + + self.assertEqual(50, cells_merged[1][1].bbox.x_top_left) + self.assertEqual(29, cells_merged[1][1].bbox.y_top_left) + self.assertEqual(90, cells_merged[1][1].bbox.x_bottom_right) + self.assertEqual(50, cells_merged[1][1].bbox.y_bottom_right) def test_merge_close_borders_one_cell(self) -> None: - cells = [[Cell(x_top_left=0, y_top_left=0, x_bottom_right=50, y_bottom_right=30)]] + cells = [[Cell(BBox(x_top_left=0, y_top_left=0, width=50, height=30))]] cells_merged = self.splitter._merge_close_borders(cells) - self.assertEqual(0, cells_merged[0][0].x_top_left) - self.assertEqual(0, cells_merged[0][0].y_top_left) - self.assertEqual(50, cells_merged[0][0].x_bottom_right) - self.assertEqual(30, cells_merged[0][0].y_bottom_right) + self.assertEqual(0, cells_merged[0][0].bbox.x_top_left) + self.assertEqual(0, cells_merged[0][0].bbox.y_top_left) + self.assertEqual(50, cells_merged[0][0].bbox.x_bottom_right) + self.assertEqual(30, cells_merged[0][0].bbox.y_bottom_right) def test_merge_zero_cells(self) -> None: cells = [[]] @@ -58,24 +60,24 @@ def test_split_zero_cells(self) -> None: self.assertListEqual([[]], matrix) def test_split_one_cell(self) -> None: - cells = [[Cell(x_top_left=0, y_top_left=0, x_bottom_right=10, y_bottom_right=15)]] + cells = [[Cell(BBox(x_top_left=0, y_top_left=0, width=10, height=15))]] matrix = self.splitter.split(cells=cells) self.assertEqual(1, len(matrix)) self.assertEqual(1, len(matrix[0])) new_cell = matrix[0][0] - self.assertEqual(0, new_cell.x_top_left) - self.assertEqual(0, new_cell.y_top_left) - self.assertEqual(10, new_cell.x_bottom_right) - self.assertEqual(15, new_cell.y_bottom_right) + self.assertEqual(0, new_cell.bbox.x_top_left) + self.assertEqual(0, new_cell.bbox.y_top_left) + self.assertEqual(10, new_cell.bbox.x_bottom_right) + self.assertEqual(15, new_cell.bbox.y_bottom_right) def test_horizontal_split(self) -> None: cells = [ [ - Cell(x_top_left=0, y_top_left=0, x_bottom_right=3, y_bottom_right=5), - Cell(x_top_left=3, y_top_left=0, x_bottom_right=7, y_bottom_right=3), + Cell(BBox(x_top_left=0, y_top_left=0, width=3, height=5)), + Cell(BBox(x_top_left=3, y_top_left=0, width=4, height=3)), ], [ - Cell(x_top_left=3, y_top_left=3, x_bottom_right=7, y_bottom_right=5), + Cell(BBox(x_top_left=3, y_top_left=3, width=4, height=2)), ] ] matrix = self.splitter.split(cells) @@ -83,34 +85,34 @@ def test_horizontal_split(self) -> None: self.assertEqual(2, len(matrix[0])) self.assertEqual(2, len(matrix[1])) [cell_a, cell_b], [cell_c, cell_d] = matrix - self.assertEqual(0, cell_a.x_top_left) - self.assertEqual(0, cell_a.y_top_left) - self.assertEqual(3, cell_a.x_bottom_right) - self.assertEqual(3, cell_a.y_bottom_right) - - self.assertEqual(3, cell_b.x_top_left) - self.assertEqual(0, cell_b.y_top_left) - self.assertEqual(7, cell_b.x_bottom_right) - self.assertEqual(3, cell_b.y_bottom_right) - - self.assertEqual(0, cell_c.x_top_left) - self.assertEqual(3, cell_c.y_top_left) - self.assertEqual(3, cell_c.x_bottom_right) - self.assertEqual(5, cell_c.y_bottom_right) - - self.assertEqual(3, cell_d.x_top_left) - self.assertEqual(3, cell_d.y_top_left) - self.assertEqual(7, cell_d.x_bottom_right) - self.assertEqual(5, cell_d.y_bottom_right) + self.assertEqual(0, cell_a.bbox.x_top_left) + self.assertEqual(0, cell_a.bbox.y_top_left) + self.assertEqual(3, cell_a.bbox.x_bottom_right) + self.assertEqual(3, cell_a.bbox.y_bottom_right) + + self.assertEqual(3, cell_b.bbox.x_top_left) + self.assertEqual(0, cell_b.bbox.y_top_left) + self.assertEqual(7, cell_b.bbox.x_bottom_right) + self.assertEqual(3, cell_b.bbox.y_bottom_right) + + self.assertEqual(0, cell_c.bbox.x_top_left) + self.assertEqual(3, cell_c.bbox.y_top_left) + self.assertEqual(3, cell_c.bbox.x_bottom_right) + self.assertEqual(5, cell_c.bbox.y_bottom_right) + + self.assertEqual(3, cell_d.bbox.x_top_left) + self.assertEqual(3, cell_d.bbox.y_top_left) + self.assertEqual(7, cell_d.bbox.x_bottom_right) + self.assertEqual(5, cell_d.bbox.y_bottom_right) def test_vertical_split(self) -> None: cells = [ [ - Cell(x_top_left=0, y_top_left=0, x_bottom_right=8, y_bottom_right=2), + Cell(BBox(x_top_left=0, y_top_left=0, width=8, height=2)), ], [ - Cell(x_top_left=0, y_top_left=2, x_bottom_right=5, y_bottom_right=5), - Cell(x_top_left=5, y_top_left=2, x_bottom_right=8, y_bottom_right=5), + Cell(BBox(x_top_left=0, y_top_left=2, width=5, height=3)), + Cell(BBox(x_top_left=5, y_top_left=2, width=3, height=3)), ] ] matrix = self.splitter.split(cells) @@ -118,35 +120,35 @@ def test_vertical_split(self) -> None: self.assertEqual(2, len(matrix[0])) self.assertEqual(2, len(matrix[1])) [cell_a, cell_b], [cell_c, cell_d] = matrix - self.assertEqual(0, cell_a.x_top_left) - self.assertEqual(0, cell_a.y_top_left) - self.assertEqual(5, cell_a.x_bottom_right) - self.assertEqual(2, cell_a.y_bottom_right) - - self.assertEqual(5, cell_b.x_top_left) - self.assertEqual(0, cell_b.y_top_left) - self.assertEqual(8, cell_b.x_bottom_right) - self.assertEqual(2, cell_b.y_bottom_right) - - self.assertEqual(0, cell_c.x_top_left) - self.assertEqual(2, cell_c.y_top_left) - self.assertEqual(5, cell_c.x_bottom_right) - self.assertEqual(5, cell_c.y_bottom_right) - - self.assertEqual(5, cell_d.x_top_left) - self.assertEqual(2, cell_d.y_top_left) - self.assertEqual(8, cell_d.x_bottom_right) - self.assertEqual(5, cell_d.y_bottom_right) + self.assertEqual(0, cell_a.bbox.x_top_left) + self.assertEqual(0, cell_a.bbox.y_top_left) + self.assertEqual(5, cell_a.bbox.x_bottom_right) + self.assertEqual(2, cell_a.bbox.y_bottom_right) + + self.assertEqual(5, cell_b.bbox.x_top_left) + self.assertEqual(0, cell_b.bbox.y_top_left) + self.assertEqual(8, cell_b.bbox.x_bottom_right) + self.assertEqual(2, cell_b.bbox.y_bottom_right) + + self.assertEqual(0, cell_c.bbox.x_top_left) + self.assertEqual(2, cell_c.bbox.y_top_left) + self.assertEqual(5, cell_c.bbox.x_bottom_right) + self.assertEqual(5, cell_c.bbox.y_bottom_right) + + self.assertEqual(5, cell_d.bbox.x_top_left) + self.assertEqual(2, cell_d.bbox.y_top_left) + self.assertEqual(8, cell_d.bbox.x_bottom_right) + self.assertEqual(5, cell_d.bbox.y_bottom_right) def test_no_split(self) -> None: cells = [ [ - Cell(x_top_left=160, y_top_left=321, x_bottom_right=825, y_bottom_right=369), - Cell(x_top_left=825, y_top_left=321, x_bottom_right=1494, y_bottom_right=369) + Cell(BBox(x_top_left=160, y_top_left=321, width=665, height=48)), + Cell(BBox(x_top_left=825, y_top_left=321, width=669, height=48)) ], [ - Cell(x_top_left=160, y_top_left=374, x_bottom_right=825, y_bottom_right=423), - Cell(x_top_left=825, y_top_left=374, x_bottom_right=1494, y_bottom_right=423) + Cell(BBox(x_top_left=160, y_top_left=374, width=665, height=49)), + Cell(BBox(x_top_left=825, y_top_left=374, width=669, height=49)) ] ] diff --git a/tests/unit_tests/test_module_gost_frame_recognizer.py b/tests/unit_tests/test_module_gost_frame_recognizer.py index a2c33f09..1ac3a7c2 100644 --- a/tests/unit_tests/test_module_gost_frame_recognizer.py +++ b/tests/unit_tests/test_module_gost_frame_recognizer.py @@ -31,8 +31,6 @@ def _get_params_for_parse(self, parameters: Optional[dict], file_path: Optional[ file_path = file_path if file_path else "" params_for_parse = ParametersForParseDoc( language=param_utils.get_param_language(parameters), - orient_analysis_cells=param_utils.get_param_orient_analysis_cells(parameters), - orient_cell_angle=param_utils.get_param_orient_cell_angle(parameters), is_one_column_document=param_utils.get_param_is_one_column_document(parameters), document_orientation=param_utils.get_param_document_orientation(parameters), need_header_footers_analysis=param_utils.get_param_need_header_footers_analysis(parameters), diff --git a/tests/unit_tests/test_module_table_detection.py b/tests/unit_tests/test_module_table_detection.py index 0aef1be0..29d2e8da 100644 --- a/tests/unit_tests/test_module_table_detection.py +++ b/tests/unit_tests/test_module_table_detection.py @@ -7,13 +7,12 @@ from dedoc.readers.pdf_reader.data_classes.tables.scantable import ScanTable from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_recognizer import TableRecognizer -from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_utils.accuracy_table_rec import get_quantitative_parameters -from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_utils.utils import equal_with_eps, similarity as utils_similarity +from dedoc.readers.pdf_reader.pdf_image_reader.table_recognizer.table_utils.utils import equal_with_eps, get_statistic_values, similarity as sim from tests.test_utils import get_full_path, get_test_config def similarity(s1: str, s2: str, threshold: float = 0.8) -> bool: - return True if utils_similarity(s1, s2) > threshold else False + return True if sim(s1, s2) > threshold else False class TestRecognizedTable(unittest.TestCase): @@ -21,12 +20,7 @@ class TestRecognizedTable(unittest.TestCase): table_recognizer = TableRecognizer(config=get_test_config()) def get_table(self, image: np.ndarray, language: str = "rus", table_type: str = "") -> List[ScanTable]: - image, tables = self.table_recognizer.recognize_tables_from_image(image=image, - page_number=0, - language=language, - orient_analysis_cells=False, - orient_cell_angle=0, - table_type=table_type) + image, tables = self.table_recognizer.recognize_tables_from_image(image=image, page_number=0, language=language, table_type=table_type) return tables def test_table_wo_external_bounds(self) -> None: @@ -50,13 +44,13 @@ def test_table_split_right_column(self) -> None: image = cv2.imread(path_image, 0) tables = self.get_table(image, "rus+eng", table_type="split_last_column+wo_external_bounds") - self.assertTrue(tables[0].matrix_cells[4][-1].get_text(), "40703978900000345077") - self.assertTrue(tables[0].matrix_cells[5][-1].get_text(), "049401814") - self.assertTrue(tables[0].matrix_cells[6][-1].get_text(), "30101810200000000814") - self.assertTrue(tables[0].matrix_cells[7][-1].get_text(), "049401814") - self.assertTrue(tables[0].matrix_cells[8][-1].get_text(), "30101810200000000814") - self.assertTrue(tables[0].matrix_cells[9][-1].get_text(), "30110978700000070815") - self.assertTrue(tables[0].matrix_cells[10][-1].get_text(), "30110978700000070815") + self.assertTrue(tables[0].cells[4][-1].get_text(), "40703978900000345077") + self.assertTrue(tables[0].cells[5][-1].get_text(), "049401814") + self.assertTrue(tables[0].cells[6][-1].get_text(), "30101810200000000814") + self.assertTrue(tables[0].cells[7][-1].get_text(), "049401814") + self.assertTrue(tables[0].cells[8][-1].get_text(), "30101810200000000814") + self.assertTrue(tables[0].cells[9][-1].get_text(), "30110978700000070815") + self.assertTrue(tables[0].cells[10][-1].get_text(), "30110978700000070815") def test_table_extract_one_cell_and_one_cell_tables(self) -> None: path_image = get_full_path("data/lising/platezhka.jpg") @@ -115,73 +109,73 @@ def test_table_recognition_1(self) -> None: image = cv2.imread(get_full_path("data/tables/example_with_table3.png"), 0) tables = self.get_table(image) - cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_quantitative_parameters(tables[0].matrix_cells) + cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_statistic_values(tables[0].cells) self.assertEqual(cnt_rows, 8) self.assertEqual(cnt_columns, 3) self.assertEqual(cnt_a_cell, 3) self.assertEqual(cnt_cell, 24) - self.assertTrue(similarity(tables[0].matrix_cells[0][1].get_text(), "Наименование данных")) - self.assertTrue(similarity(tables[0].matrix_cells[0][2].get_text(), "Данные")) - self.assertTrue(similarity(tables[0].matrix_cells[4][1].get_text().capitalize(), "Инн")) - self.assertTrue(similarity(tables[0].matrix_cells[3][1].get_text(), "Руководитель (ФИО, телефон,\nфакс, электронный адрес)")) + self.assertTrue(similarity(tables[0].cells[0][1].get_text(), "Наименование данных")) + self.assertTrue(similarity(tables[0].cells[0][2].get_text(), "Данные")) + self.assertTrue(similarity(tables[0].cells[4][1].get_text().capitalize(), "Инн")) + self.assertTrue(similarity(tables[0].cells[3][1].get_text(), "Руководитель (ФИО, телефон,\nфакс, электронный адрес)")) def test_table_recognition_2(self) -> None: image = cv2.imread(get_full_path("data/tables/example_with_table4.jpg"), 0) tables = self.get_table(image) - cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_quantitative_parameters(tables[0].matrix_cells) + cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_statistic_values(tables[0].cells) self.assertEqual(cnt_rows, 5) self.assertEqual(cnt_columns, 3) self.assertEqual(cnt_a_cell, 3) self.assertEqual(cnt_cell, 15) - self.assertTrue(similarity(tables[0].matrix_cells[0][1].get_text(), "Перечень основных данных и\nтребований")) - self.assertTrue(similarity(tables[0].matrix_cells[0][2].get_text(), "Основные данные и требования")) - self.assertTrue(similarity(tables[0].matrix_cells[3][1].get_text(), "Количество")) - self.assertTrue(similarity(tables[0].matrix_cells[4][1].get_text(), "Технические параметры оборудования")) + self.assertTrue(similarity(tables[0].cells[0][1].get_text(), "Перечень основных данных и\nтребований")) + self.assertTrue(similarity(tables[0].cells[0][2].get_text(), "Основные данные и требования")) + self.assertTrue(similarity(tables[0].cells[3][1].get_text(), "Количество")) + self.assertTrue(similarity(tables[0].cells[4][1].get_text(), "Технические параметры оборудования")) def test_table_recognition_3(self) -> None: image = cv2.imread(get_full_path("data/tables/example_with_table5.png"), 0) tables = self.get_table(image) - cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_quantitative_parameters(tables[0].matrix_cells) + cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_statistic_values(tables[0].cells) self.assertEqual(cnt_rows, 13) self.assertEqual(cnt_columns, 3) self.assertEqual(cnt_a_cell, 3) self.assertEqual(cnt_cell, 39) - self.assertTrue(similarity(tables[0].matrix_cells[0][1].get_text(), "Техническая характеристика")) - self.assertTrue(similarity(tables[0].matrix_cells[0][2].get_text(), "Показатель")) - self.assertTrue(similarity(tables[0].matrix_cells[6][1].get_text(), "Использование крана и его механизмов")) - self.assertTrue(similarity(tables[0].matrix_cells[7][1].get_text(), "Тип привода:")) + self.assertTrue(similarity(tables[0].cells[0][1].get_text(), "Техническая характеристика")) + self.assertTrue(similarity(tables[0].cells[0][2].get_text(), "Показатель")) + self.assertTrue(similarity(tables[0].cells[6][1].get_text(), "Использование крана и его механизмов")) + self.assertTrue(similarity(tables[0].cells[7][1].get_text(), "Тип привода:")) def test_table_recognition_4(self) -> None: image = cv2.imread(get_full_path("data/tables/example_with_table5.png"), 0) tables = self.get_table(image) - cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_quantitative_parameters(tables[0].matrix_cells) + cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_statistic_values(tables[0].cells) self.assertEqual(cnt_rows, 13) self.assertEqual(cnt_columns, 3) self.assertEqual(cnt_a_cell, 3) self.assertEqual(cnt_cell, 39) - self.assertTrue(similarity(tables[0].matrix_cells[0][1].get_text(), "Техническая характеристика")) - self.assertTrue(similarity(tables[0].matrix_cells[0][2].get_text(), "Показатель")) - self.assertTrue(similarity(tables[0].matrix_cells[6][1].get_text(), "Использование крана и его механизмов")) - self.assertTrue(similarity(tables[0].matrix_cells[7][1].get_text(), "Тип привода:")) + self.assertTrue(similarity(tables[0].cells[0][1].get_text(), "Техническая характеристика")) + self.assertTrue(similarity(tables[0].cells[0][2].get_text(), "Показатель")) + self.assertTrue(similarity(tables[0].cells[6][1].get_text(), "Использование крана и его механизмов")) + self.assertTrue(similarity(tables[0].cells[7][1].get_text(), "Тип привода:")) def test_table_recognition_with_rotate_5(self) -> None: image = cv2.imread(get_full_path("data/tables/example_with_table6.png"), 0) tables = self.get_table(image) - cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_quantitative_parameters(tables[0].matrix_cells) + cnt_a_cell, cnt_cell, cnt_columns, cnt_rows = get_statistic_values(tables[0].cells) self.assertEqual(cnt_rows, 3) self.assertEqual(cnt_columns, 7) self.assertEqual(cnt_a_cell, 7) self.assertEqual(cnt_cell, 21) - self.assertTrue(similarity(tables[0].matrix_cells[0][1].get_text(), "Группа")) - self.assertTrue(similarity(tables[0].matrix_cells[0][3].get_text(), "Наименование")) - self.assertTrue(similarity(tables[0].matrix_cells[2][2].get_text(), "Новая\nпозиция")) - self.assertTrue(similarity(tables[0].matrix_cells[2][5].get_text(), "3 (три)\nшт.")) + self.assertTrue(similarity(tables[0].cells[0][1].get_text(), "Группа")) + self.assertTrue(similarity(tables[0].cells[0][3].get_text(), "Наименование")) + self.assertTrue(similarity(tables[0].cells[2][2].get_text(), "Новая\nпозиция")) + self.assertTrue(similarity(tables[0].cells[2][5].get_text(), "3 (три)\nшт."))