diff --git a/MANIFEST.in b/MANIFEST.in index 04a35bd..c13f04e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,7 +4,6 @@ include README.rst include setup.cfg include tox.ini -recursive-include docs * recursive-include requirements * recursive-include test * diff --git a/README.rst b/README.rst index ed33b49..3d3da4a 100644 --- a/README.rst +++ b/README.rst @@ -3,23 +3,29 @@ sqlitebiter .. image:: https://img.shields.io/pypi/pyversions/sqlitebiter.svg :target: https://pypi.python.org/pypi/sqlitebiter -.. image:: https://travis-ci.org/thombashi/sqlitebiter.svg?branch=master + +.. image:: https://img.shields.io/travis/thombashi/sqlitebiter/master.svg?label=Linux :target: https://travis-ci.org/thombashi/sqlitebiter -.. image:: https://ci.appveyor.com/api/projects/status/hunqrvo1inm2jjnj?svg=true + :alt: Linux CI test status + +.. image:: https://img.shields.io/appveyor/ci/thombashi/sqlitebiter/master.svg?label=Windows :target: https://ci.appveyor.com/project/thombashi/sqlitebiter + :alt: Windows CI test status Summary ------- -sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Google-Sheets to a SQLite database file. +sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Markdown/Google-Sheets to a SQLite database file. Features -------- - Create a SQLite database file from: - CSV file(s) - - JSON file(s) - Microsoft Excel :superscript:`TM` file(s) + - HTML file(s): extract table tag data + - JSON file(s) + - Markdown file(s): extract Markdown table - `Google Sheets `_ Usage @@ -58,8 +64,10 @@ Dependency python packages are automatically installed during - `click `__ - `DataPropery `__ - `path.py `__ +- `pytablereader `__ - `SimpleSQLite `__ + Google Sheets dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/pages/installation.rst b/docs/pages/installation.rst index b469fc7..8ba84dd 100644 --- a/docs/pages/installation.rst +++ b/docs/pages/installation.rst @@ -23,8 +23,10 @@ Dependency python packages are automatically installed during - `click `__ - `DataPropery `__ - `path.py `__ +- `pytablereader `__ - `SimpleSQLite `__ + Google Sheets dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/pages/introduction/badges.txt b/docs/pages/introduction/badges.txt index 2f61a2a..a2a1f68 100644 --- a/docs/pages/introduction/badges.txt +++ b/docs/pages/introduction/badges.txt @@ -1,6 +1,10 @@ .. image:: https://img.shields.io/pypi/pyversions/sqlitebiter.svg :target: https://pypi.python.org/pypi/sqlitebiter -.. image:: https://travis-ci.org/thombashi/sqlitebiter.svg?branch=master + +.. image:: https://img.shields.io/travis/thombashi/sqlitebiter/master.svg?label=Linux :target: https://travis-ci.org/thombashi/sqlitebiter -.. image:: https://ci.appveyor.com/api/projects/status/hunqrvo1inm2jjnj?svg=true + :alt: Linux CI test status + +.. image:: https://img.shields.io/appveyor/ci/thombashi/sqlitebiter/master.svg?label=Windows :target: https://ci.appveyor.com/project/thombashi/sqlitebiter + :alt: Windows CI test status diff --git a/docs/pages/introduction/feature.txt b/docs/pages/introduction/feature.txt index ea23b76..92f4553 100644 --- a/docs/pages/introduction/feature.txt +++ b/docs/pages/introduction/feature.txt @@ -3,6 +3,8 @@ Features - Create a SQLite database file from: - CSV file(s) - - JSON file(s) - Microsoft Excel :superscript:`TM` file(s) + - HTML file(s): extract table tag data + - JSON file(s) + - Markdown file(s): extract Markdown table - `Google Sheets `_ diff --git a/docs/pages/introduction/summary.txt b/docs/pages/introduction/summary.txt index edba12d..f4d8bcd 100644 --- a/docs/pages/introduction/summary.txt +++ b/docs/pages/introduction/summary.txt @@ -1 +1 @@ -sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Google-Sheets to a SQLite database file. +sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Markdown/Google-Sheets to a SQLite database file. diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f6ec73c..e02fd54 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,6 @@ click -DataProperty>=0.9.0 +DataProperty>=0.10.0 logbook path.py -SimpleSQLite>=0.5.5 +pytablereader>=0.2.0 +SimpleSQLite>=0.6.0 diff --git a/sqlitebiter/__init__.py b/sqlitebiter/__init__.py index 91ee717..867e4b1 100644 --- a/sqlitebiter/__init__.py +++ b/sqlitebiter/__init__.py @@ -1 +1 @@ -VERSION = "0.2.1" +VERSION = "0.3.0" diff --git a/sqlitebiter/sqlitebiter.py b/sqlitebiter/sqlitebiter.py index a351131..d997aae 100644 --- a/sqlitebiter/sqlitebiter.py +++ b/sqlitebiter/sqlitebiter.py @@ -6,18 +6,14 @@ """ from __future__ import absolute_import -import collections -import re import sys import click import dataproperty import logbook import path +import pytablereader as ptr import simplesqlite -from simplesqlite.loader import ValidationError -from simplesqlite.loader import InvalidDataError -from simplesqlite.loader import OpenError from ._counter import ResultCounter @@ -31,42 +27,6 @@ handler.push_application() -class LoaderNotFound(Exception): - pass - - -class LoaderFactory(object): - LoaderTuple = collections.namedtuple( - "LoaderTuple", "filename_regexp loader") - - __LOADERTUPLE_LIST = [ - LoaderTuple( - re.compile("[\.]csv$"), - simplesqlite.loader.CsvTableFileLoader()), - LoaderTuple( - re.compile("[\.]html$|[\.]htm$"), - simplesqlite.loader.HtmlTableFileLoader()), - LoaderTuple( - re.compile("[\.]json$"), - simplesqlite.loader.JsonTableFileLoader()), - LoaderTuple( - re.compile("[\.]xlsx$|[\.]xlsm$|[\.]xls$"), - simplesqlite.loader.ExcelTableFileLoader()), - ] - - @classmethod - def get_loader(cls, file_path): - for loadertuple in cls.__LOADERTUPLE_LIST: - if loadertuple.filename_regexp.search(file_path) is None: - continue - - loadertuple.loader.source = file_path - - return loadertuple.loader - - raise LoaderNotFound(file_path) - - def create_database(database_path): db_path = path.Path(database_path) dir_path = db_path.dirname() @@ -110,9 +70,12 @@ def cmd(ctx, log_level): @click.pass_context def file(ctx, files, output_path): """ - Convert CSV/Excel/HTML/JSON file(s) to a SQLite database file. + Convert CSV/Excel/HTML/Markdown/JSON file(s) to a SQLite database file. """ + if dataproperty.is_empty_sequence(files): + return 0 + con = create_database(output_path) result_counter = ResultCounter() @@ -125,8 +88,15 @@ def file(ctx, files, output_path): continue try: - loader = LoaderFactory.get_loader(file_path) - except LoaderNotFound: + loader_factory = ptr.FileLoaderFactory(file_path) + except ptr.InvalidFilePathError: + continue + + try: + loader = loader_factory.create_from_file_path() + except ptr.LoaderNotFound: + logger.debug( + "loader not found that coincide with '{}'".format(file_path)) continue try: @@ -142,14 +112,14 @@ def file(ctx, files, output_path): click.echo("convert '{:s}' to '{:s}' table".format( file_path, tabledata.table_name)) - except OpenError as e: + except ptr.OpenError as e: logger.error(e) - except ValidationError as e: + except ptr.ValidationError as e: logger.error( "invalid {:s} data format: path={:s}, message={:s}".format( _get_format_type_from_path(file_path), file_path, str(e))) result_counter.inc_fail() - except InvalidDataError as e: + except ptr.InvalidDataError as e: logger.error( "invalid {:s} data: path={:s}, message={:s}".format( _get_format_type_from_path(file_path), file_path, str(e))) @@ -193,13 +163,13 @@ def gs(ctx, credentials, title, output_path): try: con.create_table_from_tabledata(tabledata) result_counter.inc_success() - except (ValidationError, InvalidDataError): + except (ptr.ValidationError, ptr.InvalidDataError): result_counter.inc_fail() - except OpenError as e: + except ptr.OpenError as e: logger.error(e) except AttributeError: logger.error("invalid credentials data: path={:s}".format(credentials)) - except (ValidationError, InvalidDataError) as e: + except (ptr.ValidationError, ptr.InvalidDataError) as e: logger.error( "invalid credentials data: path={:s}, message={:s}".format( credentials, str(e))) diff --git a/test/test_sqlitebiter.py b/test/test_sqlitebiter.py index 9d9ec1f..ef3d36d 100644 --- a/test/test_sqlitebiter.py +++ b/test/test_sqlitebiter.py @@ -11,7 +11,7 @@ import xlsxwriter from sqlitebiter.sqlitebiter import cmd -from simplesqlite.loader.interface import TableLoader +from pytablereader.interface import TableLoader def valid_json_single_file(): @@ -237,6 +237,19 @@ def invalid_html_file(): return file_path +def valid_markdown_file(): + file_path = "valid_mdtable.md" + with open(file_path, "w") as f: + f.write(""" a | b | c +--:|----:|--- + 1|123.1|a + 2| 2.2|bb + 3| 3.3|ccc +""") + + return file_path + + class Test_sqlitebiter: def setup_method(self, method): @@ -264,6 +277,7 @@ def test_normal_smoke(self): valid_csv_file2(), valid_excel_file(), valid_html_file(), + valid_markdown_file(), ] for file_path in file_list: @@ -271,6 +285,17 @@ def test_normal_smoke(self): cmd, ["file", file_path, "-o", db_path]) assert result.exit_code == 0, file_path + def test_abnormal_empty(self): + runner = CliRunner() + + with runner.isolated_filesystem(): + result = runner.invoke( + cmd, ["file"]) + + assert result.exit_code == 0 + assert not path.Path( + "out.sqlite").exists(), "output file must not exist" + def test_abnormal_smoke(self): db_path = "test.sqlite" runner = CliRunner() @@ -309,6 +334,8 @@ def test_normal_multi(self): valid_html_file(), invalid_html_file(), + + valid_markdown_file(), ] result = runner.invoke(cmd, ["file"] + file_list + ["-o", db_path]) @@ -320,6 +347,7 @@ def test_normal_multi(self): 'csv_a', "insert_csv", 'excel_sheet_a', 'excel_sheet_c', 'excel_sheet_d', 'htmltable_tablename', 'htmltable_html2', + 'valid_mdtable_markdown1', ] message = "expected-tables={}, actual-tables={}".format( @@ -335,7 +363,8 @@ def test_normal_multi(self): "multijson_table2": [(1, '4'), (2, 'NULL'), (3, '120.9')], "csv_a": [(1, 4.0, 'a'), (2, 2.1, 'bb'), (3, 120.9, 'ccc')], - "insert_csv": [(1, 4.0, 'a'), (2, 2.1, 'bb'), (3, 120.9, 'ccc')], + "insert_csv": + [(1, 4.0, 'a'), (2, 2.1, 'bb'), (3, 120.9, 'ccc')], "excel_sheet_a": [(1.0, 1.1, 'a'), (2.0, 2.2, 'bb'), (3.0, 3.3, 'cc')], "excel_sheet_c": @@ -346,6 +375,8 @@ def test_normal_multi(self): [(1, 123.1, 'a'), (2, 2.2, 'bb'), (3, 3.3, 'ccc')], "htmltable_html2": [(1, 123.1), (2, 2.2), (3, 3.3)], + "valid_mdtable_markdown1": + [(1, 123.1, 'a'), (2, 2.2, 'bb'), (3, 3.3, 'ccc')], } for table in con.get_table_name_list(): result = con.select("*", table_name=table)