Merge pull request #16 from thombashi/develop

Add support for Markdown

MANIFEST.in

@@ -4,7 +4,6 @@ include README.rst
 include setup.cfg
 include tox.ini
 
-recursive-include docs *
 recursive-include requirements *
 recursive-include test *
 

README.rst

@@ -3,23 +3,29 @@ sqlitebiter
 
 .. image:: https://img.shields.io/pypi/pyversions/sqlitebiter.svg
    :target: https://pypi.python.org/pypi/sqlitebiter
-.. image:: https://travis-ci.org/thombashi/sqlitebiter.svg?branch=master
+
+.. image:: https://img.shields.io/travis/thombashi/sqlitebiter/master.svg?label=Linux
     :target: https://travis-ci.org/thombashi/sqlitebiter
-.. image:: https://ci.appveyor.com/api/projects/status/hunqrvo1inm2jjnj?svg=true
+    :alt: Linux CI test status
+
+.. image:: https://img.shields.io/appveyor/ci/thombashi/sqlitebiter/master.svg?label=Windows
     :target: https://ci.appveyor.com/project/thombashi/sqlitebiter
+    :alt: Windows CI test status
 
 Summary
 -------
 
-sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Google-Sheets to a SQLite database file.
+sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Markdown/Google-Sheets to a SQLite database file.
 
 Features
 --------
 
 - Create a SQLite database file from:
     - CSV file(s)
-    - JSON file(s)
     - Microsoft Excel :superscript:`TM` file(s)
+    - HTML file(s): extract table tag data
+    - JSON file(s)
+    - Markdown file(s): extract Markdown table
     - `Google Sheets <https://www.google.com/intl/en_us/sheets/about/>`_
 
 Usage
@@ -58,8 +64,10 @@ Dependency python packages are automatically installed during
 - `click <http://click.pocoo.org/>`__
 - `DataPropery <https://github.com/thombashi/DataProperty>`__
 - `path.py <https://github.com/jaraco/path.py>`__
+- `pytablereader <https://github.com/thombashi/pytablereader>`__
 - `SimpleSQLite <https://github.com/thombashi/SimpleSQLite>`__
 
+
 Google Sheets dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/pages/installation.rst

@@ -23,8 +23,10 @@ Dependency python packages are automatically installed during
 - `click <http://click.pocoo.org/>`__
 - `DataPropery <https://github.com/thombashi/DataProperty>`__
 - `path.py <https://github.com/jaraco/path.py>`__
+- `pytablereader <https://github.com/thombashi/pytablereader>`__
 - `SimpleSQLite <https://github.com/thombashi/SimpleSQLite>`__
 
+
 Google Sheets dependencies
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/pages/introduction/badges.txt

@@ -1,6 +1,10 @@
 .. image:: https://img.shields.io/pypi/pyversions/sqlitebiter.svg
    :target: https://pypi.python.org/pypi/sqlitebiter
-.. image:: https://travis-ci.org/thombashi/sqlitebiter.svg?branch=master
+
+.. image:: https://img.shields.io/travis/thombashi/sqlitebiter/master.svg?label=Linux
     :target: https://travis-ci.org/thombashi/sqlitebiter
-.. image:: https://ci.appveyor.com/api/projects/status/hunqrvo1inm2jjnj?svg=true
+    :alt: Linux CI test status
+
+.. image:: https://img.shields.io/appveyor/ci/thombashi/sqlitebiter/master.svg?label=Windows
     :target: https://ci.appveyor.com/project/thombashi/sqlitebiter
+    :alt: Windows CI test status

docs/pages/introduction/feature.txt

@@ -3,6 +3,8 @@ Features
 
 - Create a SQLite database file from:
     - CSV file(s)
-    - JSON file(s)
     - Microsoft Excel :superscript:`TM` file(s)
+    - HTML file(s): extract table tag data
+    - JSON file(s)
+    - Markdown file(s): extract Markdown table
     - `Google Sheets <https://www.google.com/intl/en_us/sheets/about/>`_

docs/pages/introduction/summary.txt

@@ -1 +1 @@
-sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Google-Sheets to a SQLite database file.
+sqlitebiter is a CLI tool to convert CSV/Excel/HTML/JSON/Markdown/Google-Sheets to a SQLite database file.

requirements/requirements.txt

@@ -1,5 +1,6 @@
 click
-DataProperty>=0.9.0
+DataProperty>=0.10.0
 logbook
 path.py
-SimpleSQLite>=0.5.5
+pytablereader>=0.2.0
+SimpleSQLite>=0.6.0

sqlitebiter/__init__.py

@@ -1 +1 @@
-VERSION = "0.2.1"
+VERSION = "0.3.0"

sqlitebiter/sqlitebiter.py

@@ -6,18 +6,14 @@
 """
 
 from __future__ import absolute_import
-import collections
-import re
 import sys
 
 import click
 import dataproperty
 import logbook
 import path
+import pytablereader as ptr
 import simplesqlite
-from simplesqlite.loader import ValidationError
-from simplesqlite.loader import InvalidDataError
-from simplesqlite.loader import OpenError
 
 from ._counter import ResultCounter
 
@@ -31,42 +27,6 @@
 handler.push_application()
 
 
-class LoaderNotFound(Exception):
-    pass
-
-
-class LoaderFactory(object):
-    LoaderTuple = collections.namedtuple(
-        "LoaderTuple", "filename_regexp loader")
-
-    __LOADERTUPLE_LIST = [
-        LoaderTuple(
-            re.compile("[\.]csv$"),
-            simplesqlite.loader.CsvTableFileLoader()),
-        LoaderTuple(
-            re.compile("[\.]html$|[\.]htm$"),
-            simplesqlite.loader.HtmlTableFileLoader()),
-        LoaderTuple(
-            re.compile("[\.]json$"),
-            simplesqlite.loader.JsonTableFileLoader()),
-        LoaderTuple(
-            re.compile("[\.]xlsx$|[\.]xlsm$|[\.]xls$"),
-            simplesqlite.loader.ExcelTableFileLoader()),
-    ]
-
-    @classmethod
-    def get_loader(cls, file_path):
-        for loadertuple in cls.__LOADERTUPLE_LIST:
-            if loadertuple.filename_regexp.search(file_path) is None:
-                continue
-
-            loadertuple.loader.source = file_path
-
-            return loadertuple.loader
-
-        raise LoaderNotFound(file_path)
-
-
 def create_database(database_path):
     db_path = path.Path(database_path)
     dir_path = db_path.dirname()
@@ -110,9 +70,12 @@ def cmd(ctx, log_level):
 @click.pass_context
 def file(ctx, files, output_path):
     """
-    Convert CSV/Excel/HTML/JSON file(s) to a SQLite database file.
+    Convert CSV/Excel/HTML/Markdown/JSON file(s) to a SQLite database file.
     """
 
+    if dataproperty.is_empty_sequence(files):
+        return 0
+
     con = create_database(output_path)
     result_counter = ResultCounter()
 
@@ -125,8 +88,15 @@ def file(ctx, files, output_path):
             continue
 
         try:
-            loader = LoaderFactory.get_loader(file_path)
-        except LoaderNotFound:
+            loader_factory = ptr.FileLoaderFactory(file_path)
+        except ptr.InvalidFilePathError:
+            continue
+
+        try:
+            loader = loader_factory.create_from_file_path()
+        except ptr.LoaderNotFound:
+            logger.debug(
+                "loader not found that coincide with '{}'".format(file_path))
             continue
 
         try:
@@ -142,14 +112,14 @@ def file(ctx, files, output_path):
 
                 click.echo("convert '{:s}' to '{:s}' table".format(
                     file_path, tabledata.table_name))
-        except OpenError as e:
+        except ptr.OpenError as e:
             logger.error(e)
-        except ValidationError as e:
+        except ptr.ValidationError as e:
             logger.error(
                 "invalid {:s} data format: path={:s}, message={:s}".format(
                     _get_format_type_from_path(file_path), file_path, str(e)))
             result_counter.inc_fail()
-        except InvalidDataError as e:
+        except ptr.InvalidDataError as e:
             logger.error(
                 "invalid {:s} data: path={:s}, message={:s}".format(
                     _get_format_type_from_path(file_path), file_path, str(e)))
@@ -193,13 +163,13 @@ def gs(ctx, credentials, title, output_path):
             try:
                 con.create_table_from_tabledata(tabledata)
                 result_counter.inc_success()
-            except (ValidationError, InvalidDataError):
+            except (ptr.ValidationError, ptr.InvalidDataError):
                 result_counter.inc_fail()
-    except OpenError as e:
+    except ptr.OpenError as e:
         logger.error(e)
     except AttributeError:
         logger.error("invalid credentials data: path={:s}".format(credentials))
-    except (ValidationError, InvalidDataError) as e:
+    except (ptr.ValidationError, ptr.InvalidDataError) as e:
         logger.error(
             "invalid credentials data: path={:s}, message={:s}".format(
                 credentials, str(e)))

test/test_sqlitebiter.py

@@ -11,7 +11,7 @@
 import xlsxwriter
 
 from sqlitebiter.sqlitebiter import cmd
-from simplesqlite.loader.interface import TableLoader
+from pytablereader.interface import TableLoader
 
 
 def valid_json_single_file():
@@ -237,6 +237,19 @@ def invalid_html_file():
     return file_path
 
 
+def valid_markdown_file():
+    file_path = "valid_mdtable.md"
+    with open(file_path, "w") as f:
+        f.write(""" a |  b  | c 
+--:|----:|---
+  1|123.1|a  
+  2|  2.2|bb 
+  3|  3.3|ccc
+""")
+
+    return file_path
+
+
 class Test_sqlitebiter:
 
     def setup_method(self, method):
@@ -264,13 +277,25 @@ def test_normal_smoke(self):
                 valid_csv_file2(),
                 valid_excel_file(),
                 valid_html_file(),
+                valid_markdown_file(),
             ]
 
             for file_path in file_list:
                 result = runner.invoke(
                     cmd, ["file", file_path, "-o", db_path])
                 assert result.exit_code == 0, file_path
 
+    def test_abnormal_empty(self):
+        runner = CliRunner()
+
+        with runner.isolated_filesystem():
+            result = runner.invoke(
+                cmd, ["file"])
+
+            assert result.exit_code == 0
+            assert not path.Path(
+                "out.sqlite").exists(), "output file must not exist"
+
     def test_abnormal_smoke(self):
         db_path = "test.sqlite"
         runner = CliRunner()
@@ -309,6 +334,8 @@ def test_normal_multi(self):
 
                 valid_html_file(),
                 invalid_html_file(),
+
+                valid_markdown_file(),
             ]
 
             result = runner.invoke(cmd, ["file"] + file_list + ["-o", db_path])
@@ -320,6 +347,7 @@ def test_normal_multi(self):
                 'csv_a', "insert_csv",
                 'excel_sheet_a', 'excel_sheet_c', 'excel_sheet_d',
                 'htmltable_tablename', 'htmltable_html2',
+                'valid_mdtable_markdown1',
             ]
 
             message = "expected-tables={}, actual-tables={}".format(
@@ -335,7 +363,8 @@ def test_normal_multi(self):
                 "multijson_table2":
                     [(1, '4'), (2, 'NULL'), (3, '120.9')],
                 "csv_a": [(1, 4.0, 'a'), (2, 2.1, 'bb'), (3, 120.9, 'ccc')],
-                "insert_csv": [(1, 4.0, 'a'), (2, 2.1, 'bb'), (3, 120.9, 'ccc')],
+                "insert_csv":
+                    [(1, 4.0, 'a'), (2, 2.1, 'bb'), (3, 120.9, 'ccc')],
                 "excel_sheet_a":
                     [(1.0, 1.1, 'a'), (2.0, 2.2, 'bb'), (3.0, 3.3, 'cc')],
                 "excel_sheet_c":
@@ -346,6 +375,8 @@ def test_normal_multi(self):
                     [(1, 123.1, 'a'), (2, 2.2, 'bb'), (3, 3.3, 'ccc')],
                 "htmltable_html2":
                     [(1, 123.1), (2, 2.2), (3, 3.3)],
+                "valid_mdtable_markdown1":
+                    [(1, 123.1, 'a'), (2, 2.2, 'bb'), (3, 3.3, 'ccc')],
             }
             for table in con.get_table_name_list():
                 result = con.select("*", table_name=table)