Remove python 2.7 support

.travis.yml

@@ -1,6 +1,5 @@
 language: python
 python:
-  - "2.7"
   - "3.6"
   - "3.7"
   - "3.8"

README.md

@@ -50,7 +50,6 @@ Get the code::
 
 Install the dependencies::
 
-    pip install -r requirements.txt
     pip install -r requirements-dev.txt
     pip install -e .
 
@@ -101,7 +100,6 @@ Supervisor to keep the process up.
 
 	 # Install the DataPusher and its requirements
 	 cd datapusher
-	 sudo /usr/lib/ckan/datapusher/bin/pip install -r requirements.txt
 	 sudo /usr/lib/ckan/datapusher/bin/python setup.py develop
 
      # Create a user to run the web service (if necessary)

datapusher/__init__.py

@@ -1 +1 @@
-__version__ = '0.0.17'
+__version__ = '0.1.0'

datapusher/jobs.py

@@ -1,12 +1,9 @@
 # -*- coding: utf-8 -*-
 
-
 import json
 import requests
-try:
-    from urllib.parse import urlsplit
-except ImportError:
-    from urlparse import urlsplit
+
+from urllib.parse import urlsplit
 
 import itertools
 import datetime
@@ -217,7 +214,7 @@ def datastore_resource_exists(resource_id, api_key, ckan_url):
         response = requests.post(search_url,
                                  verify=SSL_VERIFY,
                                  data=json.dumps({'id': resource_id,
-                                         'limit': 0}),
+                                                 'limit': 0}),
                                  headers={'Content-Type': 'application/json',
                                           'Authorization': api_key}
                                  )
@@ -338,6 +335,7 @@ def push_to_datastore(task_id, input, dry_run=False):
     except util.JobError as e:
         # try again in 5 seconds just incase CKAN is slow at adding resource
         time.sleep(5)
+        logger.debug('get_resource raised an error, retrying...' + e)
         resource = get_resource(resource_id, ckan_url, api_key)
 
     # check if the resource url_type is a datastore
@@ -418,12 +416,13 @@ def push_to_datastore(task_id, input, dry_run=False):
         table_set = messytables.any_tableset(tmp, mimetype=ct, extension=ct)
     except messytables.ReadError as e:
         # try again with format
+        logger.debug('messytables raised an error, trying with format...' + e)
         tmp.seek(0)
         try:
             format = resource.get('format')
             table_set = messytables.any_tableset(tmp, mimetype=format, extension=format)
-        except:
-            raise util.JobError(e)
+        except util.JobError as e:
+            raise (e)
 
     get_row_set = web.app.config.get('GET_ROW_SET',
                                      lambda table_set: table_set.tables.pop())
@@ -434,7 +433,7 @@ def push_to_datastore(task_id, input, dry_run=False):
     existing_info = None
     if existing:
         existing_info = dict((f['id'], f['info'])
-            for f in existing.get('fields', []) if 'info' in f)
+                             for f in existing.get('fields', []) if 'info' in f)
 
     # Some headers might have been converted from strings to floats and such.
     headers = [str(header) for header in headers]

datapusher/main.py

@@ -1,5 +1,4 @@
 import os
-import six
 import ckanserviceprovider.web as web
 
 from datapusher import jobs
@@ -25,12 +24,10 @@ def main():
         description='Service that allows automatic migration of data to the CKAN DataStore',
         epilog='''"He reached out and pressed an invitingly large red button on a nearby panel.
                 The panel lit up with the words Please do not press this button again."''')
-    if six.PY3:
-        argparser.add_argument('config', metavar='CONFIG', type=argparse.FileType('r'),
-                            help='configuration file')
-    if six.PY2:
-        argparser.add_argument('config', metavar='CONFIG', type=file,
-                            help='configuration file')
+
+    argparser.add_argument('config', metavar='CONFIG', type=argparse.FileType('r'),
+                        help='configuration file')
+
     args = argparser.parse_args()
 
     os.environ['JOB_CONFIG'] = os.path.abspath(args.config.name)

requirements-dev.txt

@@ -1,3 +1,3 @@
--r requirements.txt
-httpretty==0.9.4
+httpretty==1.0.2
 nose
+coverage

setup.py

@@ -29,6 +29,9 @@
     # Choose your license
     license='AGPL',
 
+    # Supported python distributions
+    python_requires='>=3.6',
+
     # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
         # How mature is this project? Common values are
@@ -42,7 +45,6 @@
 
         # Specify the Python versions you support here. In particular, ensure
         # that you indicate whether you support Python 2, Python 3 or both.
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
@@ -60,11 +62,15 @@
     # project is installed. For an analysis of "install_requires" vs pip's
     # requirements files see:
     # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files
-    install_requires=[],
+    install_requires=['argparse',
+                      'certifi',
+                      'ckanserviceprovider==0.0.10',
+                      'html5lib==1.1',
+                      'messytables==0.15.2',
+                      'requests[security]==2.24.0'],
 
     # If there are data files included in your packages that need to be
-    # installed, specify them here.  If using Python 2.6 or less, then these
-    # have to be included in MANIFEST.in as well.
+    # installed, specify them here.
     package_data={
     },