Merge branch 'master' into consulta_de_licitacoes

* master:
  adding useful notebooks that helps in project
  adding new integration test
  encapsuling possibly errors on text exctration to avoid breaks total process
  Updating README.md
  reorganizing structure of tests to isolate unit tests from integration tests
  Creating new class called that wraps Dados.gov.br API for ComprasNet database (#9)
  {WIP} Fix date to search auctions #3
  {WIP} Get "endereço", "telefones", "fax", "entrada da proposta" and "abertura da proposta", info from  auction page #3
  {WIP} Fix tests #3
  {WIP} Get data from auctions page

Conflicts:
	comprasnet/__init__.py
	tests/test_class_comprasnet.py

Makefile

@@ -0,0 +1,6 @@
+test:
+	pytest tests/unit -s -vv --cov=comprasnet --cov-report term-missing
+
+integration_test:
+	pytest tests/integration -s -vv --cov=comprasnet --cov-report term-missing
+

README.md

@@ -12,5 +12,29 @@ How to install
 How to test
 ===========
 
-1. Install `pytest`
-1. Run `pytest tests/`
+To run all tests you need to install test requirements before:
+
+```bash
+pip install -r requirements-test.txt
+```
+
+Unit tests
+----------
+
+```bash
+make test
+```
+
+Integration tests
+-----------------
+
+```bash
+make integration_test
+```
+
+Examples to how to use
+======================
+
+See `python contrib/example.py`
+
+

comprasnet/api.py

@@ -0,0 +1,45 @@
+import logging
+
+import requests
+
+log = logging.getLogger('comprasnet')
+
+
+class ComprasNetApi:
+    BASE_SEARCH_URL = "http://compras.dados.gov.br/{modulo}/{version}/{metodo}.{formato}"
+    BASE_DETAIL_URL = "http://compras.dados.gov.br/{modulo}/{version}/{metodo}/{item}.{formato}"
+
+    def __init__(self):
+        self.last_response = None
+
+    def _raw_request(self, url, **params):
+        log.info('requesting to {}...'.format(url))
+        response = requests.get(url, params=params)
+        log.debug('response status code: {}'.format(response.status_code))
+
+        self.last_response = response
+        return response
+
+    def _request_search(self, modulo, metodo, version='v1', formato='json', **params):
+        url = self.BASE_SEARCH_URL.format(modulo=modulo, metodo=metodo, version=version,
+                                          formato=formato)
+        return self._raw_request(url, **params)
+
+    def _request_detail(self, modulo, metodo, item, version='id', formato='json', **params):
+        url = self.BASE_DETAIL_URL.format(modulo=modulo, metodo=metodo, version=version,
+                                          formato=formato, item=item)
+        return self._raw_request(url, **params)
+
+    def get_licitacoes_uasg(self, uasg_id, **params):
+        """http://compras.dados.gov.br/docs/licitacoes/uasg.html"""
+        response = self._request_detail('licitacoes', 'uasg', uasg_id, **params)
+        if response.status_code == 200:
+            return response.json()
+
+    def get_licitacoes_uasgs(self, **params):
+        """http://compras.dados.gov.br/docs/licitacoes/v1/uasgs.html"""
+        response = self._request_search('licitacoes', 'uasgs', **params)
+        if response.status_code == 200:
+            return response.json()
+
+

contrib/example.py

@@ -0,0 +1,36 @@
+from comprasnet import ComprasNet, ComprasNetApi
+import logging
+import sys
+from datetime import datetime
+
+logging.config.dictConfig({
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s'
+        },
+    },
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',
+            'formatter': 'standard',
+        },
+    },
+    'loggers': {
+        '': {
+            'level': 'DEBUG',
+            'handlers': ['console'],
+            'propagate': False
+        },
+    },
+})
+
+comprasnet = ComprasNet()
+results = comprasnet.search_auctions_by_date(datetime.strptime(sys.argv[1], '%d/%m/%Y'))
+print(results)
+
+
+
+comprasnet_api = ComprasNetApi()
+print(comprasnet_api.get_licitacoes_uasgs())

contrib/notebooks/Verificar quantas vezes cada item aparece em um json.ipynb

@@ -0,0 +1,79 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'abertura-da-proposta': 266,\n",
+      " 'abertura-da-proposta-str': 266,\n",
+      " 'cabecalho': 273,\n",
+      " 'cidade': 273,\n",
+      " 'codigo-da-uasg': 265,\n",
+      " 'edital-a-partir-de': 273,\n",
+      " 'edital-a-partir-de-str': 273,\n",
+      " 'endereco': 273,\n",
+      " 'entrega-da-proposta': 266,\n",
+      " 'entrega-da-proposta-str': 266,\n",
+      " 'fax': 273,\n",
+      " 'objeto': 273,\n",
+      " 'pregao-eletronico': 266,\n",
+      " 'td-style-padding': 8,\n",
+      " 'telefone': 273,\n",
+      " 'uf': 273}\n"
+     ]
+    }
+   ],
+   "source": [
+    "from json_tricks import load\n",
+    "\n",
+    "with open('/tmp/file.json') as handle:\n",
+    "    results = load(handle)\n",
+    "\n",
+    "keys = {}\n",
+    "for item in results:\n",
+    "    for key in item:\n",
+    "        if not key in keys:\n",
+    "            keys[key] = 0\n",
+    "        keys[key] += 1\n",
+    "\n",
+    "from pprint import pprint\n",
+    "pprint(keys)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

requirements-test.txt

@@ -0,0 +1,3 @@
+pytest==3.5.1
+pytest-cov==2.5.1
+

requirements.txt

@@ -1,2 +1,3 @@
 beautifulsoup4==4.6.0
+python-slugify==1.2.5
 requests==2.18.4