Merge pull request #60 from marioba/master

PUM integration

.travis.yml

@@ -1,14 +1,16 @@
 language: python
 
+python:
+  - 3.5
+
 addons:
   postgresql: "9.6"
   apt:
     packages:
       - postgresql-9.6-postgis-2.3
 
 install:
-  - pip install psycopg2 --quiet
-  - pip install pyyaml --quiet
+  - pip install -r requirements.txt
 
 env:
   # Github OAUTH token for creating pull requests
@@ -23,15 +25,42 @@ before_install:
   - git submodule update --init --recursive
 
 before_script:
-  - printf "[pg_qgep]\nhost=localhost\ndbname=qgep\nuser=postgres" > ~/.pg_service.conf
+  - printf "[pg_qgep]\nhost=localhost\ndbname=qgep\nuser=postgres\n" > ~/.pg_service.conf
   - psql -c 'CREATE DATABASE qgep;' -U postgres
   - psql -c 'CREATE EXTENSION postgis;' -U postgres -d qgep
   - psql -c 'CREATE EXTENSION hstore;' -U postgres -d qgep
 
+  # Append the 3 services used for pum's checks to pg_service.conf
+  - printf "[qgep_prod]\nhost=localhost\ndbname=qgep_prod\nuser=postgres\n\n[qgep_test]\nhost=localhost\ndbname=qgep_test\nuser=postgres\n\n[qgep_comp]\nhost=localhost\ndbname=qgep_comp\nuser=postgres\n" >> ~/.pg_service.conf
+
+  # Create the 3 databases used for pum's check
+  - psql -c 'CREATE DATABASE qgep_prod;' -U postgres
+  - psql -c 'CREATE DATABASE qgep_test;' -U postgres
+  - psql -c 'CREATE DATABASE qgep_comp;' -U postgres
+
+  - psql -c 'CREATE GROUP qgep;' -U postgres
+  - psql -c 'CREATE ROLE qgepuser LOGIN;' -U postgres
+  - psql -c 'GRANT qgep TO qgepuser;' -U postgres
+
 script:
   - $TRAVIS_BUILD_DIR/scripts/db_setup.sh -r > /dev/null
   - nosetests
 
+  # Get current version
+  - export VERSION=$(sed 'r' "$TRAVIS_BUILD_DIR/system/CURRENT_VERSION.txt")
+
+  # Create a db from a dump file. This simulate the prod db
+  - pum restore -p qgep_prod -x $TRAVIS_BUILD_DIR/test_data/qgep_dump_20180201.dump
+  - pum baseline -p qgep_prod -t qgep_sys.pum_info -d $TRAVIS_BUILD_DIR/delta/ -b 2018.02.01
+
+  # Create last version of qgep db using db_setup.sh script as comp db
+  - export PGSERVICE=qgep_comp
+  - $TRAVIS_BUILD_DIR/scripts/db_setup.sh > /dev/null
+  - pum baseline -p qgep_comp -t qgep_sys.pum_info -d $TRAVIS_BUILD_DIR/delta/ -b $VERSION
+
+  # Run pum's test and upgrade
+  - yes | pum test-and-upgrade -pp qgep_prod -pt qgep_test -pc qgep_comp -t qgep_sys.pum_info -d $TRAVIS_BUILD_DIR/delta/ -f /tmp/qwat_dump -i constraints views
+
 after_success:
   - $TRAVIS_BUILD_DIR/scripts/make_pull_request.sh
   - python $TRAVIS_BUILD_DIR/scripts/release_db_template.py

delta/delta_2018.02.01_bar.post.sql

@@ -0,0 +1 @@
+DROP TABLE foobar;

delta/delta_2018.02.01_foo.sql

@@ -0,0 +1,5 @@
+CREATE TABLE foobar (
+    code        char(5),
+    title       varchar(40),
+    did         integer
+);

export/vw_export_reach.py

@@ -105,4 +105,4 @@
     fkey: fk_operator
 """)
 
-print SqlExportView(pg_service, definition).sql()
+print(SqlExportView(pg_service, definition).sql())

export/vw_export_wasterwater_structure.py

@@ -107,4 +107,4 @@
     fkey: relevance
 """)
 
-print SqlExportView(pg_service, definition).sql()
+print(SqlExportView(pg_service, definition).sql())

requirements.txt

@@ -1 +1,3 @@
 psycopg2
+pyyaml
+pum

scripts/release_db_template.py

@@ -1,6 +1,6 @@
 import json
 import time
-import httplib
+import http.client
 import os
 from subprocess import call
 
@@ -15,7 +15,7 @@
     releasename=time.strftime("%Y%m%d-%H%M%S")
 
     # Create a release
-    conn = httplib.HTTPSConnection('api.github.com')
+    conn = http.client.HTTPSConnection('api.github.com')
     headers = {
       'User-Agent' : 'Deploy-Script',
       'Authorization': 'token {}'.format(os.environ['OAUTH_TOKEN'])
@@ -28,15 +28,15 @@
 
     release = json.load(response)
 
-    print release
+    print(release)
 
 
-    conn = httplib.HTTPSConnection('uploads.github.com')
+    conn = http.client.HTTPSConnection('uploads.github.com')
     headers['Content-Type'] = 'application/zip'
     url='{}?name={}'.format(release['upload_url'][:-13], 'template_db.dump')
-    print 'Upload to ' + url
+    print('Upload to ' + url)
 
     with open('/tmp/template_db.dump', 'r') as f:
         conn.request('POST', url, f, headers)
 
-    print conn.getresponse().read()
+    print(conn.getresponse().read())

system/CURRENT_VERSION.txt

@@ -0,0 +1 @@
+2018.02.02

test/utils.py

@@ -17,28 +17,28 @@ def cursor(self):
     def insert(self, table, row):
         cur = self.conn.cursor()
 
-        cols = row.keys()
+        cols = list(row.keys())
         cols_str = ','.join(cols)
-        vals = [unicode(row[x]) for x in cols]
+        vals = [str(row[x]) for x in cols]
         vals_str_list = ["%s"] * len(vals)
         vals_str = ','.join(vals_str_list)
 
         cur.execute(
             "INSERT INTO qgep_od.{table} ({cols}) VALUES ({vals_str}) RETURNING obj_id".format(table = table, cols=cols_str, vals_str=vals_str),
-            row.values()
+            list(row.values())
         )
 
         return cur.fetchone()[0]
 
     def update(self, table, row, obj_id):
         cur = self.conn.cursor()
 
-        cols = ['{}=%s'.format(key) for key, _ in row.iteritems()]
+        cols = ['{}=%s'.format(key) for key, _ in row.items()]
         cols_str = ','.join(cols)
 
         cur.execute(
             "UPDATE qgep_od.{table} SET {cols_str} WHERE obj_id=%s".format(table = table, cols_str=cols_str),
-            row.values() + [obj_id]
+            list(row.values()) + [obj_id]
         )
 
     def delete(self, table, obj_id):
@@ -73,8 +73,8 @@ def check_result(self, expected, result, table, test_name):
         # currently broken, that's the reason at the moment.
         self.assertTrue(result, "No result set received.")
 
-        for key, value in expected.iteritems():
-          self.assertEquals(unicode(result[key]), unicode(value), """
+        for key, value in expected.items():
+          self.assertEqual(str(result[key]), str(value), """
              ========================================================
 
              Data: {expected}

view/vw_damage.py

@@ -36,4 +36,4 @@
 """
 
 
-print pgiv.PGInheritanceView(pg_service, damage).sql_all()
+print(pgiv.PGInheritanceView(pg_service, damage).sql_all())

view/vw_maintenance_examination.py

@@ -33,4 +33,4 @@
 """
 
 
-print pgiv.PGInheritanceView(pg_service, maintenance).sql_all()
+print(pgiv.PGInheritanceView(pg_service, maintenance).sql_all())

view/vw_oo_organisation.py

@@ -63,4 +63,4 @@
 """
 
 
-print(pgiv.PGInheritanceView(pg_service, organisation).sql_all())
+print((pgiv.PGInheritanceView(pg_service, organisation).sql_all()))

view/vw_oo_overflow.py

@@ -56,4 +56,4 @@
 """.format(srid)
 
 
-print(pgiv.PGInheritanceView(pg_service, overflow).sql_all())
+print((pgiv.PGInheritanceView(pg_service, overflow).sql_all()))