Merge pull request #74 from PRUNERS/devel

Devel

data/custom.mk

@@ -29,7 +29,7 @@ DEV_CFLAGS     +=
 DEV_LDFLAGS    +=
 
 # required compiler flags for CUDA
-NVCC_FLAGS     +=
+NVCC_CFLAGS    +=
 
 # required link flags for CUDA
 NVCC_LINK      +=

data/db/InstallFlitDB.sh

@@ -1014,14 +1014,14 @@ $$;
 -- Name: dofullflitimport(text, text); Type: FUNCTION; Schema: public; Owner: -
 --
 
-CREATE FUNCTION dofullflitimport(path text, notes text) RETURNS integer[]
+CREATE FUNCTION dofullflitimport(path text, label text) RETURNS integer[]
     LANGUAGE plpython3u
     AS $$
 import datetime
 
-query = ("INSERT INTO runs (rdate, notes) "
+query = ("INSERT INTO runs (rdate, label) "
          "VALUES ('" + str(datetime.datetime.now())  +
-         "','" + notes + "')")
+         "','" + label + "')")
 plpy.execute(query)
 query = ("SELECT MAX(index) from runs")
 res = plpy.execute(query)
@@ -1340,7 +1340,7 @@ CREATE TABLE opcodes (
 CREATE TABLE runs (
     index integer NOT NULL,
     rdate timestamp without time zone,
-    notes text
+    label text
 );
 
 

data/db/tables-psql.sql

@@ -355,14 +355,14 @@ $$;
 -- Name: dofullflitimport(text, text); Type: FUNCTION; Schema: public; Owner: -
 --
 
-CREATE FUNCTION dofullflitimport(path text, notes text) RETURNS integer[]
+CREATE FUNCTION dofullflitimport(path text, label text) RETURNS integer[]
     LANGUAGE plpython3u
     AS $$
 import datetime
 
-query = ("INSERT INTO runs (rdate, notes) "
+query = ("INSERT INTO runs (rdate, label) "
          "VALUES ('" + str(datetime.datetime.now())  +
-         "','" + notes + "')")
+         "','" + label + "')")
 plpy.execute(query)
 query = ("SELECT MAX(index) from runs")
 res = plpy.execute(query)
@@ -681,7 +681,7 @@ CREATE TABLE opcodes (
 CREATE TABLE runs (
     index integer NOT NULL,
     rdate timestamp without time zone,
-    notes text
+    label text
 );
 
 

data/db/tables-sqlite.sql

@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS runs (
   rdate          timestamp,
 
   -- The message describing what this run is all about
-  notes          text
+  label          text
   );
 
 --

documentation/database-structure.md

@@ -30,8 +30,8 @@ CREATE TABLE runs (
   -- it and convert it to a sqlite3 basic type and back.
   rdate          timestamp,
 
-  -- The message describing what this run is all about
-  notes          text
+  -- The label for the run describing what it is about
+  label          text
   );
 CREATE TABLE tests (
   id             integer    primary key autoincrement     not null,
@@ -57,7 +57,7 @@ This output is as of this writing.  You can execute those same commands to see
 the exact schema used in your version of FLiT.
 
 The `runs` table only stores information about each executed full run, the id,
-datetime and user-specified label for the run (called `notes`).
+datetime and user-specified label for the run (called `label`).
 
 The `tests` table contains the actual test results.  Each row has a run number
 that matches the `id` field of the `runs` table, so you can do things like:

scripts/flitcli/flit_import.py

@@ -35,34 +35,30 @@ def main(arguments, prog=sys.argv[0]):
                             File(s) to import into the database.  These files
                             may be csv files or sqlite3 databases.
                             ''')
-    #parser.add_argument('-t', '--table', default='tests',
-    #                    help='''
-    #                        The database table used for import (default is tests)
-    #                        ''')
-    parser.add_argument('-r', '--run', type=int, default=-1,
+    parser.add_argument('-a', '--append', type=int, default=None, metavar='RUN_ID',
                         help='''
-                            The run number to import under.  If the run does
-                            not exist in the runs table, then a new entry will
-                            be created with an autogenerated message.  The
-                            default behavior is to use the latest run in the
-                            database (but one will be created if there is no
-                            runs).  If importing from an sqlite database, the
-                            run column of the tests table is ignored since
-                            there is no necessary correlation between run
-                            numbers of that database and this database.  So you
-                            would still want to use the --run option.  For the
-                            sqlite database case, the results imported will
-                            only be from the latest run in the importing
-                            database.
+                            Append the import to the specified run id.  The
+                            default behavior is to add a new run to include the
+                            results of the import.  You must specify a run id
+                            that already exists in the database.
                             ''')
-    parser.add_argument('--new-run', action='store_true',
+    parser.add_argument('-l', '--label', default='Imported using flit import',
                         help='''
-                            Specifies that this import should be under a new
-                            run number that will be autogenerated.  This option
-                            conflicts with the --run option, meaning if this
-                            argument is specified, then the --run argument will
-                            be ignored.  This option is also implied if the
-                            destination database has no runs in it.
+                            The label to attach to the run.  Only applicable
+                            when creating a new run.  This argument is ignored
+                            if --append is specified.  The default label is
+                            'Imported using flit import'.
+                            ''')
+    parser.add_argument('-r', '--run', type=int, default=None,
+                        help='''
+                            Only applicable to the importing of sqlite3
+                            database files.  This will apply to all sqlite3
+                            database files passed in.  Only this run id will be
+                            imported from the provided database.  The default
+                            behavior is to import the latest run.  You cannot
+                            specify more than one run to be imported, you must
+                            call this program multiple times, each one with
+                            --run specified to the next run you want to import.
                             ''')
     args = parser.parse_args(arguments)
 
@@ -77,47 +73,42 @@ def main(arguments, prog=sys.argv[0]):
             'Only sqlite database supported'
     db = util.sqlite_open(projconf['database']['filepath'])
 
-    # if the database has no runs, then turn on --new-run
-    run_ids = [x['id'] for x in db.execute('select id from runs')]
-    if len(run_ids) == 0:
-        args.new_run = True
-    print('run_ids: ', run_ids)
-
-    # Find the destination run
-    if not args.new_run:
-        assert args.run <= 0 or args.run in run_ids, \
-                'Specified run {0} is not in the runs table'.format(args.run)
-        if args.run not in run_ids:
-            args.run = sorted(run_ids)[-1]
-    else: # args.new_run
+    # create a new run and set the args.append run id
+    if args.append is None:
         # Create a new run to use in import
-        db.execute('insert into runs(rdate,notes) values (?,?)',
-                (datetime.datetime.now(), 'Imported using flit import'))
+        db.execute('insert into runs(rdate,label) values (?,?)',
+                (datetime.datetime.now(), args.label))
         db.commit()
-        args.run = db.execute('select id from runs order by id').fetchall()[-1]['id']
+        args.append = db.execute('select id from runs order by id').fetchall()[-1]['id']
+
+    # Make sure the run id exists.
+    run_ids = [x['id'] for x in db.execute('select id from runs')]
+    assert args.append in run_ids, \
+            'Specified append run id {0} is not in the runs ' \
+            'table'.format(args.append)
 
     for importee in args.importfile:
-        print(importee)
+        print('Importing', importee)
         if util.is_sqlite(importee):
-            # Try to treat the importfile like a sqlite database
             import_db = util.sqlite_open(importee)
             cur = import_db.cursor()
             cur.execute('select id from runs')
-            run_ids = sorted([x['id'] for x in cur])
-            if len(run_ids) == 0:
-                print('  nothing to import')
+            importee_run_ids = sorted([x['id'] for x in cur])
+            if len(importee_run_ids) == 0:
+                print('  no runs in database: nothing to import')
                 continue
-            latest_run = run_ids[-1]
+            latest_run = importee_run_ids[-1]
+            import_run = args.run if args.run is not None else latest_run
             cur.execute('select name,host,compiler,optl,switches,precision,'
                         'comparison,comparison_d,file,nanosec '
-                        'from tests where run = ?', (latest_run,))
-            rows = cur.fetchall()
+                        'from tests where run = ?', (import_run,))
+            rows = [dict(x) for x in cur]
         else:
             with open(importee, 'r') as csvin:
                 reader = csv.DictReader(csvin)
                 rows = [row for row in reader]
         if len(rows) == 0:
-            print('  nothing to import')
+            print('  zero rows: nothing to import')
             continue
         to_insert = []
         for row in rows:
@@ -126,7 +117,7 @@ def main(arguments, prog=sys.argv[0]):
                 row[key] = val if val != 'NULL' else None
             # Insert
             to_insert.append((
-                args.run,
+                args.append,
                 row['name'],
                 row['host'],
                 row['compiler'],

scripts/flitcli/flit_update.py

@@ -81,7 +81,8 @@ def main(arguments, prog=sys.argv[0]):
             'ground_truth_switches': gt_switches,
             'flit_include_dir': conf.include_dir,
             'flit_lib_dir': conf.lib_dir,
-            'flit_script': os.path.join(conf.script_dir, 'flit.py'),
+            'flit_data_dir': conf.data_dir,
+            'flit_script_dir': conf.script_dir,
         },
         overwrite=True)
 

scripts/run_all.py

@@ -23,7 +23,7 @@
 home_dir = os.path.dirname(os.path.realpath(__file__))
 
 #vars
-notes = ''
+label = ''
 DB_HOST_AUX = '/tmp/flitDbDir'
 DBINIT = 'prepDBHost.py'
 db_host = hostfile.DB_HOST
@@ -37,7 +37,7 @@
 pwds = {}
 
 def usage():
-    print('usage: ' + sys.argv[0] + ' "notes"')
+    print('usage: ' + sys.argv[0] + ' "label"')
     print('\tyou must populate ' + home_dir + '/hostfile.py with')
     print('\trun and db host info (see file for details)')
 
@@ -116,7 +116,7 @@ def getPasswords():
             )
 
 if len(sys.argv) == 2:
-    notes = sys.argv[1]
+    label = sys.argv[1]
 
 else:
     usage()
@@ -150,8 +150,8 @@ def getPasswords():
 # #get run# from db
 print(check_output(['sshpass', '-e', *SSHL,
                     db_host[0] + '@' + db_host[1],
-              'psql flit -t -c "insert into runs (rdate, notes) ' +
-              'values (\'' + str(datetime.now()) + '\', \'' + notes + '\')"'],
+              'psql flit -t -c "insert into runs (rdate, label) ' +
+              'values (\'' + str(datetime.now()) + '\', \'' + label + '\')"'],
                    env=new_env).decode("utf-8"))
 run_num = int(check_output(['sshpass', '-e', *SSHL,
                             db_host[0] + '@' + db_host[1],