Merge pull request #24 from jburke-cadc/t74572

T74572
opencadc · Sep 22, 2017 · 69410c9 · 69410c9
2 parents e790a77 + b4ed971
commit 69410c9
Show file tree

Hide file tree

Showing 26 changed files with 8,651 additions and 8,352 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,6 +1,12 @@
 language: java
+
+dist: trusty
+sudo: required
+
 jdk:
   - openjdk7
   - oraclejdk8
-script: for mod in cadc-tap-schema cadc-jsqlparser-compat cadc-tap-server   cadc-adql cadc-test-tap  example-tap; do cd $mod; gradle --stacktrace build javadoc install || break -1; cd ..; done 
 
+before_script: openssl s_client -CApath /etc/ssl/certs/ -connect plugins.gradle.org:443 </dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' > /tmp/gradle.crt; sudo keytool -importcert -noprompt -file /tmp/gradle.crt -trustcacerts -keystore $JAVA_HOME/jre/lib/security/cacerts -alias root -storepass changeit;
+
+script: for mod in cadc-tap-schema cadc-jsqlparser-compat cadc-tap-server cadc-adql cadc-test-tap example-tap; do cd $mod; gradle --stacktrace build javadoc install || break -1; cd ..; done
diff --git a/cadc-adql/build.gradle b/cadc-adql/build.gradle
@@ -14,15 +14,15 @@ sourceCompatibility = 1.7
 
 group = 'org.opencadc'
 
-version = '1.0.1'
+version = '1.0.2'
 
 dependencies {
     compile 'log4j:log4j:1.2.+'
 
     compile 'org.opencadc:cadc-util:1.+'
     compile 'org.opencadc:cadc-tap-server:1.+'
     compile 'org.opencadc:cadc-tap-schema:1.+'
-    compile 'org.opencadc:cadc-jsqlparser-compat:0.6.2a'
+    compile 'org.opencadc:cadc-jsqlparser-compat:0.6.4'
 
     testCompile 'junit:junit:4.+'
 }
diff --git a/cadc-adql/src/main/java/ca/nrc/cadc/tap/AdqlQuery.java b/cadc-adql/src/main/java/ca/nrc/cadc/tap/AdqlQuery.java
@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2009.                            (c) 2009.
+*  (c) 2017.                            (c) 2017.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -139,27 +139,24 @@ public AdqlQuery()
      */
     protected void init()
     {
-        ExpressionNavigator endef = new ExpressionNavigator();
-        ReferenceNavigator rndef = new ReferenceNavigator();
-        FromItemNavigator fndef = new FromItemNavigator();
-
         // default validator: table and columns in tap_schema, 
         // blobs and clobs in select list only
-        ExpressionNavigator en = new ExpressionValidator(tapSchema);
-        ReferenceNavigator rn = new BlobClobColumnValidator(tapSchema);
-        FromItemNavigator fn = new TapSchemaTableValidator(tapSchema);
-        SelectNavigator sn = new SelectNavigator(en, rn, fn);
+        SelectNavigator sn = new SelectNavigator(new ExpressionValidator(tapSchema),
+                                                 new BlobClobColumnValidator(tapSchema),
+                                                 new TapSchemaTableValidator(tapSchema));
         navigatorList.add(sn);
 
         // convert * to fixed select-list
-        sn = new AllColumnConverter(endef, rndef, fndef, tapSchema);
+        sn = new AllColumnConverter(new ExpressionNavigator(),
+                                    new ReferenceNavigator(),
+                                    new FromItemNavigator(),
+                                    tapSchema);
         navigatorList.add(sn);
 
         // extract select-list
-        en = new SelectListExpressionExtractor(tapSchema);
-        rn = rndef;
-        fn = fndef;
-        sn = new SelectListExtractor(en, rn, fn);
+        sn = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema),
+                                     new ReferenceNavigator(),
+                                     new FromItemNavigator());
         navigatorList.add(sn);
 
         // support for file uploads to map the upload table name to the query table name.
@@ -174,7 +171,7 @@ protected void init()
                 tnc.put(tableDesc.getTableName(), newName);
                 log.debug("TableNameConverter " + tableDesc.getTableName() + " -> " + newName);
             }
-            sn = new SelectNavigator(endef, rndef, tnc);
+            sn = new SelectNavigator(new ExpressionNavigator(), new ReferenceNavigator(), tnc);
             navigatorList.add(sn);
         }
 

diff --git a/cadc-adql/src/main/java/ca/nrc/cadc/tap/SqlQuery.java b/cadc-adql/src/main/java/ca/nrc/cadc/tap/SqlQuery.java
@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2009.                            (c) 2009.
+*  (c) 2017.                            (c) 2017.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -125,24 +125,22 @@ public SqlQuery() { }
      */
     protected void init()
     {
-        ExpressionNavigator endef = new ExpressionNavigator();
-        ReferenceNavigator rndef = new ReferenceNavigator();
-        FromItemNavigator fndef = new FromItemNavigator();
-
-        ReferenceNavigator rn = new TapSchemaColumnValidator(tapSchema);
-        FromItemNavigator fn = new TapSchemaTableValidator(tapSchema);
-        SelectNavigator sn = new SelectNavigator(endef, rn, fn);
+        SelectNavigator sn = new SelectNavigator(new ExpressionNavigator(),
+                                                 new TapSchemaColumnValidator(tapSchema),
+                                                 new TapSchemaTableValidator(tapSchema));
         navigatorList.add(sn);
 
         // convert * to fixed select-list
-        sn = new AllColumnConverter(endef, rndef, fndef, tapSchema);
+        sn = new AllColumnConverter(new ExpressionNavigator(),
+                                    new ReferenceNavigator(),
+                                    new FromItemNavigator(),
+                                    tapSchema);
         navigatorList.add(sn);
 
         // extract select-list
-        ExpressionNavigator en = new SelectListExpressionExtractor(tapSchema);
-        rn = rndef;
-        fn = fndef;
-        sn = new SelectListExtractor(en, rn, fn);
+        sn = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema),
+                                     new ReferenceNavigator(),
+                                     new FromItemNavigator());
         navigatorList.add(sn);
 
         // support for file uploads to map the upload table name to the query table name.
@@ -157,7 +155,7 @@ protected void init()
                 tnc.put(tableDesc.getTableName(), newName);
                 log.debug("TableNameConverter " + tableDesc.getTableName() + " -> " + newName);
             }
-            sn = new SelectNavigator(endef, rndef, tnc);
+            sn = new SelectNavigator(new ExpressionNavigator(), new ReferenceNavigator(), tnc);
             navigatorList.add(sn);
         }
     }
@@ -247,5 +245,4 @@ public String getInfo()
         return queryString;
     }
 
-
 }
diff --git a/cadc-adql/src/main/java/ca/nrc/cadc/tap/parser/extractor/SelectListExpressionExtractor.java b/cadc-adql/src/main/java/ca/nrc/cadc/tap/parser/extractor/SelectListExpressionExtractor.java
@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2009.                            (c) 2009.
+*  (c) 2017.                            (c) 2017.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -69,26 +69,22 @@
 
 package ca.nrc.cadc.tap.parser.extractor;
 
-import java.util.ArrayList;
-import java.util.List;
-
-import net.sf.jsqlparser.expression.Expression;
-import net.sf.jsqlparser.schema.Column;
-import net.sf.jsqlparser.statement.select.AllColumns;
-import net.sf.jsqlparser.statement.select.AllTableColumns;
-import net.sf.jsqlparser.statement.select.PlainSelect;
-import net.sf.jsqlparser.statement.select.SelectExpressionItem;
 import ca.nrc.cadc.tap.parser.navigator.ExpressionNavigator;
-import ca.nrc.cadc.tap.schema.TapSchema;
+import ca.nrc.cadc.tap.parser.navigator.FromItemNavigator;
+import ca.nrc.cadc.tap.parser.navigator.ReferenceNavigator;
 import ca.nrc.cadc.tap.parser.schema.TapSchemaUtil;
-import ca.nrc.cadc.tap.schema.ColumnDesc;
-import ca.nrc.cadc.tap.schema.FunctionDesc;
-import ca.nrc.cadc.tap.schema.ParamDesc;
-import ca.nrc.cadc.tap.schema.TapSchemaDAO;
+import ca.nrc.cadc.tap.schema.*;
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.ExpressionVisitor;
 import net.sf.jsqlparser.expression.Function;
 import net.sf.jsqlparser.expression.operators.relational.ExpressionList;
+import net.sf.jsqlparser.schema.Column;
+import net.sf.jsqlparser.statement.select.*;
 import org.apache.log4j.Logger;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * Extract a list of TapSelectItem from query.
  * 
@@ -142,31 +138,53 @@ public void visit(SelectExpressionItem selectExpressionItem)
         PlainSelect plainSelect = selectNavigator.getPlainSelect();
         String alias = selectExpressionItem.getAlias();
 
-        Expression expression = selectExpressionItem.getExpression();
-        if (expression instanceof Column)
+        Expression selectExpression = selectExpressionItem.getExpression();
+        if (selectExpression instanceof Column)
         {
-            Column column = (Column) expression;
+            Column column = (Column) selectExpression;
             ColumnDesc columnDesc = TapSchemaUtil.findColumnDesc(tapSchema, plainSelect, column);
             log.debug("visit(column) " + column + "found: " + columnDesc);
             paramDesc = new ParamDesc(columnDesc, alias);
         }
-        else if (expression instanceof Function)
+        else if (selectExpression instanceof Function)
         {
-            Function function = (Function) expression;
+            Function function = (Function) selectExpression;
             FunctionDesc functionDesc = getFunctionDesc(function, plainSelect);
             log.debug("visit(function) " + function + " fiund: " + functionDesc);
             paramDesc = new ParamDesc(functionDesc, alias);
             paramDesc.columnDesc = functionDesc.arg;
         }
+        else if (selectExpression instanceof SubSelect)
+        {
+            SubSelect subSelect = (SubSelect) selectExpression;
+            log.debug("visit(subSelect) " + subSelect);
+
+            SelectListExtractor sle = new SelectListExtractor(new SelectListExpressionExtractor(tapSchema),
+                                                              new ReferenceNavigator(),
+                                                              new FromItemNavigator());
+            subSelect.getSelectBody().accept(sle);
+            SelectListExpressionExtractor slee = (SelectListExpressionExtractor) sle.getExpressionNavigator();
+            List <ParamDesc> selectList = slee.getSelectList();
+            if (selectList.size() != 1)
+            {
+                final String error = "Expected 1 ParamDesc in SelectList, found " + selectList.size();
+                throw new IllegalStateException(error);
+            }
+            paramDesc = selectList.get(0);
+        }
         else
         {
-            String datatype = getDatatypeFromExpression(expression);
+            String datatype = getDatatypeFromExpression(selectExpression);
             if (alias == null || alias.isEmpty())
-                paramDesc = new ParamDesc(expression.toString(), expression.toString(), datatype);
+                paramDesc = new ParamDesc(selectExpression.toString(), selectExpression.toString(), datatype);
             else
-                paramDesc = new ParamDesc(expression.toString(), alias, datatype);
+                paramDesc = new ParamDesc(selectExpression.toString(), alias, datatype);
+        }
+
+        if (paramDesc != null)
+        {
+            selectList.add(paramDesc);
         }
-        selectList.add(paramDesc);
     }
 
     public List<ParamDesc> getSelectList()

diff --git a/cadc-adql/src/main/java/ca/nrc/cadc/tap/parser/navigator/FromItemNavigator.java b/cadc-adql/src/main/java/ca/nrc/cadc/tap/parser/navigator/FromItemNavigator.java
@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2009.                            (c) 2009.
+*  (c) 2017.                            (c) 2017.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -119,8 +119,8 @@ public void visit(SubSelect subSelect)
         VisitingPart visiting = selectNavigator.getVisitingPart();
         if (visiting.equals(VisitingPart.FROM))
             throw new UnsupportedOperationException("sub-select not supported in FROM clause.");
-        else if (visiting.equals(VisitingPart.SELECT_ITEM))
-            throw new UnsupportedOperationException("sub-select not supported in SELECT ITEM.");
+//        else if (visiting.equals(VisitingPart.SELECT_ITEM))
+//            throw new UnsupportedOperationException("sub-select not supported in SELECT ITEM.");
         else
             subSelect.getSelectBody().accept(selectNavigator);
     }

diff --git a/cadc-adql/src/test/java/ca/nrc/cadc/tap/AdqlQueryTest.java b/cadc-adql/src/test/java/ca/nrc/cadc/tap/AdqlQueryTest.java
@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2009.                            (c) 2009.
+*  (c) 2018.                            (c) 2017.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -73,6 +73,7 @@
 package ca.nrc.cadc.tap;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 import java.util.List;
 
@@ -123,7 +124,7 @@ public class AdqlQueryTest
         public String getID() { return "abcdefg"; }
     };
 
-    private void doit()
+    private List<ParamDesc> doit()
     {
         try
         {
@@ -138,6 +139,7 @@ private void doit()
             log.debug("QUERY: \r\n" + _query);
             log.debug("SQL: \r\n" + sql);
             assertEquals(_expected.toLowerCase().trim(), sql.toLowerCase().trim());
+            return selectList;
         }
         finally
         {
@@ -169,6 +171,45 @@ public void testColumnAlias()
         doit();
     }
 
+    @Test
+    public void testSubSelectInSelect()
+    {
+        _query = "select schema_name as xx, (select t_integer from tap_schema.alldatatypes) from tap_schema.tables";
+        _expected = "select schema_name as xx, (select t_integer from tap_schema.alldatatypes) from tap_schema.tables";
+        List<ParamDesc> selectList = doit();
+        assertTrue(selectList.size() == 2);
+        ParamDesc paramDesc = selectList.get(1);
+        assertEquals("t_integer", paramDesc.name);
+        assertEquals("adql:INTEGER", paramDesc.datatype);
+        assertEquals("int column", paramDesc.description);
+
+        _query = "select schema_name as xx, (select t_varchar from tap_schema.alldatatypes) from tap_schema.tables";
+        _expected = "select schema_name as xx, (select t_varchar from tap_schema.alldatatypes) from tap_schema.tables";
+        selectList = doit();
+        assertTrue(selectList.size() == 2);
+        paramDesc = selectList.get(1);
+        assertEquals("t_varchar", paramDesc.name);
+        assertEquals("adql:VARCHAR", paramDesc.datatype);
+        assertEquals("varchar column", paramDesc.description);
+        assertEquals(8L, paramDesc.arraysize, 0.0);
+
+        _query = "select schema_name, (select count(distinct t_bytes) from tap_schema.alldatatypes) from tap_schema.tables";
+        _expected = "select schema_name, (select count(distinct t_bytes) from tap_schema.alldatatypes) from tap_schema.tables";
+        selectList = doit();
+        assertTrue(selectList.size() == 2);
+        paramDesc = selectList.get(1);
+        assertEquals("COUNT", paramDesc.name);
+        assertEquals("adql:INTEGER", paramDesc.datatype);
+
+        _query = "select schema_name, (select count(*) from tap_schema.alldatatypes) from tap_schema.tables";
+        _expected = "select schema_name, (select count(*) from tap_schema.alldatatypes) from tap_schema.tables";
+        selectList = doit();
+        assertTrue(selectList.size() == 2);
+        paramDesc = selectList.get(1);
+        assertEquals("COUNT", paramDesc.name);
+        assertEquals("adql:INTEGER", paramDesc.datatype);
+    }
+
     //@Test
     public void testJoin()
     {

diff --git a/cadc-adql/src/test/java/ca/nrc/cadc/tap/parser/AllColumnConverterTest.java b/cadc-adql/src/test/java/ca/nrc/cadc/tap/parser/AllColumnConverterTest.java
@@ -3,7 +3,7 @@
 *******************  CANADIAN ASTRONOMY DATA CENTRE  *******************
 **************  CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES  **************
 *
-*  (c) 2009.                            (c) 2009.
+*  (c) 2017.                            (c) 2017.
 *  Government of Canada                 Gouvernement du Canada
 *  National Research Council            Conseil national de recherches
 *  Ottawa, Canada, K1A 0R6              Ottawa, Canada, K1A 0R6
@@ -197,7 +197,7 @@ public void testJoin()
     {
         _query = "select * from tap_schema.keys as aa, tap_schema.tables as bb " +
         		" where aa.key_id = bb.utype";
-        _expected = "select aa.key_id, aa.from_table, aa.target_table, aa.utype, aa.description, bb.schema_name, bb.table_name, bb.utype, bb.description from tap_schema.keys as aa , tap_schema.tables as bb where aa.key_id = bb.utype";
+        _expected = "select aa.key_id, aa.from_table, aa.target_table, aa.utype, aa.description, bb.schema_name, bb.table_name, bb.utype, bb.description from tap_schema.keys as aa, tap_schema.tables as bb where aa.key_id = bb.utype";
         doit();
     }
 }
diff --git a/cadc-jsqlparser-compat/build.gradle b/cadc-jsqlparser-compat/build.gradle
@@ -14,7 +14,7 @@ sourceCompatibility = 1.7
 
 group = 'org.opencadc'
 
-version = '0.6.2a'
+version = '0.6.4'
 
 dependencies {
     compile 'log4j:log4j:1.2.+'