Merge pull request #48 from k0105/master

Backend modification enabled

conf/backendURLs.json

@@ -0,0 +1,23 @@
+[
+  [
+    "http://dbpedia.ailao.eu:3030/dbpedia/query",
+    "http://freebase.ailao.eu:3030/freebase/query",
+    "http://dbp-labels.ailao.eu:5000",
+    "http://dbp-labels.ailao.eu:5001",
+    "http://enwiki.ailao.eu:8983/solr/"
+  ],
+  [
+    "http://127.0.0.1:3037/dbpedia/query",
+    "http://127.0.0.1:3030/freebase/query",
+    "http://127.0.0.1:5000",
+    "http://127.0.0.1:5001",
+    "http://127.0.0.1:8983/solr"
+  ],
+  [
+    "http://sth.somecompany.com:3037/dbpedia/query",
+    "http://sth.somecompany.com:3030/freebase/query",
+    "http://sth.somecompany.com:5000",
+    "http://sth.somecompany.com:5001",
+    "http://sth.somecompany.com:8983/solr"
+  ]
+]

data/dbpedia/Dockerfile

@@ -0,0 +1,38 @@
+######################################
+## DBpedia / Freebase Service Image ##
+######################################
+
+# Proposed image name: dbpedia
+# Readme https://github.com/brmson/yodaqa/blob/master/data/dbpedia/
+
+# Inherit official Java image, see https://hub.docker.com/_/java/
+FROM java:8
+
+# Copy fuseki files
+ADD ./jena-fuseki-1.1.1 /jena-fuseki-1.1.1
+
+# Same as "export TERM=dumb"; prevents error "Could not open terminal for stdout: $TERM not set"
+ENV TERM dumb
+
+# Define working directory
+WORKDIR /jena-fuseki-1.1.1
+
+# Expose ports (3030 for Freebase, 3037 for DBpedia)
+EXPOSE 3037
+EXPOSE 3030
+
+##########
+# BEWARE #####################################################################################
+# With SELinux you need to run chcon -Rt svirt_sandbox_file_t /run/media/<user>/<longid>/home/<user>/Downloads/Backends/DBpedia/jena-fuseki-1.1.1/db/
+##############################################################################################
+
+# Can be built with: "docker build -t fuseki ."
+
+# .:: DBpedia
+# docker run -it -v /home/fp/docker/data/db/:/jena-fuseki-1.1.1/db/ --entrypoint="./fuseki-server" -p 3037:3037 fuseki --port 3037 --loc db /dbpedia
+# RUN ./fuseki-server --port 3037 --loc db /dbpedia is done in run command; need to map  as volume (read-only via :ro)
+
+# .:: Freebase
+# docker run -it -v /home/fp/docker/data/d-freebase/:/jena-fuseki-1.1.1/d-freebase/ --entrypoint="./fuseki-server" -p 3030:3030 fuseki --loc d-freebase /freebase
+# RUN ./fuseki-server --loc d-freebase /freebase
+

data/enwiki/Dockerfile

@@ -0,0 +1,36 @@
+##########################
+## enwiki Service Image ##
+##########################
+
+# Proposed image name: enwiki
+# Readme https://github.com/brmson/yodaqa/tree/master/data/enwiki
+
+# Inherit official Java image, see https://hub.docker.com/_/java/
+FROM java:8
+
+# Copy fuseki files
+ADD ./solr-4.6.0 /solr-4.6.0
+
+
+# JAVA_HOME is not set by default
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64/
+
+# Same as "export TERM=dumb"; prevents error "Could not open terminal for stdout: $TERM not set"
+ENV TERM dumb
+
+# Define working directory
+WORKDIR /solr-4.6.0/example
+
+# Expose port
+EXPOSE 8983
+
+##########
+# BEWARE #####################################################################################
+# With SELinux you need to run chcon -Rt svirt_sandbox_file_t /run/media/<user>/<longid>/home/<user>/Downloads/Backends/enwiki/solr-4.6.0/example/enwiki/collection1/
+##############################################################################################
+
+# Can be built with: "docker build -t enwiki ."
+
+# docker run -it -v /run/media/<user>/<longid>/home/<user>/Downloads/Backends/enwiki/solr-4.6.0/example/enwiki/collection1/:/solr-4.6.0/example/enwiki/collection1/ --entrypoint="java" -p 8983:8983 enwiki -Dsolr.solr.home=enwiki -jar start.jar
+#docker run -it -v /home/fp/docker/data/enwiki/collection1/:/solr-4.6.0/example/enwiki/collection1/ --entrypoint="java" -p 8983:8983 enwiki -Dsolr.solr.home=enwiki -jar start.jar
+# RUN java -Dsolr.solr.home=enwiki -jar start.jar is done in run command; need to map  as volume (read-only via :ro) 

data/freebase/Dockerfile_README.txt

@@ -0,0 +1 @@
+See ../dbpedia directory. The Fuseki image can be used for both DBpedia and Freebase depending on which volume is mapped.

src/main/java/cz/brmlab/yodaqa/io/web/WebInterface.java

@@ -14,6 +14,7 @@
 import java.util.Map;
 import java.util.Random;
 
+import cz.brmlab.yodaqa.provider.UrlManager;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -135,6 +136,16 @@ public Object handle(Request request, Response response) {
 				return "[" + StringUtils.join(qJson, ",\n") + "]";
 			}
 		});
+
+		get(new Route("/dataUrls") {
+			@Override
+			public Object handle(Request request, Response response) {
+				response.type("application/json");
+				response.header("Access-Control-Allow-Origin", "*");
+				return UrlManager.printState();
+
+			}
+		});
 	}
 
 	private List<QuestionConcept> retrieveArtificialConcepts(Request request){

src/main/java/cz/brmlab/yodaqa/pipeline/YodaQA.java

@@ -1,5 +1,6 @@
 package cz.brmlab.yodaqa.pipeline;
 
+import cz.brmlab.yodaqa.provider.UrlManager;
 import org.apache.uima.analysis_engine.AnalysisEngineDescription;
 import org.apache.uima.cas.CAS;
 import org.apache.uima.fit.factory.AggregateBuilder;
@@ -54,7 +55,8 @@ public class YodaQA /* XXX: extends AggregateBuilder ? */ {
 			 * instructions on how to obtain an example one. */
 
 			//SolrNamedSource.register("guten", "data/guten", null);
-			SolrNamedSource.register("enwiki", "collection1", "http://enwiki.ailao.eu:8983/solr/");
+			//SolrNamedSource.register("enwiki", "collection1", "http://127.0.0.1:8983/solr/");
+			SolrNamedSource.register("enwiki", "collection1", UrlManager.lookUpUrl(UrlManager.DataBackends.SOLR.ordinal()));
 		} catch (Exception e) {
 			e.printStackTrace();
 			System.err.println("*** Exception caught during SolrNamedSource initialization. ***");

src/main/java/cz/brmlab/yodaqa/provider/UrlManager.java

@@ -0,0 +1,198 @@
+package cz.brmlab.yodaqa.provider;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+
+/**
+ * Created by fp on 5/9/16.
+ */
+public class UrlManager {
+
+    static {
+        updateUrlTable();
+    }
+
+    public enum DataBackends {
+        DBPEDIA, FREEBASE, LABEL1, LABEL2, SOLR
+    }
+
+    private static String[][] urlLookUpTable;
+
+    public static String[][] getUrlLookUpTable() {
+        return urlLookUpTable;
+    }
+    public static void setUrlLookUpTable(String[][] newLookUpTable) {
+        urlLookUpTable = newLookUpTable;
+    }
+
+    private static int currentBackend = 0; //Default is 0: ailao.eu, 1: loopback, different if custom tables loaded
+    public static int getCurrentBackend() {
+        return currentBackend;
+    }
+    public static boolean setCurrentBackend(int currentBackend) {
+        if (currentBackend < urlLookUpTable.length) {
+            UrlManager.currentBackend = currentBackend;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public static String printState() {
+        StringBuilder stringBuilder = new StringBuilder();
+        stringBuilder.append("Currently selected backend: ");
+        stringBuilder.append(Integer.toString(currentBackend));
+        stringBuilder.append("\n\nCurrent Lookup Table:\n");
+        for (int i=0; i<urlLookUpTable.length; i++){
+            for(int j=0; j<urlLookUpTable[i].length; j++) {
+                stringBuilder.append(urlLookUpTable[i][j]);
+                stringBuilder.append(" ");
+            }
+            stringBuilder.append("\n");
+        }
+        stringBuilder.append("\nCurrent properties:");
+        stringBuilder.append("System.getProperty(\"cz.brmlab.yodaqa.dbpediaurl\") ");
+        stringBuilder.append(System.getProperty("cz.brmlab.yodaqa.dbpediaurl"));
+        stringBuilder.append("\nSystem.getProperty(\"cz.brmlab.yodaqa.freebaseurl\") ");
+        stringBuilder.append(System.getProperty("cz.brmlab.yodaqa.freebaseurl"));
+        stringBuilder.append("\nSystem.getProperty(\"cz.brmlab.yodaqa.label1url\") ");
+        stringBuilder.append(System.getProperty("cz.brmlab.yodaqa.label1url"));
+        stringBuilder.append("\nSystem.getProperty(\"cz.brmlab.yodaqa.label2url\") ");
+        stringBuilder.append(System.getProperty("cz.brmlab.yodaqa.label2url"));
+        stringBuilder.append("\nSystem.getProperty(\"cz.brmlab.yodaqa.solrurl\") ");
+        stringBuilder.append(System.getProperty("cz.brmlab.yodaqa.solrurl"));
+        stringBuilder.append("\n\nURL each backend receives:");
+        stringBuilder.append("\nDBpedia: ");
+        stringBuilder.append(lookUpUrl(UrlManager.DataBackends.DBPEDIA.ordinal()));
+        stringBuilder.append("\nFreebase: ");
+        stringBuilder.append(lookUpUrl(UrlManager.DataBackends.FREEBASE.ordinal()));
+        stringBuilder.append("\nLabel Service 1: ");
+        stringBuilder.append(lookUpUrl(UrlManager.DataBackends.LABEL1.ordinal()));
+        stringBuilder.append("\nLabel Service 2: ");
+        stringBuilder.append(lookUpUrl(UrlManager.DataBackends.LABEL2.ordinal()));
+        stringBuilder.append("\nSolr: ");
+        stringBuilder.append(lookUpUrl(UrlManager.DataBackends.SOLR.ordinal()));
+        return stringBuilder.toString();
+    }
+
+    /**
+     * Reads configuration file
+     * @return Raw content of configuration file (should be JSON)
+     */
+    private static String readConfigurationFile() {
+        StringBuilder configurationJson = new StringBuilder();
+
+        try (BufferedReader bufferedReader = new BufferedReader(new FileReader("conf/backendURLs.json")))
+        {
+            String currentLine;
+
+            while ((currentLine = bufferedReader.readLine()) != null) {
+                configurationJson.append(currentLine);
+            }
+
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+        }
+
+        return configurationJson.toString();
+    }
+
+    /**
+     * Parses configuration to two-dimensional String array, where each subarray contains
+     * DBpedia, Freebase, Label Service 1, Label Service 2, Solr/enwiki
+     * @return Two-dimensional array with URLs
+     */
+    private static String[][] parseConfiguration(String jsonString) {
+
+        Gson gson = new GsonBuilder().create();
+
+        // Could get it as list or proceed manually...
+        // List<String> backends = gson.fromJson(jsonObject.get("offline"), new TypeToken<List<String>>(){}.getType());
+        //JsonParser jsonParser  = new JsonParser();
+        //JsonElement jsonElement = jsonParser.parse(jsonString);
+        // JsonArray  jsonArray = jsonElement.isJsonArray()?jsonElement.getAsJsonArray():null;
+
+        // ...but we can load it directly
+        return gson.fromJson(jsonString, String[][].class);
+    }
+
+    /**
+     * Reads and parses configuration file
+     * @return Success state
+     */
+    public static boolean updateUrlTable() {
+        try{
+            String[][] result = parseConfiguration(readConfigurationFile());
+            if (result != null) {
+                urlLookUpTable = result;
+                return true;
+            }
+            return false;
+        } catch (JsonParseException jpe) {
+            jpe.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * Looks up entire URL set for a backend (DBpedia, Freebase, Label Service 1, Label Service 2
+     * and then Solr/enwiki)
+     * @param id Backend to return all URLs for
+     * @return All backends in order
+     */
+    public static String[] lookUpUrlSet(int id) {
+        if(urlLookUpTable != null && id<urlLookUpTable.length) {
+            return urlLookUpTable[id];
+        }
+        return null;
+    }
+
+    /**
+     * Looks up particular URL in current backend, used by cz.brmlab.yodaqa.provider.rdf.FreebaseLookup,
+     * cz.brmlab.yodaqa.provider.rdf.DBpediaTitles.java cz.brmlab.yodaqa.provider.rdf.DBpediaLookup and
+     * cz.brmlab.yodaqa.pipeline.YodaQA
+     * @param fieldId Field to lookup
+     * @return URL for that particular backend
+     */
+    public static String lookUpUrl(int fieldId) {
+        switch (fieldId) {
+            case 0:
+                return System.getProperty("cz.brmlab.yodaqa.dbpediaurl")!=null?
+                        System.getProperty("cz.brmlab.yodaqa.dbpediaurl"):lookUpUrl(currentBackend, fieldId);
+            case 1:
+                return System.getProperty("cz.brmlab.yodaqa.freebaseurl")!=null?
+                        System.getProperty("cz.brmlab.yodaqa.freebaseurl"):lookUpUrl(currentBackend, fieldId);
+            case 2:
+                return System.getProperty("cz.brmlab.yodaqa.label1url")!=null?
+                        System.getProperty("cz.brmlab.yodaqa.label1url"):lookUpUrl(currentBackend, fieldId);
+            case 3:
+                return System.getProperty("cz.brmlab.yodaqa.label2url")!=null?
+                        System.getProperty("cz.brmlab.yodaqa.label2url"):lookUpUrl(currentBackend, fieldId);
+            case 4:
+                return System.getProperty("cz.brmlab.yodaqa.solrurl")!=null?
+                        System.getProperty("cz.brmlab.yodaqa.solrurl"):lookUpUrl(currentBackend, fieldId);
+            default:
+                return null;
+        }
+    }
+
+    /**
+     * Looks up URL by set and field
+     * @param fieldId Field to lookup
+     * @param setId URL set to use, number is position is JSON configuration under conf/backendURLs.json
+     * @return URL for that particular set and field
+     */
+    public static String lookUpUrl(int setId, int fieldId) {
+        if(urlLookUpTable != null && setId<urlLookUpTable.length &&
+                fieldId<urlLookUpTable[setId].length) {
+            return urlLookUpTable[setId][fieldId];
+        }
+        return null;
+    }
+
+}

src/main/java/cz/brmlab/yodaqa/provider/rdf/DBpediaLookup.java

@@ -1,12 +1,14 @@
 package cz.brmlab.yodaqa.provider.rdf;
 
+import cz.brmlab.yodaqa.provider.UrlManager;
+
 /** This is an abstract base class for accessing DBpedia. */
 
 public abstract class DBpediaLookup extends CachedJenaLookup {
 	public DBpediaLookup() {
 		/* Replace the first URL below with http://dbpedia.org/sparql
 		 * to use the public DBpedia SPARQL endpoint. */
-		super("http://dbpedia.ailao.eu:3030/dbpedia/query",
+		super(UrlManager.lookUpUrl(UrlManager.DataBackends.DBPEDIA.ordinal()),
 			"PREFIX : <http://dbpedia.org/resource/>\n" +
 			"PREFIX dbpedia2: <http://dbpedia.org/property/>\n" +
 			"PREFIX dbpedia: <http://dbpedia.org/>\n" +

src/main/java/cz/brmlab/yodaqa/provider/rdf/DBpediaTitles.java

@@ -18,6 +18,7 @@
 import com.google.gson.Gson;
 import com.hp.hpl.jena.rdf.model.Literal;
 
+import cz.brmlab.yodaqa.provider.UrlManager;
 import org.slf4j.Logger;
 
 /** A wrapper around DBpedia "Titles" dataset that maps titles to
@@ -30,8 +31,8 @@
  * and disambiguation pages. */
 
 public class DBpediaTitles extends DBpediaLookup {
-	protected static final String fuzzyLookupUrl = "http://dbp-labels.ailao.eu:5000";
-	protected static final String crossWikiLookupUrl = "http://dbp-labels.ailao.eu:5001";
+	protected static final String fuzzyLookupUrl = UrlManager.lookUpUrl(UrlManager.DataBackends.LABEL1.ordinal());
+	protected static final String crossWikiLookupUrl = UrlManager.lookUpUrl(UrlManager.DataBackends.LABEL2.ordinal());
 
 	/** A container of enwiki article metadata.
 	 * This must 1:1 map to label-lookup API. */

src/main/java/cz/brmlab/yodaqa/provider/rdf/FreebaseLookup.java

@@ -1,10 +1,12 @@
 package cz.brmlab.yodaqa.provider.rdf;
 
+import cz.brmlab.yodaqa.provider.UrlManager;
+
 /** This is an abstract base class for accessing Freebase. */
 
 public abstract class FreebaseLookup extends CachedJenaLookup {
 	public FreebaseLookup() {
-		super("http://freebase.ailao.eu:3030/freebase/query",
+		super(UrlManager.lookUpUrl(UrlManager.DataBackends.FREEBASE.ordinal()),
 			"PREFIX ns: <http://rdf.freebase.com/ns/>\n" +
 			"");
 	}