diff --git a/conf/backendURLs.json b/conf/backendURLs.json new file mode 100644 index 00000000..15643136 --- /dev/null +++ b/conf/backendURLs.json @@ -0,0 +1,23 @@ +[ + [ + "http://dbpedia.ailao.eu:3030/dbpedia/query", + "http://freebase.ailao.eu:3030/freebase/query", + "http://dbp-labels.ailao.eu:5000", + "http://dbp-labels.ailao.eu:5001", + "http://enwiki.ailao.eu:8983/solr/" + ], + [ + "http://127.0.0.1:3037/dbpedia/query", + "http://127.0.0.1:3030/freebase/query", + "http://127.0.0.1:5000", + "http://127.0.0.1:5001", + "http://127.0.0.1:8983/solr" + ], + [ + "http://sth.somecompany.com:3037/dbpedia/query", + "http://sth.somecompany.com:3030/freebase/query", + "http://sth.somecompany.com:5000", + "http://sth.somecompany.com:5001", + "http://sth.somecompany.com:8983/solr" + ] +] \ No newline at end of file diff --git a/data/dbpedia/Dockerfile b/data/dbpedia/Dockerfile new file mode 100644 index 00000000..c6825dac --- /dev/null +++ b/data/dbpedia/Dockerfile @@ -0,0 +1,38 @@ +###################################### +## DBpedia / Freebase Service Image ## +###################################### + +# Proposed image name: dbpedia +# Readme https://github.com/brmson/yodaqa/blob/master/data/dbpedia/ + +# Inherit official Java image, see https://hub.docker.com/_/java/ +FROM java:8 + +# Copy fuseki files +ADD ./jena-fuseki-1.1.1 /jena-fuseki-1.1.1 + +# Same as "export TERM=dumb"; prevents error "Could not open terminal for stdout: $TERM not set" +ENV TERM dumb + +# Define working directory +WORKDIR /jena-fuseki-1.1.1 + +# Expose ports (3030 for Freebase, 3037 for DBpedia) +EXPOSE 3037 +EXPOSE 3030 + +########## +# BEWARE ##################################################################################### +# With SELinux you need to run chcon -Rt svirt_sandbox_file_t /run/media///home//Downloads/Backends/DBpedia/jena-fuseki-1.1.1/db/ +############################################################################################## + +# Can be built with: "docker build -t fuseki ." + +# .:: DBpedia +# docker run -it -v /home/fp/docker/data/db/:/jena-fuseki-1.1.1/db/ --entrypoint="./fuseki-server" -p 3037:3037 fuseki --port 3037 --loc db /dbpedia +# RUN ./fuseki-server --port 3037 --loc db /dbpedia is done in run command; need to map as volume (read-only via :ro) + +# .:: Freebase +# docker run -it -v /home/fp/docker/data/d-freebase/:/jena-fuseki-1.1.1/d-freebase/ --entrypoint="./fuseki-server" -p 3030:3030 fuseki --loc d-freebase /freebase +# RUN ./fuseki-server --loc d-freebase /freebase + diff --git a/data/enwiki/Dockerfile b/data/enwiki/Dockerfile new file mode 100644 index 00000000..dcba7496 --- /dev/null +++ b/data/enwiki/Dockerfile @@ -0,0 +1,36 @@ +########################## +## enwiki Service Image ## +########################## + +# Proposed image name: enwiki +# Readme https://github.com/brmson/yodaqa/tree/master/data/enwiki + +# Inherit official Java image, see https://hub.docker.com/_/java/ +FROM java:8 + +# Copy fuseki files +ADD ./solr-4.6.0 /solr-4.6.0 + + +# JAVA_HOME is not set by default +ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64/ + +# Same as "export TERM=dumb"; prevents error "Could not open terminal for stdout: $TERM not set" +ENV TERM dumb + +# Define working directory +WORKDIR /solr-4.6.0/example + +# Expose port +EXPOSE 8983 + +########## +# BEWARE ##################################################################################### +# With SELinux you need to run chcon -Rt svirt_sandbox_file_t /run/media///home//Downloads/Backends/enwiki/solr-4.6.0/example/enwiki/collection1/ +############################################################################################## + +# Can be built with: "docker build -t enwiki ." + +# docker run -it -v /run/media///home//Downloads/Backends/enwiki/solr-4.6.0/example/enwiki/collection1/:/solr-4.6.0/example/enwiki/collection1/ --entrypoint="java" -p 8983:8983 enwiki -Dsolr.solr.home=enwiki -jar start.jar +#docker run -it -v /home/fp/docker/data/enwiki/collection1/:/solr-4.6.0/example/enwiki/collection1/ --entrypoint="java" -p 8983:8983 enwiki -Dsolr.solr.home=enwiki -jar start.jar +# RUN java -Dsolr.solr.home=enwiki -jar start.jar is done in run command; need to map as volume (read-only via :ro) diff --git a/data/freebase/Dockerfile_README.txt b/data/freebase/Dockerfile_README.txt new file mode 100644 index 00000000..7e090f6a --- /dev/null +++ b/data/freebase/Dockerfile_README.txt @@ -0,0 +1 @@ +See ../dbpedia directory. The Fuseki image can be used for both DBpedia and Freebase depending on which volume is mapped. diff --git a/src/main/java/cz/brmlab/yodaqa/io/web/WebInterface.java b/src/main/java/cz/brmlab/yodaqa/io/web/WebInterface.java index 1438f478..6da493e7 100644 --- a/src/main/java/cz/brmlab/yodaqa/io/web/WebInterface.java +++ b/src/main/java/cz/brmlab/yodaqa/io/web/WebInterface.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Random; +import cz.brmlab.yodaqa.provider.UrlManager; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -135,6 +136,16 @@ public Object handle(Request request, Response response) { return "[" + StringUtils.join(qJson, ",\n") + "]"; } }); + + get(new Route("/dataUrls") { + @Override + public Object handle(Request request, Response response) { + response.type("application/json"); + response.header("Access-Control-Allow-Origin", "*"); + return UrlManager.printState(); + + } + }); } private List retrieveArtificialConcepts(Request request){ diff --git a/src/main/java/cz/brmlab/yodaqa/pipeline/YodaQA.java b/src/main/java/cz/brmlab/yodaqa/pipeline/YodaQA.java index 3e0a4027..14958a9a 100644 --- a/src/main/java/cz/brmlab/yodaqa/pipeline/YodaQA.java +++ b/src/main/java/cz/brmlab/yodaqa/pipeline/YodaQA.java @@ -1,5 +1,6 @@ package cz.brmlab.yodaqa.pipeline; +import cz.brmlab.yodaqa.provider.UrlManager; import org.apache.uima.analysis_engine.AnalysisEngineDescription; import org.apache.uima.cas.CAS; import org.apache.uima.fit.factory.AggregateBuilder; @@ -54,7 +55,8 @@ public class YodaQA /* XXX: extends AggregateBuilder ? */ { * instructions on how to obtain an example one. */ //SolrNamedSource.register("guten", "data/guten", null); - SolrNamedSource.register("enwiki", "collection1", "http://enwiki.ailao.eu:8983/solr/"); + //SolrNamedSource.register("enwiki", "collection1", "http://127.0.0.1:8983/solr/"); + SolrNamedSource.register("enwiki", "collection1", UrlManager.lookUpUrl(UrlManager.DataBackends.SOLR.ordinal())); } catch (Exception e) { e.printStackTrace(); System.err.println("*** Exception caught during SolrNamedSource initialization. ***"); diff --git a/src/main/java/cz/brmlab/yodaqa/provider/UrlManager.java b/src/main/java/cz/brmlab/yodaqa/provider/UrlManager.java new file mode 100644 index 00000000..7b9eadd1 --- /dev/null +++ b/src/main/java/cz/brmlab/yodaqa/provider/UrlManager.java @@ -0,0 +1,198 @@ +package cz.brmlab.yodaqa.provider; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; + +/** + * Created by fp on 5/9/16. + */ +public class UrlManager { + + static { + updateUrlTable(); + } + + public enum DataBackends { + DBPEDIA, FREEBASE, LABEL1, LABEL2, SOLR + } + + private static String[][] urlLookUpTable; + + public static String[][] getUrlLookUpTable() { + return urlLookUpTable; + } + public static void setUrlLookUpTable(String[][] newLookUpTable) { + urlLookUpTable = newLookUpTable; + } + + private static int currentBackend = 0; //Default is 0: ailao.eu, 1: loopback, different if custom tables loaded + public static int getCurrentBackend() { + return currentBackend; + } + public static boolean setCurrentBackend(int currentBackend) { + if (currentBackend < urlLookUpTable.length) { + UrlManager.currentBackend = currentBackend; + return true; + } else { + return false; + } + } + + public static String printState() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("Currently selected backend: "); + stringBuilder.append(Integer.toString(currentBackend)); + stringBuilder.append("\n\nCurrent Lookup Table:\n"); + for (int i=0; i backends = gson.fromJson(jsonObject.get("offline"), new TypeToken>(){}.getType()); + //JsonParser jsonParser = new JsonParser(); + //JsonElement jsonElement = jsonParser.parse(jsonString); + // JsonArray jsonArray = jsonElement.isJsonArray()?jsonElement.getAsJsonArray():null; + + // ...but we can load it directly + return gson.fromJson(jsonString, String[][].class); + } + + /** + * Reads and parses configuration file + * @return Success state + */ + public static boolean updateUrlTable() { + try{ + String[][] result = parseConfiguration(readConfigurationFile()); + if (result != null) { + urlLookUpTable = result; + return true; + } + return false; + } catch (JsonParseException jpe) { + jpe.printStackTrace(); + return false; + } + } + + /** + * Looks up entire URL set for a backend (DBpedia, Freebase, Label Service 1, Label Service 2 + * and then Solr/enwiki) + * @param id Backend to return all URLs for + * @return All backends in order + */ + public static String[] lookUpUrlSet(int id) { + if(urlLookUpTable != null && id\n" + "PREFIX dbpedia2: \n" + "PREFIX dbpedia: \n" + diff --git a/src/main/java/cz/brmlab/yodaqa/provider/rdf/DBpediaTitles.java b/src/main/java/cz/brmlab/yodaqa/provider/rdf/DBpediaTitles.java index ad3313e7..87625799 100644 --- a/src/main/java/cz/brmlab/yodaqa/provider/rdf/DBpediaTitles.java +++ b/src/main/java/cz/brmlab/yodaqa/provider/rdf/DBpediaTitles.java @@ -18,6 +18,7 @@ import com.google.gson.Gson; import com.hp.hpl.jena.rdf.model.Literal; +import cz.brmlab.yodaqa.provider.UrlManager; import org.slf4j.Logger; /** A wrapper around DBpedia "Titles" dataset that maps titles to @@ -30,8 +31,8 @@ * and disambiguation pages. */ public class DBpediaTitles extends DBpediaLookup { - protected static final String fuzzyLookupUrl = "http://dbp-labels.ailao.eu:5000"; - protected static final String crossWikiLookupUrl = "http://dbp-labels.ailao.eu:5001"; + protected static final String fuzzyLookupUrl = UrlManager.lookUpUrl(UrlManager.DataBackends.LABEL1.ordinal()); + protected static final String crossWikiLookupUrl = UrlManager.lookUpUrl(UrlManager.DataBackends.LABEL2.ordinal()); /** A container of enwiki article metadata. * This must 1:1 map to label-lookup API. */ diff --git a/src/main/java/cz/brmlab/yodaqa/provider/rdf/FreebaseLookup.java b/src/main/java/cz/brmlab/yodaqa/provider/rdf/FreebaseLookup.java index e0a6c958..b0d48946 100644 --- a/src/main/java/cz/brmlab/yodaqa/provider/rdf/FreebaseLookup.java +++ b/src/main/java/cz/brmlab/yodaqa/provider/rdf/FreebaseLookup.java @@ -1,10 +1,12 @@ package cz.brmlab.yodaqa.provider.rdf; +import cz.brmlab.yodaqa.provider.UrlManager; + /** This is an abstract base class for accessing Freebase. */ public abstract class FreebaseLookup extends CachedJenaLookup { public FreebaseLookup() { - super("http://freebase.ailao.eu:3030/freebase/query", + super(UrlManager.lookUpUrl(UrlManager.DataBackends.FREEBASE.ordinal()), "PREFIX ns: \n" + ""); }