Merge pull request #52 from FUB-HCC/34-simulate-wtas

34 simulate wtas

.gitignore

@@ -70,6 +70,7 @@ assets/config/secrets.json
 #data
 assets/data/data_augmentation/institutions.csv
 assets/data/database_dump/dump.sqlite
+assets/data/project_input/gepris.*
 
 .idea/
 database/

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "assets/IKON-backend-config"]
 	path = assets/IKON-backend-config
 	url = https://github.com/FUB-HCC/IKON-backend-config
+[submodule "src/topicextraction/nlp/bert"]
+	path = src/topicextraction/nlp/bert
+	url = https://github.com/google-research/bert.git

docker-compose.yml

@@ -26,6 +26,7 @@ services:
       - "5433:8080"
     depends_on:
       - postgres
+      - mwc
     environment:
       PGUSER: ikonuser
       PGHOST: Postgres
@@ -42,21 +43,30 @@ services:
       context: ./src/mwc/
       dockerfile: Dockerfile
     ports:
-      - "5434:8080"
+      - "5434:8081"
     environment:
-      PGUSER: ikonuser
-      PGHOST: Postgres
-      PGDATABASE: ikon
-      PGPORT: 5432
       IKONCODE: /run/secrets/ikoncode_secrets
       SSL_KEY: /run/secrets/ssl_key
-      SSL_CERT: /run/secrets/ssl_cert
+      SSL_CRT: /run/secrets/ssl_crt
     secrets:
-      - postgres_password
       - ssl_key
       - ssl_crt
       - ikoncode_secrets
 
+  topicextraction:
+    container_name: TopicExtractionService
+    build:
+      context: ./src/topicextraction/
+      dockerfile: Dockerfile
+    ports:
+      - "5435:80"
+    environment:
+      SSL_KEY: /run/secrets/ssl_key
+      SSL_CRT: /run/secrets/ssl_crt
+    secrets:
+      - ssl_key
+      - ssl_crt
+
 secrets:
   postgres_password:
     file: ./assets/IKON-backend-config/secrets/postgres_password

src/dal/src/dal.js

@@ -1,5 +1,4 @@
 
-
 const fs = require('fs');
 // const hash = require('object-hash')
 const express = require('express');
@@ -62,10 +61,11 @@ router.get('/projects', async (req, res) => {
   let rows = '';
   try {
     rows = (await pool.query(queries.getAllProjects, [institution, offset, limit])).rows;
+    res.status(200).json(rows);
   } catch (err) {
     console.log(err);
+    res.status(500).send(err);
   }
-  res.status(200).json(rows);
 });
 
 router.get('/institutions', async (req, res) => {
@@ -75,23 +75,25 @@ router.get('/institutions', async (req, res) => {
   let rows = '';
   try {
     rows = (await pool.query(queries.getConnectedInstitutions, [institution])).rows;
+    res.status(200).json(rows);
   } catch (err) {
     console.log(err);
+    res.status(500).send(err);
   }
-  res.status(200).json(rows);
 });
 
 router.patch('/institutions', async (req, res) => {
   try {
-    initGeolocations(pool, queries)
+    initGeolocations(pool, queries);
+    res.status(200).send();
   } catch (err) {
-    res.status(500).send()
+    res.status(500).send(err);
   }
-  res.status(200).send();
 });
 
 // exit strategy
 process.on('SIGINT', async (err) => {
   console.log(err);
   await pool.end();
+  process.exit(0);
 });

src/dal/src/geocode.js

@@ -23,20 +23,20 @@ const geocodeLocation = async (loc) => {
   return result.data || { lat: null, lon: null };
 };
 
-exports.initGeolocations = async (pool, {insertGeolocation, getAllInstitutions }) => {
+exports.initGeolocations = async (pool, { insertGeolocation, getAllInstitutions }) => {
   // check if all institutions are geolocated
   try {
     const rows = (await pool.query(getAllInstitutions)).rows;
-    const missingGeocodes = rows.map((row, index) =>{return [index, row]})
-      .filter(([index, {lat, lon}]) => !(lat && lon))
-      .map(([index, {address}]) => {return [index, address]});
+    const missingGeocodes = rows.map((row, index) => [index, row])
+      .filter(([index, { lat, lon }]) => !(lat && lon))
+      .map(([index, { address }]) => [index, address]);
 
     for (const [index, address] of missingGeocodes) {
       const gecode = await geocodeLocation(address); // eslint-disable-line no-await-in-loop
       pool.query(insertGeolocation, [rows[index].id, gecode[0].lat, gecode[0].lon]);
     }
   } catch (e) {
-  console.log(e);
-  return e;
+    console.log(e);
+    return e;
   }
-}
+};

src/mwc/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:9
+FROM node:latest
 
 # Create app directory
 WORKDIR /usr/src/app

src/mwc/package.json

@@ -5,7 +5,7 @@
   "main": "server.js",
   "scripts": {
     "start": "node src/mwc.js",
-    "local": "IKONCODE='../../assets/IKON-backend-config/secrets/ikoncode_secrets' NODE_ENV='dev' SSL_KEY='../../assets/ssl/server.key' SSL_CERT='../../assets/ssl/server.crt' node src/mwc.js"
+    "local": "IKONCODE='../../assets/IKON-backend-config/secrets/ikoncode_secrets' NODE_ENV='dev' SSL_KEY='../../assets/ssl/server.key' SSL_CRT='../../assets/ssl/server.crt' node src/mwc.js"
   },
   "author": "",
   "license": "ISC",

src/mwc/src/MediaWikiConnector/MediaWikiConnector.js

@@ -0,0 +1,81 @@
+const MWC = require('nodemw');
+const Promise = require('bluebird');
+
+const ikoncode = Promise.promisifyAll(new MWC(process.env.IKONCODE));
+ikoncode.api = Promise.promisifyAll(ikoncode.api, { multiArgs: true });
+
+// connect to IKON CODE
+exports.wikiLogin = () => {
+  try {
+    return (ikoncode.logInAsync());
+  } catch (e) {
+    console.log(e);
+    process.exit(1);
+  }
+};
+
+const fetchAllProjects = async (login) => {
+  try {
+    await login;
+  } catch (e) {
+    console.log(e);
+    process.exit(1);
+  }
+
+  let projects = [];
+
+  try {
+    const params1 = {
+      action: 'ask',
+      query: '[[Category:Drittmittelprojekt]][[RedaktionelleBeschreibung::+]][[Status::Freigegeben]]|?Identifier|limit=100000',
+    };
+    projects = await ikoncode.api.callAsync(params1);
+    console.log(Object.keys(projects[2].query.results));
+  } catch (e) {
+    console.log(e);
+  }
+
+  const results = [];
+  for (const key of Object.keys(projects[2].query.results)) {
+    try {
+      const params2 = {
+        action: 'browsebysubject',
+        subject: key,
+      };
+      results.push(ikoncode.api.callAsync(params2));
+    } catch (e) {
+      console.log(params2, e);
+    }
+  }
+  // for some reason this is necessary
+  await Promise.all(results);
+  return Promise.all(results);
+};
+
+const getNameMapping = (name) => {
+  const mapping = {
+    Projektbeginn: 'funding_start_year',
+    Projektende: 'funding_end_year',
+    Zusammenfassung: 'project_summary',
+    subject: 'id',
+    RedaktionelleBeschreibung: 'description',
+    HatFach: 'participating_subject_area',
+    TitelProjekt: 'title',
+    HatOrganisationseinheit: 'organisational_unit',
+    Akronym: 'acronym',
+
+  };
+
+  return (Object.keys(mapping).includes(name)) ? mapping[name] : name;
+};
+
+exports.getAllProjects = async (loginPromise) => {
+  return (await fetchAllProjects(loginPromise)).map(([a, b, { query: { subject, data } }]) => data.reduce((dict, { property, dataitem }) => {
+      dict[getNameMapping(property)] = dataitem.map(({ item }) => item);
+      return dict;
+    }, { subject }));
+} 
+
+exports.parseMediaWikiResponse = (response) => {
+
+}