From 73efd4c9a16dcd4520956f5b8f40841f3b657e5b Mon Sep 17 00:00:00 2001 From: Liliana Date: Mon, 8 Apr 2024 19:01:51 +0200 Subject: [PATCH 01/21] new queries --- .../wikidataExtractor/wikidataQueries.js | 187 +++++++++++++----- 1 file changed, 142 insertions(+), 45 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index 9d54d8fc..34772461 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -2,6 +2,26 @@ const wikidata = require("./wikidataConnexion"); class WikiQueries { + /* CIENCIA */ + + static async obtenerSimboloQuimico() { + const query = + `SELECT ?elementLabel ?symbol WHERE { + ?element wdt:P31 wd:Q11344. + ?element wdt:P246 ?symbol. + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } + } + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + + + /* GEOGRAFÍA */ + static async obtenerPaisYCapital() { const query = ` SELECT ?countryLabel ?capitalLabel WHERE { @@ -16,93 +36,123 @@ class WikiQueries { return results; } - static async obtenerPeliculasAñosYDirector() { + static async obtenerPaisYBandera() { const query = ` - SELECT ?peliculaLabel ?directorLabel ?fecha - WHERE { - ?pelicula wdt:P31 wd:Q11424. # Filtramos por instancias de películas - ?pelicula wdt:P577 ?fecha. # Obtenemos la fecha de publicación - ?pelicula wdt:P57 ?director. # Obtenemos el director de la película - FILTER (YEAR(?fecha) > 2000). # Filtramos por películas posteriores al 2000 - SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + SELECT ?flag ?flagLabel ?countryLabel WHERE { + ?country wdt:P31 wd:Q6256; + wdt:P41 ?flag. + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } } - LIMIT 2000 - `; + LIMIT 200 + `; const results = await wikidata.consulta(query); // console.log(results) return results; + } - static async obtenerMangaYFecha() { + static async obtenerUnPaisEuropeo(){ const query = ` - SELECT ?titulo ?fechaEstreno + SELECT ?countryLabel ?continentLabel + WHERE { + { + SELECT ?countryLabel ?continentLabel WHERE { - ?obra wdt:P31/wdt:P279* wd:Q1107. # Filtramos por instancias de "anime" o "manga" - ?obra rdfs:label ?titulo. # Obtenemos el título de la obra - ?obra wdt:P580 ?fechaEstreno. # Obtenemos la fecha de estreno - FILTER (LANG(?titulo) = "es" || LANG(?titulo) = "en") # Filtramos por títulos en español o inglés + ?country wdt:P31 wd:Q6256; # Selecciona instancias de país + wdt:P30 ?continent. # Obtiene el continente del país + FILTER(?continent = wd:Q46) # Filtra para que el continente sea Europa + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } } - LIMIT 2000 + ORDER BY RAND() + LIMIT 1 + } + UNION + { + SELECT ?countryLabel ?continentLabel + WHERE { + ?country wdt:P31 wd:Q6256; # Selecciona instancias de país + wdt:P30 ?continent. # Obtiene el continente del país + FILTER(?continent != wd:Q46) # Filtra para excluir Europa + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + } + ORDER BY RAND() + LIMIT 3 + }} `; const results = await wikidata.consulta(query); // console.log(results) return results; - } + } - static async obtenerMonumentoYAñoDescubOAñoConst() { + static async obtenerUnPaisEuropeo2(){ const query = ` - SELECT ?monumento ?titulo ?anioConstruccion ?anioDescubrimiento - WHERE { - ?monumento wdt:P31 wd:Q4989906. # Filtramos por instancias de "monumento" - ?monumento rdfs:label ?titulo. # Obtenemos el título del monumento - OPTIONAL { ?monumento wdt:P571 ?anioConstruccion. } # Obtenemos el año de construcción (si está disponible) - OPTIONAL { ?monumento wdt:P575 ?anioDescubrimiento. } # Obtenemos el año de descubrimiento (si está disponible) - FILTER ((LANG(?titulo) = "es" || LANG(?titulo) = "en") && (BOUND(?anioConstruccion) || BOUND(?anioDescubrimiento))) # Al menos uno de los valores de año debe ser distinto de nulo - } - LIMIT 1000 + SELECT ?countryLabel + WHERE { + ?country wdt:P31 wd:Q6256; + wdt:P30 wd:Q46. # Filtra para que el país esté en Europa + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + } + `; + const europa = await wikidata.consulta(query); + + query = ` + SELECT ?countryLabel + WHERE { + ?country wdt:P31 wd:Q6256; + wdt:P30 ?continent. + FILTER(?continent != wd:Q46) # Filtra para excluir Europa + ?continent wdt:P31 wd:Q5107. # Asegura que el continente sea una instancia de continente + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + } `; + const noEuropa = await wikidata.consulta(query); - const results = await wikidata.consulta(query); + // Contendrá un elemento aleatorio del array de europa y 3 que no lo sean + const results = []; // console.log(results) return results; } - // REVISAR // - static async obtenerPaisYLenguaje() { + /* ENTRETENIMIENTO */ + + static async obtenerPeliculasAñosYDirector() { const query = ` - SELECT ?countryLabel ?languageLabel WHERE { - ?country wdt:P31 wd:Q6256. - ?country wdt:P37 ?language. - SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } + SELECT ?peliculaLabel ?directorLabel ?fecha + WHERE { + ?pelicula wdt:P31 wd:Q11424. # Filtramos por instancias de películas + ?pelicula wdt:P577 ?fecha. # Obtenemos la fecha de publicación + ?pelicula wdt:P57 ?director. # Obtenemos el director de la película + FILTER (YEAR(?fecha) > 2000). # Filtramos por películas posteriores al 2000 + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } } - LIMIT 500 - `; + LIMIT 2000 + `; const results = await wikidata.consulta(query); // console.log(results) return results; - } - static async obtenerPaisYBandera() { + static async obtenerMangaYFecha() { const query = ` - SELECT ?flag ?flagLabel ?countryLabel WHERE { - ?country wdt:P31 wd:Q6256; - wdt:P41 ?flag. - SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } + SELECT ?titulo ?fechaEstreno + WHERE { + ?obra wdt:P31/wdt:P279* wd:Q1107. # Filtramos por instancias de "anime" o "manga" + ?obra rdfs:label ?titulo. # Obtenemos el título de la obra + ?obra wdt:P580 ?fechaEstreno. # Obtenemos la fecha de estreno + FILTER (LANG(?titulo) = "es" || LANG(?titulo) = "en") # Filtramos por títulos en español o inglés } - LIMIT 200 + LIMIT 2000 `; const results = await wikidata.consulta(query); // console.log(results) return results; - } static async obtenerCantanteYCancion() { @@ -126,6 +176,53 @@ class WikiQueries { return results; } + + + /* ARTE */ + + static async obtenerMonumentoYAñoDescubOAñoConst() { + const query = ` + SELECT ?monumento ?titulo ?anioConstruccion ?anioDescubrimiento + WHERE { + ?monumento wdt:P31 wd:Q4989906. # Filtramos por instancias de "monumento" + ?monumento rdfs:label ?titulo. # Obtenemos el título del monumento + OPTIONAL { ?monumento wdt:P571 ?anioConstruccion. } # Obtenemos el año de construcción (si está disponible) + OPTIONAL { ?monumento wdt:P575 ?anioDescubrimiento. } # Obtenemos el año de descubrimiento (si está disponible) + FILTER ((LANG(?titulo) = "es" || LANG(?titulo) = "en") && (BOUND(?anioConstruccion) || BOUND(?anioDescubrimiento))) # Al menos uno de los valores de año debe ser distinto de nulo + } + LIMIT 1000 + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + + + + + + + + + + // static async obtenerPaisYLenguaje() { + // const query = ` + // SELECT ?countryLabel ?languageLabel WHERE { + // ?country wdt:P31 wd:Q6256. + // ?country wdt:P37 ?language. + // SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } + // } + // LIMIT 500 + // `; + + // const results = await wikidata.consulta(query); + // // console.log(results) + // return results; + + // } + } From c00bc9d4f0619c314a9a6637aa4115d6f0fda621 Mon Sep 17 00:00:00 2001 From: Liliana Date: Wed, 10 Apr 2024 11:33:10 +0200 Subject: [PATCH 02/21] borrada query innecesaria --- .../wikidataExtractor/wikidataQueries.js | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index 34772461..c5094b95 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -53,41 +53,6 @@ class WikiQueries { } static async obtenerUnPaisEuropeo(){ - const query = ` - SELECT ?countryLabel ?continentLabel - WHERE { - { - SELECT ?countryLabel ?continentLabel - WHERE { - ?country wdt:P31 wd:Q6256; # Selecciona instancias de país - wdt:P30 ?continent. # Obtiene el continente del país - FILTER(?continent = wd:Q46) # Filtra para que el continente sea Europa - SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } - } - ORDER BY RAND() - LIMIT 1 - } - UNION - { - SELECT ?countryLabel ?continentLabel - WHERE { - ?country wdt:P31 wd:Q6256; # Selecciona instancias de país - wdt:P30 ?continent. # Obtiene el continente del país - FILTER(?continent != wd:Q46) # Filtra para excluir Europa - SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } - } - ORDER BY RAND() - LIMIT 3 - }} - `; - - const results = await wikidata.consulta(query); - // console.log(results) - return results; - - } - - static async obtenerUnPaisEuropeo2(){ const query = ` SELECT ?countryLabel WHERE { From 4a35c1731424e4a5d9389480130eb6b2891d81b5 Mon Sep 17 00:00:00 2001 From: Liliana Date: Fri, 12 Apr 2024 16:40:33 +0200 Subject: [PATCH 03/21] consultation for country and monument --- .../wikidataExtractor/wikidataQueries.js | 55 +++---------------- 1 file changed, 8 insertions(+), 47 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index c5094b95..a1dc7e18 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -52,36 +52,21 @@ class WikiQueries { } - static async obtenerUnPaisEuropeo(){ + static async obtenerMonumentoYPais(){ const query = ` - SELECT ?countryLabel - WHERE { - ?country wdt:P31 wd:Q6256; - wdt:P30 wd:Q46. # Filtra para que el país esté en Europa - SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } - } - `; - const europa = await wikidata.consulta(query); - - query = ` - SELECT ?countryLabel - WHERE { - ?country wdt:P31 wd:Q6256; - wdt:P30 ?continent. - FILTER(?continent != wd:Q46) # Filtra para excluir Europa - ?continent wdt:P31 wd:Q5107. # Asegura que el continente sea una instancia de continente - SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } - } + SELECT ?preguntaLabel ?respuestaLabel WHERE { + ?pregunta wdt:P31 wd:Q570116; wdt:P17 ?respuesta. + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } + } + LIMIT 200 `; - const noEuropa = await wikidata.consulta(query); - // Contendrá un elemento aleatorio del array de europa y 3 que no lo sean - const results = []; + const results = await wikidata.consulta(query); // console.log(results) return results; } - + /* ENTRETENIMIENTO */ @@ -164,30 +149,6 @@ class WikiQueries { } - - - - - - - - - // static async obtenerPaisYLenguaje() { - // const query = ` - // SELECT ?countryLabel ?languageLabel WHERE { - // ?country wdt:P31 wd:Q6256. - // ?country wdt:P37 ?language. - // SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } - // } - // LIMIT 500 - // `; - - // const results = await wikidata.consulta(query); - // // console.log(results) - // return results; - - // } - } From c2ee62cba5848045eda39e58125400d65aa95039 Mon Sep 17 00:00:00 2001 From: Liliana Date: Sun, 14 Apr 2024 10:23:57 +0200 Subject: [PATCH 04/21] F1 queries --- .../wikidataExtractor/wikidataQueries.js | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index a1dc7e18..8cedada2 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -127,6 +127,46 @@ class WikiQueries { } + static async obtenerAñoYGanadorF1(){ + const query = ` + SELECT ?year ?winnerLabel + WHERE { + wd:Q1968 wdt:P793 ?event. + ?event wdt:P585 ?date. + ?event wdt:P1346 ?winner. + ?winner wdt:P31 wd:Q5. + BIND(YEAR(?date) AS ?year) + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + } + ORDER BY ?year + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + + static async obtenerAñoYEquipoGanadorF1(){ + const query = ` + SELECT ?year ?winnerLabel + WHERE { + wd:Q1968 wdt:P793 ?event. + ?event wdt:P585 ?date. + ?event wdt:P1346 ?winner. + ?winner wdt:P31 wd:Q10497835. + BIND(YEAR(?date) AS ?year) + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + } + ORDER BY ?year + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + /* ARTE */ From 75d3e6284145dcd60424d28998f854d864bc8acd Mon Sep 17 00:00:00 2001 From: Liliana Date: Thu, 18 Apr 2024 09:21:21 +0200 Subject: [PATCH 05/21] sport queries --- .../wikidataExtractor/wikidataQueries.js | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index 8cedada2..a96f6d9a 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -189,6 +189,45 @@ class WikiQueries { } + + /* DEPORTE */ + + static async obtenerJugadorYPais() { //País en el que juega + const query = ` + SELECT ?playerLabel ?countryLabel + WHERE { + ?player wdt:P106 wd:Q3665646; + wdt:P54 ?team. + ?team wdt:P31 wd:Q13393265; + wdt:P17 ?country. + SERVICE wikibase:label {bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } + } + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + + static async obtenerJugadorYDeporte() { + const query = ` + SELECT ?personLabel ?sportLabel + WHERE { + ?person wdt:P101 ?trabajo. + ?trabajo wdt:P31/wdt:P279* wd:Q31629. + ?person wdt:P106 ?sport. + + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } + } + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + } From ac99ae5305a40659e572c246903c25b89c4aae1a Mon Sep 17 00:00:00 2001 From: Liliana Date: Thu, 18 Apr 2024 15:28:55 +0200 Subject: [PATCH 06/21] new query --- .../wikidataExtractor/wikidataQueries.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index a96f6d9a..6b84e2a4 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -228,6 +228,25 @@ class WikiQueries { } + static async obtenerEstadioYAñoFund() { + const query = ` + SELECT DISTINCT ?estadioLabel ?fundacion + WHERE { + ?estadio wdt:P31 wd:Q483110 ; # Instancia de estadio deportivo + wdt:P17 wd:Q29 ; # Ubicado en España + wdt:P571 ?fechaInicio . # Fecha de inicio de la construcción + BIND(YEAR(?fechaInicio) AS ?fundacion) + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } + } + ORDER BY ?fundacion + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results; + + } + } From 7808ccd405944ececac84aea8a0252378fa459b3 Mon Sep 17 00:00:00 2001 From: Abel Date: Thu, 25 Apr 2024 20:08:40 +0200 Subject: [PATCH 07/21] Refactoriced WikidataExtractor in order to make it extensible and to execute all the different queries implemented, one at a time --- .../wikidataextractor-service.js | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 9d7084d1..4508d562 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -17,34 +17,53 @@ db.once('open', () => console.log("Connected to MongoDB: %s", mongoUri)); // Middleware to parse JSON in request body app.use(express.json()); -async function extractData() { - var data = await WikiQueries.obtenerPaisYCapital(); +const templates = [ + { + extractMethod: WikiQueries.obtenerPaisYCapital(), + filtro: (element) => { return { pais: String(element.countryLabel) }}, + campo_actualizar: (element) => { return { capital: element.capitalLabel }}, + saveMethod: (transactions) => Pais.bulkWrite(transactions) + }, + { + extractMethod: WikiQueries.obtenerPaisYLenguaje(), + filtro: (element) => { return { pais: String(element.countryLabel) }}, + campo_actualizar: (element) => { return { lenguaje: element.languageLabel }}, + saveMethod: (transactions) => Pais.bulkWrite(transactions) + }, + { + extractMethod: WikiQueries.obtenerPaisYBandera(), + filtro: (element) => { return { pais: String(element.countryLabel) }}, + campo_actualizar: (element) => { return { bandera: element.flagLabel }}, + saveMethod: (transactions) => Pais.bulkWrite(transactions) + } +]; + +async function extractData(template) { + var data = await template.extractMethod; console.log(data); - var paises = data.map(function (element) { - var p = { + var transactions = data.map(function (element) { + var transaction = { updateOne: { - filter: { pais: String(element.countryLabel) }, - update: { - capital: element.capitalLabel, - lenguaje: element.languageLabel, - bandera: element.flagLabel - }, + filter: template.filtro(element), + update: template.campo_actualizar(element), upsert: true } }; - console.log(p); - return p; + console.log(transaction); + return transaction; }); - await Pais.bulkWrite(paises); + await template.saveMethod(transactions); - return paises; + return transactions; } - -var minutes = 30; +const minutes = 30; +const totalQueries = templates.length; +var query = 0; cron.schedule(`*/${minutes} * * * *`, () => { console.log(`Running a task every ${minutes} minutes: ${Date()}`); // Call function here - extractData(); + extractData(templates[query]); + query = (query+1)%totalQueries; }); /* @@ -55,8 +74,9 @@ cron.schedule(`*/${minutes} * * * *`, () => { // Route for extracting countries app.get('/extract', async (req, res) => { try { - res.json(await extractData()); + res.json(await extractData(templates[1])); } catch (error) { + console.error(error.message); res.status(500).json({ message: error.message }) // res.status(500).json({ error: 'Internal Server Error' }); } @@ -92,7 +112,7 @@ app.use((err, req, res, next) => { // Start the server const server = app.listen(port, () => { - console.log(`Questions Service listening at http://localhost:${port}`); + console.log(`Wikidata Extractor listening at http://localhost:${port}`); }); server.on('close', () => { From 3dcc27d262b446218181890453cc4846aa314ffb Mon Sep 17 00:00:00 2001 From: Abel Date: Fri, 26 Apr 2024 01:21:11 +0200 Subject: [PATCH 08/21] =?UTF-8?q?A=C3=B1adido=20filtro=20a=20los=20datos?= =?UTF-8?q?=20obtenidos=20de=20wikidata=20para=20los=20campos=20no=20defin?= =?UTF-8?q?idos=20(los=20que=20ponen=20"Q1234"=20o=20similares)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- questionsservice/wikidataExtractor/wikidataQueries.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index 564de9f3..4c679bd2 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -2,6 +2,8 @@ const wikidata = require("./wikidataConnexion"); class WikiQueries { + static regExp = /^Q\d+$/; // Expresión regular para filtrar las etiquetas del tipo "Q1234" + /* CIENCIA */ static async obtenerSimboloQuimico() { @@ -33,7 +35,11 @@ class WikiQueries { const results = await wikidata.consulta(query); // console.log(results) - return results; + return results.filter(function(element) { + const countryOk = !WikiQueries.regExp.test(element.countryLabel); + const capitalOk = !WikiQueries.regExp.test(element.capitalLabel); + return countryOk && capitalOk; + }); } static async obtenerPaisYBandera() { From 60af73c3f035c9a7f80b427c0eb64421a20dc679 Mon Sep 17 00:00:00 2001 From: Abel Date: Fri, 26 Apr 2024 01:23:15 +0200 Subject: [PATCH 09/21] Fixed bug that was producing the queries to be executed when starting the server --- .../wikidataExtractor/wikidataextractor-service.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index d8655cce..5ac6ff7c 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -19,19 +19,19 @@ app.use(express.json()); const templates = [ { - extractMethod: WikiQueries.obtenerPaisYCapital(), + extractMethod: () => WikiQueries.obtenerPaisYCapital(), filtro: (element) => { return { pais: String(element.countryLabel) }}, campo_actualizar: (element) => { return { capital: element.capitalLabel }}, saveMethod: (transactions) => Pais.bulkWrite(transactions) }, { - extractMethod: WikiQueries.obtenerPaisYLenguaje(), + extractMethod: () => WikiQueries.obtenerPaisYLenguaje(), filtro: (element) => { return { pais: String(element.countryLabel) }}, campo_actualizar: (element) => { return { lenguaje: element.languageLabel }}, saveMethod: (transactions) => Pais.bulkWrite(transactions) }, { - extractMethod: WikiQueries.obtenerPaisYBandera(), + extractMethod: () => WikiQueries.obtenerPaisYBandera(), filtro: (element) => { return { pais: String(element.countryLabel) }}, campo_actualizar: (element) => { return { bandera: element.flagLabel }}, saveMethod: (transactions) => Pais.bulkWrite(transactions) @@ -39,7 +39,7 @@ const templates = [ ]; async function extractData(template) { - var data = await template.extractMethod; + var data = await template.extractMethod(); console.log(data); var transactions = data.map(function (element) { var transaction = { From 6a6f53382da368ab75f48657f446fa0dd342e537 Mon Sep 17 00:00:00 2001 From: Abel Date: Fri, 26 Apr 2024 10:50:31 +0200 Subject: [PATCH 10/21] Added the chemical elements data extraction from wikidata --- .../wikidataExtractor/wikidataQueries.js | 6 +++++- .../wikidataextractor-model.js | 17 +++++++++++++++-- .../wikidataextractor-service.js | 8 +++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index 4c679bd2..f752d999 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -17,7 +17,11 @@ class WikiQueries { const results = await wikidata.consulta(query); // console.log(results) - return results; + return results.filter(function(element) { + const elementOk = !WikiQueries.regExp.test(element.elementLabel); + const symbolOk = !WikiQueries.regExp.test(element.symbol); + return elementOk && symbolOk; + }); } diff --git a/questionsservice/wikidataExtractor/wikidataextractor-model.js b/questionsservice/wikidataExtractor/wikidataextractor-model.js index 1b3f9971..9308f801 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-model.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-model.js @@ -17,11 +17,24 @@ const paisSchema = new mongoose.Schema({ type: String, required: false } -}, {timestamps: {}}); // Añade y gestiona automáticamente los campos createdAt y updatedAt +}, { timestamps: {} }); // Añade y gestiona automáticamente los campos createdAt y updatedAt + +const chemicalElementsSchema = new mongoose.Schema({ + elemento: { + type: String, + required: true + }, + simbolo: { + type: String, + required: false + } +}, {timestamps: {}}); const Pais = mongoose.model('Pais', paisSchema); +const Elemento = mongoose.model('Element', chemicalElementsSchema); module.exports = { - Pais + Pais, + Elemento }; \ No newline at end of file diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 5ac6ff7c..6f009bf7 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -2,7 +2,7 @@ const express = require('express'); const cron = require('node-cron'); const mongoose = require('mongoose'); const WikiQueries = require('./wikidataQueries'); -const { Pais } = require('./wikidataextractor-model'); +const { Pais, Elemento } = require('./wikidataextractor-model'); const app = express(); const port = 8008; @@ -35,6 +35,12 @@ const templates = [ filtro: (element) => { return { pais: String(element.countryLabel) }}, campo_actualizar: (element) => { return { bandera: element.flagLabel }}, saveMethod: (transactions) => Pais.bulkWrite(transactions) + }, + { + extractMethod: () => WikiQueries.obtenerSimboloQuimico(), + filtro: (element) => { return { elemento: String(element.elementLabel) }}, + campo_actualizar: (element) => { return { simbolo: element.symbol }}, + saveMethod: (transactions) => Elemento.bulkWrite(transactions) } ]; From d36c8003275a4b6bb279cb601f0dac32c250f705 Mon Sep 17 00:00:00 2001 From: Abel Date: Sat, 27 Apr 2024 01:03:36 +0200 Subject: [PATCH 11/21] Added queries and templates for extracting data about different topics --- .../wikidataExtractor/wikidataQueries.js | 87 ++++++++++++------- .../wikidataextractor-model.js | 46 +++++++++- .../wikidataextractor-service.js | 39 +++++++-- 3 files changed, 133 insertions(+), 39 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index f752d999..967dcf37 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -6,7 +6,7 @@ class WikiQueries { /* CIENCIA */ - static async obtenerSimboloQuimico() { + static async obtenerSimboloQuimico() { // En uso const query = `SELECT ?elementLabel ?symbol WHERE { ?element wdt:P31 wd:Q11344. @@ -28,7 +28,7 @@ class WikiQueries { /* GEOGRAFÍA */ - static async obtenerPaisYCapital() { + static async obtenerPaisYCapital() { // En uso const query = ` SELECT ?countryLabel ?capitalLabel WHERE { ?country wdt:P31 wd:Q6256. @@ -46,6 +46,24 @@ class WikiQueries { }); } + static async obtenerPaisYContinente() { // En uso + const query = ` + SELECT ?countryLabel ?continentLabel WHERE { + ?country wdt:P31 wd:Q6256. + ?country wdt:P30 ?continent. + SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } + } + `; + + const results = await wikidata.consulta(query); + // console.log(results) + return results.filter(function(element) { + const countryOk = !WikiQueries.regExp.test(element.countryLabel); + const continentOk = !WikiQueries.regExp.test(element.continentLabel); + return countryOk && continentOk; + }); + } + static async obtenerPaisYBandera() { const query = ` SELECT ?flag ?flagLabel ?countryLabel WHERE { @@ -78,40 +96,45 @@ class WikiQueries { } - static async obtenerMonumentoYPais(){ + static async obtenerMonumentoYPais(){ // En uso const query = ` - SELECT ?preguntaLabel ?respuestaLabel WHERE { - ?pregunta wdt:P31 wd:Q570116; wdt:P17 ?respuesta. + SELECT ?monumentLabel ?countryLabel WHERE { + ?monument wdt:P31 wd:Q570116; wdt:P17 ?country. SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } } - LIMIT 200 + LIMIT 2500 `; const results = await wikidata.consulta(query); // console.log(results) - return results; + return results.filter(function(element) { + const monumentOk = !WikiQueries.regExp.test(element.monumentLabel); + const countryOk = !WikiQueries.regExp.test(element.countryLabel); + return countryOk && monumentOk; + }); } /* ENTRETENIMIENTO */ - static async obtenerPeliculasAñosYDirector() { + static async obtenerPeliculaYDirector() { // En uso const query = ` - SELECT ?peliculaLabel ?directorLabel ?fecha - WHERE { + SELECT ?peliculaLabel ?directorLabel WHERE { ?pelicula wdt:P31 wd:Q11424. # Filtramos por instancias de películas - ?pelicula wdt:P577 ?fecha. # Obtenemos la fecha de publicación ?pelicula wdt:P57 ?director. # Obtenemos el director de la película - FILTER (YEAR(?fecha) > 2000). # Filtramos por películas posteriores al 2000 SERVICE wikibase:label { bd:serviceParam wikibase:language "es". } - } - LIMIT 2000 - `; + } + LIMIT 2500 + `; const results = await wikidata.consulta(query); // console.log(results) - return results; + return results.filter(function(element) { + const peliculaOk = !WikiQueries.regExp.test(element.peliculaLabel); + const directorOk = !WikiQueries.regExp.test(element.directorLabel); + return peliculaOk && directorOk; + }); } static async obtenerMangaYFecha() { @@ -131,25 +154,29 @@ class WikiQueries { return results; } - static async obtenerCantanteYCancion() { + static async obtenerCancionYArtista() { // En uso const query = ` - SELECT ?song ?songLabel ?singer ?singerLabel - WHERE { - ?song wdt:P31 wd:Q7366; # Canción - wdt:P175 ?singer. # Cantante - ?singer wdt:P27 wd:Q29. # Español - MINUS { - ?song wdt:P175 ?anotherSinger. # Quitamos canciones con más de un cantante - FILTER (?anotherSinger != ?singer) - } - SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } - } - LIMIT 200 + SELECT DISTINCT ?songLabel ?artistLabel WHERE { + ?song wdt:P31 wd:Q7366; # Instances of songs + wdt:P175 ?artist. # With property "performer" (artist) + + ?song wdt:P136 ?genre. # Filter by genre + VALUES ?genre { wd:Q202930 wd:Q188450 wd:Q11401 wd:Q20502 wd:Q58339 wd:Q211756 wd:Q474027 wd:Q484641 wd:Q547137 } # Specify genres + + OPTIONAL { ?song wdt:P175 ?secondArtist FILTER (?artist != ?secondArtist) } # Optional second performer + FILTER(!bound(?secondArtist)) # Filter out songs with a second performer + + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } + } `; const results = await wikidata.consulta(query); // console.log(results) - return results; + return results.filter(function(element) { + const songOk = !WikiQueries.regExp.test(element.songLabel); + const artistOk = !WikiQueries.regExp.test(element.artistLabel); + return songOk && artistOk; + }); } diff --git a/questionsservice/wikidataExtractor/wikidataextractor-model.js b/questionsservice/wikidataExtractor/wikidataextractor-model.js index 9308f801..7d1d8351 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-model.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-model.js @@ -9,6 +9,10 @@ const paisSchema = new mongoose.Schema({ type: String, required: false }, + continente: { + type: String, + required: false + }, lenguaje: { type: String, required: false @@ -19,6 +23,17 @@ const paisSchema = new mongoose.Schema({ } }, { timestamps: {} }); // Añade y gestiona automáticamente los campos createdAt y updatedAt +const monumentSchema = new mongoose.Schema({ + monumento: { + type: String, + required: true + }, + pais: { + type: String, + required: false + } +}, {timestamps: {}}); + const chemicalElementsSchema = new mongoose.Schema({ elemento: { type: String, @@ -30,11 +45,38 @@ const chemicalElementsSchema = new mongoose.Schema({ } }, {timestamps: {}}); +const filmSchema = new mongoose.Schema({ + pelicula: { + type: String, + required: true + }, + director: { + type: String, + required: false + } +}, {timestamps: {}}); + +const songSchema = new mongoose.Schema({ + cancion: { + type: String, + required: true + }, + artista: { + type: String, + required: false + } +}, {timestamps: {}}); const Pais = mongoose.model('Pais', paisSchema); -const Elemento = mongoose.model('Element', chemicalElementsSchema); +const Monumento = mongoose.model('Monumento', monumentSchema); +const Elemento = mongoose.model('Elemento', chemicalElementsSchema); +const Pelicula = mongoose.model('Pelicula', filmSchema); +const Cancion = mongoose.model('Cancion', songSchema); module.exports = { Pais, - Elemento + Monumento, + Elemento, + Pelicula, + Cancion }; \ No newline at end of file diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 6f009bf7..b0c67bd6 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -2,7 +2,7 @@ const express = require('express'); const cron = require('node-cron'); const mongoose = require('mongoose'); const WikiQueries = require('./wikidataQueries'); -const { Pais, Elemento } = require('./wikidataextractor-model'); +const { Pais, Monumento, Elemento, Pelicula, Cancion } = require('./wikidataextractor-model'); const app = express(); const port = 8008; @@ -25,22 +25,46 @@ const templates = [ saveMethod: (transactions) => Pais.bulkWrite(transactions) }, { - extractMethod: () => WikiQueries.obtenerPaisYLenguaje(), + extractMethod: () => WikiQueries.obtenerPaisYContinente(), filtro: (element) => { return { pais: String(element.countryLabel) }}, - campo_actualizar: (element) => { return { lenguaje: element.languageLabel }}, + campo_actualizar: (element) => { return { continente: element.continentLabel }}, saveMethod: (transactions) => Pais.bulkWrite(transactions) }, { - extractMethod: () => WikiQueries.obtenerPaisYBandera(), - filtro: (element) => { return { pais: String(element.countryLabel) }}, - campo_actualizar: (element) => { return { bandera: element.flagLabel }}, - saveMethod: (transactions) => Pais.bulkWrite(transactions) + extractMethod: () => WikiQueries.obtenerMonumentoYPais(), + filtro: (element) => { return { monumento: String(element.monumentLabel) }}, + campo_actualizar: (element) => { return { pais: element.countryLabel }}, + saveMethod: (transactions) => Monumento.bulkWrite(transactions) }, + // { + // extractMethod: () => WikiQueries.obtenerPaisYLenguaje(), + // filtro: (element) => { return { pais: String(element.countryLabel) }}, + // campo_actualizar: (element) => { return { lenguaje: element.languageLabel }}, + // saveMethod: (transactions) => Pais.bulkWrite(transactions) + // }, + // { + // extractMethod: () => WikiQueries.obtenerPaisYBandera(), + // filtro: (element) => { return { pais: String(element.countryLabel) }}, + // campo_actualizar: (element) => { return { bandera: element.flagLabel }}, + // saveMethod: (transactions) => Pais.bulkWrite(transactions) + // }, { extractMethod: () => WikiQueries.obtenerSimboloQuimico(), filtro: (element) => { return { elemento: String(element.elementLabel) }}, campo_actualizar: (element) => { return { simbolo: element.symbol }}, saveMethod: (transactions) => Elemento.bulkWrite(transactions) + }, + { + extractMethod: () => WikiQueries.obtenerPeliculaYDirector(), + filtro: (element) => { return { pelicula: String(element.peliculaLabel) }}, + campo_actualizar: (element) => { return { director: element.directorLabel }}, + saveMethod: (transactions) => Pelicula.bulkWrite(transactions) + }, + { + extractMethod: () => WikiQueries.obtenerCancionYArtista(), + filtro: (element) => { return { cancion: String(element.songLabel) }}, + campo_actualizar: (element) => { return { artista: element.artistLabel }}, + saveMethod: (transactions) => Cancion.bulkWrite(transactions) } ]; @@ -62,6 +86,7 @@ async function extractData(template) { return transactions; } + const minutes = 30; const totalQueries = templates.length; var query = 0; From 351dd3460663b0cf3bfe866a71624f839d89ce23 Mon Sep 17 00:00:00 2001 From: Abel Date: Sun, 28 Apr 2024 02:50:13 +0200 Subject: [PATCH 12/21] Added the extraction templates for new data topics and a new method to prevent decoy answers identical to the correct one --- .../questiongenerator-model.js | 57 +++++- .../questiongenerator.js | 167 ++++++++++++++++-- 2 files changed, 209 insertions(+), 15 deletions(-) diff --git a/questionsservice/questiongeneratorservice/questiongenerator-model.js b/questionsservice/questiongeneratorservice/questiongenerator-model.js index c589bf36..7d1d8351 100644 --- a/questionsservice/questiongeneratorservice/questiongenerator-model.js +++ b/questionsservice/questiongeneratorservice/questiongenerator-model.js @@ -9,6 +9,10 @@ const paisSchema = new mongoose.Schema({ type: String, required: false }, + continente: { + type: String, + required: false + }, lenguaje: { type: String, required: false @@ -19,9 +23,60 @@ const paisSchema = new mongoose.Schema({ } }, { timestamps: {} }); // Añade y gestiona automáticamente los campos createdAt y updatedAt +const monumentSchema = new mongoose.Schema({ + monumento: { + type: String, + required: true + }, + pais: { + type: String, + required: false + } +}, {timestamps: {}}); + +const chemicalElementsSchema = new mongoose.Schema({ + elemento: { + type: String, + required: true + }, + simbolo: { + type: String, + required: false + } +}, {timestamps: {}}); + +const filmSchema = new mongoose.Schema({ + pelicula: { + type: String, + required: true + }, + director: { + type: String, + required: false + } +}, {timestamps: {}}); + +const songSchema = new mongoose.Schema({ + cancion: { + type: String, + required: true + }, + artista: { + type: String, + required: false + } +}, {timestamps: {}}); const Pais = mongoose.model('Pais', paisSchema); +const Monumento = mongoose.model('Monumento', monumentSchema); +const Elemento = mongoose.model('Elemento', chemicalElementsSchema); +const Pelicula = mongoose.model('Pelicula', filmSchema); +const Cancion = mongoose.model('Cancion', songSchema); module.exports = { - Pais + Pais, + Monumento, + Elemento, + Pelicula, + Cancion }; \ No newline at end of file diff --git a/questionsservice/questiongeneratorservice/questiongenerator.js b/questionsservice/questiongeneratorservice/questiongenerator.js index 07005323..4fc26c1a 100644 --- a/questionsservice/questiongeneratorservice/questiongenerator.js +++ b/questionsservice/questiongeneratorservice/questiongenerator.js @@ -1,34 +1,133 @@ -const { Pais } = require('./questiongenerator-model') +const { Pais, Monumento, Elemento, Pelicula, Cancion } = require('./questiongenerator-model') class QuestionGenerator { static temas = new Map([ - ["paises", [0, 1, 2]], - ['capital', [0, 1]], - ["lenguaje", [2]] + ["Paises", [0, 1, 2, 3, 4, 5]], + ['Capitales', [0, 1]], + ['Continentes', [2, 3]], + ['Monumentos', [4, 5]], + ['Quimica', [6, 7]], + ['Peliculas', [8, 9]], + ['Canciones', [10, 11]] + // ["Lenguajes", []] ]); - ; static plantillas = [ - { + { // 0: Paises, Capitales + modelo: Pais, + generateMethod: (plantilla, respuestas) => this.generateQuestion1to1Relation(plantilla, respuestas), pregunta: (param) => `¿Cuál es la capital de ${param}?`, filtro: { pais: { $exists: true }, capital: { $exists: true } }, campo_pregunta: 'pais', campo_respuesta: 'capital' }, - { + { // 1: Paises, Capitales + modelo: Pais, + generateMethod: (plantilla, respuestas) => this.generateQuestion1to1Relation(plantilla, respuestas), pregunta: (param) => `¿De qué país es capital ${param}?`, filtro: { capital: { $exists: true }, pais: { $exists: true } }, campo_pregunta: 'capital', campo_respuesta: 'pais' }, - { - pregunta: (param) => `¿Qué lengua se habla en ${param}?`, - filtro: { pais: { $exists: true }, lenguaje: { $exists: true } }, + { // 2: Paises, Continentes - Meh, repite mucho los continentes + modelo: Pais, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿En qué continente se situa ${param}?`, + filtro: { pais: { $exists: true }, continente: { $exists: true } }, + filtro_decoys: (answer) => { return { pais: { $exists: true }, continente: { $exists: true, $ne: answer.continente} }}, campo_pregunta: 'pais', - campo_respuesta: 'lenguaje' + campo_respuesta: 'continente' + }, + { // 3: Paises, Continentes + modelo: Pais, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿Cual de los siguientes paises se situa en ${param}?`, + filtro: { pais: { $exists: true }, continente: { $exists: true } }, + filtro_decoys: (answer) => { return { pais: { $exists: true }, continente: { $exists: true, $ne: answer.continente} }}, + campo_pregunta: 'continente', + campo_respuesta: 'pais' + }, + { // 4: Paises, Monumentos + modelo: Monumento, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿En qué país se situa la atracción turística "${param}"?`, + filtro: { pais: { $exists: true }, monumento: { $exists: true } }, + filtro_decoys: (answer) => { return { monumento: { $exists: true }, pais: { $exists: true, $ne: answer.pais} }}, + campo_pregunta: 'monumento', + campo_respuesta: 'pais' + }, + { // 5: Paises, Monumentos + modelo: Monumento, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿Cuál de las siguientes atraccioones turísticas se encuentra en ${param}?`, + filtro: { pais: { $exists: true }, monumento: { $exists: true } }, + filtro_decoys: (answer) => { return { monumento: { $exists: true }, pais: { $exists: true, $ne: answer.pais} }}, + campo_pregunta: 'pais', + campo_respuesta: 'monumento' + }, + { // 6: Quimica + modelo: Elemento, + generateMethod: (plantilla, respuestas) => this.generateQuestion1to1Relation(plantilla, respuestas), + pregunta: (param) => `¿Cuál es el símbolo químico del ${param}?`, + filtro: { elemento: { $exists: true }, simbolo: { $exists: true } }, + campo_pregunta: 'elemento', + campo_respuesta: 'simbolo' + }, + { // 7: Quimica + modelo: Elemento, + generateMethod: (plantilla, respuestas) => this.generateQuestion1to1Relation(plantilla, respuestas), + pregunta: (param) => `¿Qué elemento químico representa el símbolo "${param}"?`, + filtro: { elemento: { $exists: true }, simbolo: { $exists: true } }, + campo_pregunta: 'simbolo', + campo_respuesta: 'elemento' + }, + { // 8: Peliculas + modelo: Pelicula, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿Quién fue el director de la película "${param}"?`, + filtro: { pelicula: { $exists: true }, director: { $exists: true } }, + filtro_decoys: (answer) => { return { pelicula: { $exists: true }, director: { $exists: true, $ne: answer.director} }}, + campo_pregunta: 'pelicula', + campo_respuesta: 'director' + }, + { // 9: Peliculas + modelo: Pelicula, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿Cuál de estas películas ha sido dirigida por "${param}"?`, + filtro: { pelicula: { $exists: true }, director: { $exists: true } }, + filtro_decoys: (answer) => { return { pelicula: { $exists: true }, director: { $exists: true, $ne: answer.director} }}, + campo_pregunta: 'director', + campo_respuesta: 'pelicula' + }, + { // 10: Canciones + modelo: Cancion, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿Quién canta la canción "${param}"?`, + filtro: { cancion: { $exists: true }, artista: { $exists: true } }, + filtro_decoys: (answer) => { return { cancion: { $exists: true }, artista: { $exists: true, $ne: answer.artista} }}, + campo_pregunta: 'cancion', + campo_respuesta: 'artista' + }, + { // 11: Canciones + modelo: Cancion, + generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + pregunta: (param) => `¿Cuál de las siguientes canciones es interpretada por "${param}"?`, + filtro: { cancion: { $exists: true }, artista: { $exists: true } }, + filtro_decoys: (answer) => { return { cancion: { $exists: true }, artista: { $exists: true, $ne: answer.artista} }}, + campo_pregunta: 'artista', + campo_respuesta: 'cancion' } // { + // modelo: Pais, + // generateMethod: (plantilla, respuestas) => this.generateQuestionNonDuplicatedAnswers(plantilla, respuestas), + // pregunta: (param) => `¿Qué lengua se habla en ${param}?`, + // filtro: { pais: { $exists: true }, lenguaje: { $exists: true } }, + // filtro_decoys: (answer) => { return { pais: { $exists: true }, lenguaje: { $exists: true, $ne: answer} }}, + // campo_pregunta: 'pais', + // campo_respuesta: 'lenguaje' + // }, + // { // pregunta: (param) => `¿Cuál es la bandera de ${param}?`, // filtro: { bandera: { $exists: true } }, // campo_pregunta: 'pais', @@ -40,11 +139,50 @@ class QuestionGenerator { return [ ...this.temas.keys() ]; } - static async generateQuestion(plantilla, respuestas) { + + static async generateQuestionNonDuplicatedAnswers(plantilla, respuestas) { + console.log("\nPlantilla:"); + console.log(plantilla); + + const randomAnswer = await plantilla.modelo.aggregate([ + { $match: plantilla.filtro }, + { $sample: { size: 1 } } + ]); + if (randomAnswer.length < 1) { + console.error(`Not enought data found to generate a question`); + throw new Error(`Not enought data found to generate a question`); + } + var randomDecoys = []; + if (respuestas > 1){ + randomDecoys = await plantilla.modelo.aggregate([ + { $match: plantilla.filtro_decoys(randomAnswer[0]) }, + { $sample: { size: respuestas-1 } } + ]); + } + if (randomDecoys.length < respuestas-1) { + console.error(`Not enought data found to generate a question`); + throw new Error(`Not enought data found to generate a question`); + } + + console.log("\nFind:"); + console.log(randomDecoys); + + var retQuestion = { + pregunta: plantilla.pregunta(randomAnswer[0][plantilla.campo_pregunta]), + respuesta_correcta: randomAnswer[0][plantilla.campo_respuesta], + respuestas_incorrectas: Array.from({ length: respuestas-1 }, (_, i) => randomDecoys[i][plantilla.campo_respuesta]) + }; + console.log("\nPregunta generada:"); + console.log(retQuestion); + + return retQuestion; + } + + static async generateQuestion1to1Relation(plantilla, respuestas) { console.log("\nPlantilla:"); console.log(plantilla); - const randomDocs = await Pais.aggregate([ + const randomDocs = await plantilla.modelo.aggregate([ { $match: plantilla.filtro }, { $sample: { size: respuestas } } ]); @@ -74,7 +212,8 @@ class QuestionGenerator { var retQuestions = []; for (let i = 0; i < preguntas; i++) { let index = Math.floor(Math.random() * plantillasDisponibles.length); - retQuestions.push(await this.generateQuestion(this.plantillas[plantillasDisponibles[index]], respuestas)); + let plantilla = this.plantillas[plantillasDisponibles[index]]; + retQuestions.push(await plantilla.generateMethod(plantilla, respuestas)); } return retQuestions; } From 295bb6195a57a6480c89850571c31bdda42525bb Mon Sep 17 00:00:00 2001 From: Abel Date: Sun, 28 Apr 2024 03:11:19 +0200 Subject: [PATCH 13/21] Fixed languaje on the query MonumentoYPais --- questionsservice/wikidataExtractor/wikidataQueries.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index 967dcf37..a9160993 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -100,7 +100,7 @@ class WikiQueries { const query = ` SELECT ?monumentLabel ?countryLabel WHERE { ?monument wdt:P31 wd:Q570116; wdt:P17 ?country. - SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". } + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } } LIMIT 2500 `; From 850ae5875909e79806b53d6120357c286fddb0c9 Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 00:42:40 +0200 Subject: [PATCH 14/21] Moddified the information showed in the WikidataExtractor console logs --- .../wikidataExtractor/wikidataQueries.js | 26 ++++++++++++------- .../wikidataextractor-service.js | 15 +++++++---- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataQueries.js b/questionsservice/wikidataExtractor/wikidataQueries.js index a9160993..18782279 100644 --- a/questionsservice/wikidataExtractor/wikidataQueries.js +++ b/questionsservice/wikidataExtractor/wikidataQueries.js @@ -7,12 +7,13 @@ class WikiQueries { /* CIENCIA */ static async obtenerSimboloQuimico() { // En uso - const query = - `SELECT ?elementLabel ?symbol WHERE { - ?element wdt:P31 wd:Q11344. - ?element wdt:P246 ?symbol. - SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } - } + console.log("Símbolos químicos"); + const query = ` + SELECT ?elementLabel ?symbol WHERE { + ?element wdt:P31 wd:Q11344. + ?element wdt:P246 ?symbol. + SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } + } `; const results = await wikidata.consulta(query); @@ -29,6 +30,7 @@ class WikiQueries { /* GEOGRAFÍA */ static async obtenerPaisYCapital() { // En uso + console.log("Países y Capitales"); const query = ` SELECT ?countryLabel ?capitalLabel WHERE { ?country wdt:P31 wd:Q6256. @@ -47,6 +49,7 @@ class WikiQueries { } static async obtenerPaisYContinente() { // En uso + console.log("Países y Continentes"); const query = ` SELECT ?countryLabel ?continentLabel WHERE { ?country wdt:P31 wd:Q6256. @@ -70,8 +73,8 @@ class WikiQueries { ?country wdt:P31 wd:Q6256; wdt:P41 ?flag. SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } - } - LIMIT 200 + } + LIMIT 200 `; const results = await wikidata.consulta(query); @@ -86,8 +89,8 @@ class WikiQueries { ?country wdt:P31 wd:Q6256. ?country wdt:P37 ?language. SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],es". } - } - LIMIT 500 + } + LIMIT 500 `; const results = await wikidata.consulta(query); @@ -97,6 +100,7 @@ class WikiQueries { } static async obtenerMonumentoYPais(){ // En uso + console.log("Países y Monumentos"); const query = ` SELECT ?monumentLabel ?countryLabel WHERE { ?monument wdt:P31 wd:Q570116; wdt:P17 ?country. @@ -119,6 +123,7 @@ class WikiQueries { /* ENTRETENIMIENTO */ static async obtenerPeliculaYDirector() { // En uso + console.log("Películas y Directores"); const query = ` SELECT ?peliculaLabel ?directorLabel WHERE { ?pelicula wdt:P31 wd:Q11424. # Filtramos por instancias de películas @@ -155,6 +160,7 @@ class WikiQueries { } static async obtenerCancionYArtista() { // En uso + console.log("Canciones y Artistas"); const query = ` SELECT DISTINCT ?songLabel ?artistLabel WHERE { ?song wdt:P31 wd:Q7366; # Instances of songs diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index b0c67bd6..879088ae 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -69,6 +69,7 @@ const templates = [ ]; async function extractData(template) { + console.log("Actualizando los datos sobre:") var data = await template.extractMethod(); console.log(data); var transactions = data.map(function (element) { @@ -79,7 +80,7 @@ async function extractData(template) { upsert: true } }; - console.log(transaction); + // console.log(transaction); return transaction; }); await template.saveMethod(transactions); @@ -91,10 +92,14 @@ const minutes = 30; const totalQueries = templates.length; var query = 0; cron.schedule(`*/${minutes} * * * *`, () => { - console.log(`Running a task every ${minutes} minutes: ${Date()}`); - // Call function here - extractData(templates[query]); - query = (query+1)%totalQueries; + try { + console.log(`Running a task every ${minutes} minutes: ${Date()}`); + extractData(templates[query]); + query = (query+1)%totalQueries; + } catch (error) { + console.error(error.message) + } + }); /* From 1adf724e916b0be388e131254026dff38e3237a4 Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 01:10:33 +0200 Subject: [PATCH 15/21] Modified the information showed in the QuestionGenerator console logs --- .../questiongenerator-service.js | 2 +- .../questiongenerator.js | 43 +++++++------------ 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/questionsservice/questiongeneratorservice/questiongenerator-service.js b/questionsservice/questiongeneratorservice/questiongenerator-service.js index c4a39d3b..909caa41 100644 --- a/questionsservice/questiongeneratorservice/questiongenerator-service.js +++ b/questionsservice/questiongeneratorservice/questiongenerator-service.js @@ -62,7 +62,7 @@ app.get('/questions', async (req, res) => { } } catch (error) { console.error(`Bad Request: ${error.message}`); - res.status(400).json({ message: error.message }); + res.status(400).json({ error: error.message }); } }); diff --git a/questionsservice/questiongeneratorservice/questiongenerator.js b/questionsservice/questiongeneratorservice/questiongenerator.js index 4fc26c1a..3de28955 100644 --- a/questionsservice/questiongeneratorservice/questiongenerator.js +++ b/questionsservice/questiongeneratorservice/questiongenerator.js @@ -141,9 +141,8 @@ class QuestionGenerator { static async generateQuestionNonDuplicatedAnswers(plantilla, respuestas) { - console.log("\nPlantilla:"); - console.log(plantilla); - + // console.log("\nPlantilla:"); + // console.log(plantilla); const randomAnswer = await plantilla.modelo.aggregate([ { $match: plantilla.filtro }, { $sample: { size: 1 } } @@ -163,25 +162,19 @@ class QuestionGenerator { console.error(`Not enought data found to generate a question`); throw new Error(`Not enought data found to generate a question`); } - - console.log("\nFind:"); - console.log(randomDecoys); - + // console.log("\nFind:"); + // console.log(randomDecoys); var retQuestion = { pregunta: plantilla.pregunta(randomAnswer[0][plantilla.campo_pregunta]), respuesta_correcta: randomAnswer[0][plantilla.campo_respuesta], respuestas_incorrectas: Array.from({ length: respuestas-1 }, (_, i) => randomDecoys[i][plantilla.campo_respuesta]) }; - console.log("\nPregunta generada:"); - console.log(retQuestion); - return retQuestion; } static async generateQuestion1to1Relation(plantilla, respuestas) { - console.log("\nPlantilla:"); - console.log(plantilla); - + // console.log("\nPlantilla:"); + // console.log(plantilla); const randomDocs = await plantilla.modelo.aggregate([ { $match: plantilla.filtro }, { $sample: { size: respuestas } } @@ -190,31 +183,26 @@ class QuestionGenerator { console.error(`Not enought data found to generate a question`); throw new Error(`Not enought data found to generate a question`); } - - console.log("\nFind:"); - console.log(randomDocs); - + // console.log("\nFind:"); + // console.log(randomDocs); var retQuestion = { pregunta: plantilla.pregunta(randomDocs[0][plantilla.campo_pregunta]), respuesta_correcta: randomDocs[0][plantilla.campo_respuesta], respuestas_incorrectas: Array.from({ length: respuestas-1 }, (_, i) => randomDocs[i+1][plantilla.campo_respuesta]) }; - console.log("\nPregunta generada:"); - console.log(retQuestion); - return retQuestion; } static async generateQuestions(preguntas, respuestas, temas) { - console.log(temas); const plantillasDisponibles = this.getAvailableTemplates(temas); - console.log(plantillasDisponibles); var retQuestions = []; for (let i = 0; i < preguntas; i++) { let index = Math.floor(Math.random() * plantillasDisponibles.length); let plantilla = this.plantillas[plantillasDisponibles[index]]; retQuestions.push(await plantilla.generateMethod(plantilla, respuestas)); } + console.log("\nPreguntas generadas:"); + console.log(retQuestions); return retQuestions; } @@ -223,23 +211,22 @@ class QuestionGenerator { return Array.from({ length: this.plantillas.length }, (_, i) => i); } var templates = []; + console.log("Temas a utilizar:") temas.forEach(tema => { - console.log(tema); if (this.temas.has(tema)) { templates = templates.concat(this.temas.get(tema)); - console.log(this.temas.get(tema)); + console.log(`\t${tema}`); } else { - console.error(`The topic \'${tema}\' is not currently defined`); - throw new Error(`The topic \'${tema}\' is not currently defined`); + console.error(`\tThe topic \'${tema}\' is not currently defined`); } }); if (templates.length == 0) { console.error(`No correct topics were passed`); throw new Error(`No correct topics were passed`); } - console.log(templates); - console.log([...new Set(templates)]); + console.log("Plantillas a utilizar:"); + console.log(`\t${[...new Set(templates)]}`); return [...new Set(templates)]; } From 456001864e0c33852ccfc08cf7b0b80fde17fd9e Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 01:16:15 +0200 Subject: [PATCH 16/21] Commented Develop only endpoints --- .../wikidataextractor-service.js | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 879088ae..2147ec08 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -108,37 +108,37 @@ cron.schedule(`*/${minutes} * * * *`, () => { */ // Route for extracting countries -app.get('/extract', async (req, res) => { - try { - res.json(await extractData(templates[1])); - } catch (error) { - console.error(error.message); - res.status(500).json({ message: error.message }) - // res.status(500).json({ error: 'Internal Server Error' }); - } -}); - -// // Route for geting countries -app.get('/countries', async (req, res) => { - try { - const paises = await Pais.find({}) - res.json(paises); - } catch (error) { - res.status(500).json({ message: error.message }) - // res.status(500).json({ error: 'Internal Server Error' }); - } -}); - -// // Route for deleting countries -app.delete('/countries', async (req, res) => { - try { - const paises = await Pais.deleteMany({}) - res.json(paises); - } catch (error) { - res.status(500).json({ message: error.message }) - // res.status(500).json({ error: 'Internal Server Error' }); - } -}); +// app.get('/extract', async (req, res) => { +// try { +// res.json(await extractData(templates[1])); +// } catch (error) { +// console.error(error.message); +// res.status(500).json({ message: error.message }) +// // res.status(500).json({ error: 'Internal Server Error' }); +// } +// }); + +// // // Route for geting countries +// app.get('/countries', async (req, res) => { +// try { +// const paises = await Pais.find({}) +// res.json(paises); +// } catch (error) { +// res.status(500).json({ message: error.message }) +// // res.status(500).json({ error: 'Internal Server Error' }); +// } +// }); + +// // // Route for deleting countries +// app.delete('/countries', async (req, res) => { +// try { +// const paises = await Pais.deleteMany({}) +// res.json(paises); +// } catch (error) { +// res.status(500).json({ message: error.message }) +// // res.status(500).json({ error: 'Internal Server Error' }); +// } +// }); app.use((err, req, res, next) => { From 4f294343d1e54409851fba46155679d484b025ab Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 01:20:03 +0200 Subject: [PATCH 17/21] npm audit fix --- .../package-lock.json | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/questionsservice/questiongeneratorservice/package-lock.json b/questionsservice/questiongeneratorservice/package-lock.json index 2240421b..5467d7fe 100644 --- a/questionsservice/questiongeneratorservice/package-lock.json +++ b/questionsservice/questiongeneratorservice/package-lock.json @@ -106,12 +106,12 @@ } }, "node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "dependencies": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", @@ -119,7 +119,7 @@ "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.11.0", - "raw-body": "2.5.1", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -245,9 +245,9 @@ } }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "engines": { "node": ">= 0.6" } @@ -364,16 +364,16 @@ } }, "node_modules/express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -434,9 +434,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", - "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -587,9 +587,9 @@ } }, "node_modules/hasown": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.1.tgz", - "integrity": "sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", "dependencies": { "function-bind": "^1.1.2" }, @@ -1086,9 +1086,9 @@ } }, "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", @@ -1193,16 +1193,16 @@ } }, "node_modules/set-function-length": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.1.tgz", - "integrity": "sha512-j4t6ccc+VsKwYHso+kElc5neZpjtq9EnRICFZtWyBsLojhmeF/ZBd/elqm22WJh/BziDe/SBiOeAt0m2mfLD0g==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", "dependencies": { - "define-data-property": "^1.1.2", + "define-data-property": "^1.1.4", "es-errors": "^1.3.0", "function-bind": "^1.1.2", - "get-intrinsic": "^1.2.3", + "get-intrinsic": "^1.2.4", "gopd": "^1.0.1", - "has-property-descriptors": "^1.0.1" + "has-property-descriptors": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -1214,11 +1214,11 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" }, "node_modules/side-channel": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.5.tgz", - "integrity": "sha512-QcgiIWV4WV7qWExbN5llt6frQB/lBven9pqliLXfGPB+K9ZYXxDozp0wLkHS24kWCm+6YXH/f0HhnObZnZOBnQ==", + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", "dependencies": { - "call-bind": "^1.0.6", + "call-bind": "^1.0.7", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.4", "object-inspect": "^1.13.1" From 75489c47e7054e61c15abb3a83321277bf9ec51c Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 01:44:13 +0200 Subject: [PATCH 18/21] Added containers to the sonar-cloud code checker --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index b83524d3..3bdd0f8d 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -11,7 +11,7 @@ sonar.language=js sonar.projectName=wiq_es6c sonar.coverage.exclusions=**/*.test.js -sonar.sources=webapp/src/components,users/authservice,users/userservice,gatewayservice +sonar.sources=webapp/src/components,users/authservice,users/userservice,gatewayservice,questionsservice/questiongeneratorservice,questionsservice/wikidataExtractor sonar.sourceEncoding=UTF-8 sonar.exclusions=node_modules/** sonar.javascript.lcov.reportPaths=**/coverage/lcov.info From e0624cfffe0a7304cce11519207a5d909c6461e2 Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 02:24:51 +0200 Subject: [PATCH 19/21] Trying to clean my code --- .../questiongenerator.js | 23 ++++----- .../wikidataextractor-service.js | 48 ++----------------- 2 files changed, 13 insertions(+), 58 deletions(-) diff --git a/questionsservice/questiongeneratorservice/questiongenerator.js b/questionsservice/questiongeneratorservice/questiongenerator.js index 3de28955..73b4ce1c 100644 --- a/questionsservice/questiongeneratorservice/questiongenerator.js +++ b/questionsservice/questiongeneratorservice/questiongenerator.js @@ -141,8 +141,6 @@ class QuestionGenerator { static async generateQuestionNonDuplicatedAnswers(plantilla, respuestas) { - // console.log("\nPlantilla:"); - // console.log(plantilla); const randomAnswer = await plantilla.modelo.aggregate([ { $match: plantilla.filtro }, { $sample: { size: 1 } } @@ -151,7 +149,7 @@ class QuestionGenerator { console.error(`Not enought data found to generate a question`); throw new Error(`Not enought data found to generate a question`); } - var randomDecoys = []; + let randomDecoys = []; if (respuestas > 1){ randomDecoys = await plantilla.modelo.aggregate([ { $match: plantilla.filtro_decoys(randomAnswer[0]) }, @@ -162,9 +160,8 @@ class QuestionGenerator { console.error(`Not enought data found to generate a question`); throw new Error(`Not enought data found to generate a question`); } - // console.log("\nFind:"); - // console.log(randomDecoys); - var retQuestion = { + + const retQuestion = { pregunta: plantilla.pregunta(randomAnswer[0][plantilla.campo_pregunta]), respuesta_correcta: randomAnswer[0][plantilla.campo_respuesta], respuestas_incorrectas: Array.from({ length: respuestas-1 }, (_, i) => randomDecoys[i][plantilla.campo_respuesta]) @@ -173,8 +170,7 @@ class QuestionGenerator { } static async generateQuestion1to1Relation(plantilla, respuestas) { - // console.log("\nPlantilla:"); - // console.log(plantilla); + const randomDocs = await plantilla.modelo.aggregate([ { $match: plantilla.filtro }, { $sample: { size: respuestas } } @@ -183,9 +179,8 @@ class QuestionGenerator { console.error(`Not enought data found to generate a question`); throw new Error(`Not enought data found to generate a question`); } - // console.log("\nFind:"); - // console.log(randomDocs); - var retQuestion = { + + const retQuestion = { pregunta: plantilla.pregunta(randomDocs[0][plantilla.campo_pregunta]), respuesta_correcta: randomDocs[0][plantilla.campo_respuesta], respuestas_incorrectas: Array.from({ length: respuestas-1 }, (_, i) => randomDocs[i+1][plantilla.campo_respuesta]) @@ -195,7 +190,7 @@ class QuestionGenerator { static async generateQuestions(preguntas, respuestas, temas) { const plantillasDisponibles = this.getAvailableTemplates(temas); - var retQuestions = []; + let retQuestions = []; for (let i = 0; i < preguntas; i++) { let index = Math.floor(Math.random() * plantillasDisponibles.length); let plantilla = this.plantillas[plantillasDisponibles[index]]; @@ -210,7 +205,7 @@ class QuestionGenerator { if (temas.length == 0) { return Array.from({ length: this.plantillas.length }, (_, i) => i); } - var templates = []; + let templates = []; console.log("Temas a utilizar:") temas.forEach(tema => { if (this.temas.has(tema)) { @@ -218,7 +213,7 @@ class QuestionGenerator { console.log(`\t${tema}`); } else { - console.error(`\tThe topic \'${tema}\' is not currently defined`); + console.error(`\tThe topic '${tema}' is not currently defined`); } }); if (templates.length == 0) { diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 2147ec08..1201b089 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -70,17 +70,16 @@ const templates = [ async function extractData(template) { console.log("Actualizando los datos sobre:") - var data = await template.extractMethod(); + const data = await template.extractMethod(); console.log(data); - var transactions = data.map(function (element) { - var transaction = { + const transactions = data.map(function (element) { + let transaction = { updateOne: { filter: template.filtro(element), update: template.campo_actualizar(element), upsert: true } }; - // console.log(transaction); return transaction; }); await template.saveMethod(transactions); @@ -90,7 +89,7 @@ async function extractData(template) { const minutes = 30; const totalQueries = templates.length; -var query = 0; +let query = 0; cron.schedule(`*/${minutes} * * * *`, () => { try { console.log(`Running a task every ${minutes} minutes: ${Date()}`); @@ -102,45 +101,6 @@ cron.schedule(`*/${minutes} * * * *`, () => { }); - /* - ALL ROUTES ARE ONLY FOR DEVELOPING PURPOSES, THEY SHOULD GET DELETED IN PRODUCTION - THIS SERVICE SHOULD NOT BE ACCESSIBLE FROM OUTSIDE - */ - -// Route for extracting countries -// app.get('/extract', async (req, res) => { -// try { -// res.json(await extractData(templates[1])); -// } catch (error) { -// console.error(error.message); -// res.status(500).json({ message: error.message }) -// // res.status(500).json({ error: 'Internal Server Error' }); -// } -// }); - -// // // Route for geting countries -// app.get('/countries', async (req, res) => { -// try { -// const paises = await Pais.find({}) -// res.json(paises); -// } catch (error) { -// res.status(500).json({ message: error.message }) -// // res.status(500).json({ error: 'Internal Server Error' }); -// } -// }); - -// // // Route for deleting countries -// app.delete('/countries', async (req, res) => { -// try { -// const paises = await Pais.deleteMany({}) -// res.json(paises); -// } catch (error) { -// res.status(500).json({ message: error.message }) -// // res.status(500).json({ error: 'Internal Server Error' }); -// } -// }); - - app.use((err, req, res, next) => { console.error(`An error occurred: ${err}`); res.status(500).send(`An error occurred: ${err.message}`); From a6f912c0a63ceff39032df60e9f2fe7e5595b8d9 Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 22:08:02 +0200 Subject: [PATCH 20/21] Extracted the identical data models from the two services to a common file in their parent directory and changed the way the containers are created in order to add that new common file into their respective container --- docker-compose.yml | 10 ++- ...nerator-model.js => questiondata-model.js} | 0 .../questiongeneratorservice/Dockerfile | 5 +- .../questiongenerator.js | 10 ++- questionsservice/wikidataExtractor/Dockerfile | 5 +- .../wikidataextractor-model.js | 82 ------------------- .../wikidataextractor-service.js | 5 +- 7 files changed, 24 insertions(+), 93 deletions(-) rename questionsservice/{questiongeneratorservice/questiongenerator-model.js => questiondata-model.js} (100%) delete mode 100644 questionsservice/wikidataExtractor/wikidataextractor-model.js diff --git a/docker-compose.yml b/docker-compose.yml index a0fbc49f..beaf8be1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,7 +74,9 @@ services: container_name: questiongeneratorservice-${teamname:-defaultASW} image: ghcr.io/arquisoft/wiq_es6c/questiongeneratorservice:latest profiles: ["dev", "prod"] - build: ./questionsservice/questiongeneratorservice + build: + context: ./questionsservice + dockerfile: ./questiongeneratorservice/Dockerfile depends_on: - mongodb_wiki - storequestionservice @@ -83,6 +85,7 @@ services: networks: - mynetwork environment: + DATAMODELS_URI: './questiondata-model' MONGODB_URI: mongodb://mongodb_wiki:27017/questions STORE_QUESTION_SERVICE_URL: http://storequestionservice:8004 restart: always @@ -91,7 +94,9 @@ services: container_name: wikidataextractorservice-${teamname:-defaultASW} image: ghcr.io/arquisoft/wiq_es6c/wikidataextractorservice:latest profiles: ["dev", "prod"] - build: ./questionsservice/wikidataExtractor + build: + context: ./questionsservice + dockerfile: ./wikidataExtractor/Dockerfile depends_on: - mongodb_wiki ports: @@ -99,6 +104,7 @@ services: networks: - mynetwork environment: + DATAMODELS_URI: './questiondata-model' MONGODB_URI: mongodb://mongodb_wiki:27017/questions restart: always diff --git a/questionsservice/questiongeneratorservice/questiongenerator-model.js b/questionsservice/questiondata-model.js similarity index 100% rename from questionsservice/questiongeneratorservice/questiongenerator-model.js rename to questionsservice/questiondata-model.js diff --git a/questionsservice/questiongeneratorservice/Dockerfile b/questionsservice/questiongeneratorservice/Dockerfile index d5f0af0b..1dac423d 100644 --- a/questionsservice/questiongeneratorservice/Dockerfile +++ b/questionsservice/questiongeneratorservice/Dockerfile @@ -5,13 +5,14 @@ FROM node:20 WORKDIR /usr/src/questionsservice/questiongeneratorservice # Copy package.json and package-lock.json to the working directory -COPY package*.json ./ +COPY ./questiongeneratorservice/package*.json ./ # Install app dependencies RUN npm install # Copy the app source code to the working directory -COPY . . +COPY ./questiongeneratorservice/ . +COPY questiondata-model.js . # Expose the port the app runs on EXPOSE 8007 diff --git a/questionsservice/questiongeneratorservice/questiongenerator.js b/questionsservice/questiongeneratorservice/questiongenerator.js index 73b4ce1c..6ce42012 100644 --- a/questionsservice/questiongeneratorservice/questiongenerator.js +++ b/questionsservice/questiongeneratorservice/questiongenerator.js @@ -1,4 +1,5 @@ -const { Pais, Monumento, Elemento, Pelicula, Cancion } = require('./questiongenerator-model') +const modelUri = process.env.DATAMODELS_URI || '../questiondata-model'; +const { Pais, Monumento, Elemento, Pelicula, Cancion } = require(modelUri); class QuestionGenerator { @@ -203,10 +204,13 @@ class QuestionGenerator { static getAvailableTemplates(temas) { if (temas.length == 0) { - return Array.from({ length: this.plantillas.length }, (_, i) => i); + let templates = Array.from({ length: this.plantillas.length }, (_, i) => i); + console.log("Temas a utilizar:\n\tTodos\nPlantillas a utilizar:"); + console.log(`\t${templates}`); + return templates; } let templates = []; - console.log("Temas a utilizar:") + console.log("Temas a utilizar:"); temas.forEach(tema => { if (this.temas.has(tema)) { templates = templates.concat(this.temas.get(tema)); diff --git a/questionsservice/wikidataExtractor/Dockerfile b/questionsservice/wikidataExtractor/Dockerfile index 0fa41b7a..0be0f3ef 100644 --- a/questionsservice/wikidataExtractor/Dockerfile +++ b/questionsservice/wikidataExtractor/Dockerfile @@ -5,13 +5,14 @@ FROM node:20 WORKDIR /usr/src/questionsservice/wikidataExtractor # Copy package.json and package-lock.json to the working directory -COPY package*.json ./ +COPY ./wikidataExtractor/package*.json ./ # Install app dependencies RUN npm install # Copy the app source code to the working directory -COPY . . +COPY ./wikidataExtractor/ . +COPY questiondata-model.js . # Expose the port the app runs on EXPOSE 8008 diff --git a/questionsservice/wikidataExtractor/wikidataextractor-model.js b/questionsservice/wikidataExtractor/wikidataextractor-model.js deleted file mode 100644 index 7d1d8351..00000000 --- a/questionsservice/wikidataExtractor/wikidataextractor-model.js +++ /dev/null @@ -1,82 +0,0 @@ -const mongoose = require('mongoose'); - -const paisSchema = new mongoose.Schema({ - pais: { - type: String, - required: true - }, - capital: { - type: String, - required: false - }, - continente: { - type: String, - required: false - }, - lenguaje: { - type: String, - required: false - }, - bandera: { - type: String, - required: false - } -}, { timestamps: {} }); // Añade y gestiona automáticamente los campos createdAt y updatedAt - -const monumentSchema = new mongoose.Schema({ - monumento: { - type: String, - required: true - }, - pais: { - type: String, - required: false - } -}, {timestamps: {}}); - -const chemicalElementsSchema = new mongoose.Schema({ - elemento: { - type: String, - required: true - }, - simbolo: { - type: String, - required: false - } -}, {timestamps: {}}); - -const filmSchema = new mongoose.Schema({ - pelicula: { - type: String, - required: true - }, - director: { - type: String, - required: false - } -}, {timestamps: {}}); - -const songSchema = new mongoose.Schema({ - cancion: { - type: String, - required: true - }, - artista: { - type: String, - required: false - } -}, {timestamps: {}}); - -const Pais = mongoose.model('Pais', paisSchema); -const Monumento = mongoose.model('Monumento', monumentSchema); -const Elemento = mongoose.model('Elemento', chemicalElementsSchema); -const Pelicula = mongoose.model('Pelicula', filmSchema); -const Cancion = mongoose.model('Cancion', songSchema); - -module.exports = { - Pais, - Monumento, - Elemento, - Pelicula, - Cancion -}; \ No newline at end of file diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 1201b089..5d7e632a 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -2,7 +2,8 @@ const express = require('express'); const cron = require('node-cron'); const mongoose = require('mongoose'); const WikiQueries = require('./wikidataQueries'); -const { Pais, Monumento, Elemento, Pelicula, Cancion } = require('./wikidataextractor-model'); +const modelUri = process.env.DATAMODELS_URI || '../questiondata-model'; +const { Pais, Monumento, Elemento, Pelicula, Cancion } = require(modelUri); const app = express(); const port = 8008; @@ -87,7 +88,7 @@ async function extractData(template) { return transactions; } -const minutes = 30; +const minutes = 1; const totalQueries = templates.length; let query = 0; cron.schedule(`*/${minutes} * * * *`, () => { From f3b4bfd67e71dd7505f1aa956d64862cfae6a400 Mon Sep 17 00:00:00 2001 From: Abel Date: Mon, 29 Apr 2024 22:12:16 +0200 Subject: [PATCH 21/21] Corrected the data extraction interval time for production --- questionsservice/wikidataExtractor/wikidataextractor-service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questionsservice/wikidataExtractor/wikidataextractor-service.js b/questionsservice/wikidataExtractor/wikidataextractor-service.js index 5d7e632a..b508e4ed 100644 --- a/questionsservice/wikidataExtractor/wikidataextractor-service.js +++ b/questionsservice/wikidataExtractor/wikidataextractor-service.js @@ -88,7 +88,7 @@ async function extractData(template) { return transactions; } -const minutes = 1; +const minutes = 30; const totalQueries = templates.length; let query = 0; cron.schedule(`*/${minutes} * * * *`, () => {