FP7_URLs

poltys · Feb 13, 2018 · 9370a1a · 9370a1a
1 parent 6144384
commit 9370a1a
Show file tree

Hide file tree

Showing 8 changed files with 4,048 additions and 1 deletion.
diff --git a/spiders/cordis_spider.py b/spiders/cordis_spider.py
@@ -17,7 +17,7 @@ class CordisSpider(scrapy.Spider):
     def parse(self, response):
         # Misconfiguration to check - eu in response.xpath not needed
         #for eu in response.xpath('//*[@id="container-pack"]'):
-        if response.xpath('//*[@id="ica:content"][contains(.,"water") and contains(.,"drinking water")]'):
+        if response.xpath('//*[@id="ica:content"][contains(.,"water") or contains(.,"drinking") or contains(.,"nitrates")]'):
             item = CordisItem()
             item['Meta'] = response.xpath('/html/head/meta[23]').extract()
             item['Project_ACR'] = response.xpath('//*[@id="dynamiccontent"]/div[1]/h1/text()').extract()

diff --git a/spiders/cordis_spider.pyc b/spiders/cordis_spider.pyc
diff --git a/spiders/dataset-textgenrnn.xml b/spiders/dataset-textgenrnn.xml
diff --git a/spiders/draft/urls.txt → spiders/urls.txt b/spiders/draft/urls.txt → spiders/urls.txt
diff --git a/wp_urls/FP7_ENERGY_URLs.csv b/wp_urls/FP7_ENERGY_URLs.csv
diff --git a/wp_urls/FP7_ENVIRONMENT_URLs.csv b/wp_urls/FP7_ENVIRONMENT_URLs.csv
diff --git a/wp_urls/FP7_ICT_URLs.csv b/wp_urls/FP7_ICT_URLs.csv
diff --git a/wp_urls/FP7_INFRASTRUCTURES_URLs.csv b/wp_urls/FP7_INFRASTRUCTURES_URLs.csv