Update Saarland Scraper

Gemfile

@@ -47,6 +47,7 @@ group :test do
   gem 'minitest', '5.10.1'
   gem 'minitest-stub_any_instance'
   gem 'rails-controller-testing', '~> 1.0.1'
+  gem 'webmock'
 end
 
 # Get i18n files

Gemfile.lock

@@ -65,6 +65,8 @@ GEM
     coffee-script-source (1.12.2)
     concurrent-ruby (1.1.4)
     connection_pool (2.2.2)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     crass (1.0.4)
     css_parser (1.6.0)
       addressable
@@ -148,6 +150,7 @@ GEM
       rails (>= 3.2.0)
     griddler-sendgrid (1.0.1)
       griddler
+    hashdiff (0.3.8)
     hashids (1.0.5)
     hashie (3.6.0)
     htmlentities (4.3.4)
@@ -290,6 +293,7 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rubyzip (1.2.2)
+    safe_yaml (1.0.4)
     sass (3.7.3)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
@@ -349,6 +353,10 @@ GEM
       activemodel (>= 5.0)
       bindex (>= 0.4.0)
       railties (>= 5.0)
+    webmock (3.5.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
     webrobots (0.1.2)
     websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
@@ -418,6 +426,7 @@ DEPENDENCIES
   typhoeus (~> 1.3.0)
   uglifier (>= 1.3.0)
   web-console (~> 3.0)
+  webmock
   wikidata-client (~> 0.0.10)
 
 RUBY VERSION

app/scrapers/saarland_scraper.rb

@@ -1,32 +1,54 @@
 require 'date'
 
-#
-# Saarland is based on SharePoint. (No idea why anyone thinks thats a sane idea)
-#
-# Get Version: https://www.landtag-saar.de/_vti_pvt/service.cnf
-# # vti_extenderversion:SR|15.0.0.4797
-#
-# An request to https://www.landtag-saar.de/_vti_bin/ brings us the following header:
-# # MicrosoftSharePointTeamServices:15.0.0.4569
-#
-# So it is an SharePoint 2013.
-# But all the interesting endpoints (/_api, /_vti_bin/listdata.svc/) are locked down.
-# Sad :(
-#
-# Update 2017-01-10: They had the clever idea to pack the data as json into one big
-# hidden input field.
-#
 module SaarlandScraper
   BASE_URL = 'https://www.landtag-saar.de'
-  OVERVIEW_URL = BASE_URL + '/dokumente/drucksachen'
+  OVERVIEW_URL = BASE_URL + '/umbraco/aawSearchSurfaceController/SearchSurface/GetSearchResults/'
 
   class Detail < DetailScraper
     def scrape
-      mp = mechanize.get OVERVIEW_URL
-      entry = SaarlandScraper.extract_entries(mp).find { |e| e['Dokumentnummer'].strip == full_reference }
+      mp = load_detail_page(mechanize, full_reference)
+      entry = SaarlandScraper.extract_entries(mp).find { |e| e['DocumentNumber'].strip == full_reference }
       return nil if entry.nil?
       SaarlandScraper.extract_paper(entry)
     end
+
+    def load_detail_page(mechanize_agent, full_reference)
+      headers = {"Accept" => "application/json, text/javascript, */*; q=0.01"}
+      body = {
+        "Filter" => {
+          "Periods" => []
+        },
+        "Pageination" => {
+          "Skip" => 0,
+          "Take" => 100
+        },
+        "Sections" => {
+          "Print" => true,
+          "PlenaryProtocol" => false,
+          "Law" => false,
+          "PublicConsultation" => false
+        },
+        "Sort" => {
+          "SortType" => 0,
+          "SortValue" => 0
+        },
+        "OnlyTitle" => false,
+        "Value" => "",
+        "CurrentSearchTab" => 1,
+        "KendoFilter" => {
+          "filters" => [
+            {
+              "field" => "DocumentNumber",
+              "operator" => "startswith",
+              "value" => full_reference
+            }
+          ],
+          "logic" => "and"
+        }
+      }.to_json
+
+      mechanize_agent.post(OVERVIEW_URL, body, headers).body
+    end
   end
 
   class Overview < Scraper
@@ -38,7 +60,7 @@ def scrape(&block)
       @m ||= mechanize
       papers = []
       streaming = block_given?
-      mp = @m.get OVERVIEW_URL
+      mp = load_overview_page(@m, @legislative_term)
       SaarlandScraper.extract_entries(mp).each do |entry|
         begin
           paper = SaarlandScraper.extract_paper(entry)
@@ -55,32 +77,75 @@ def scrape(&block)
       end
       papers unless streaming
     end
+
+    private
+
+    def load_overview_page(mechanize_agent, term)
+      headers = {"Accept" => "application/json, text/javascript, */*; q=0.01"}
+      body = {
+        "Filter" => {
+          "Periods" => []
+        },
+        "Pageination" => {
+          "Skip" => 0,
+          "Take" => 100
+        },
+        "Sections" => {
+          "Print" => true,
+          "PlenaryProtocol" => false,
+          "Law" => false,
+          "PublicConsultation" => false
+        },
+        "Sort" => {
+          "SortType" => 0,
+          "SortValue" => 0
+        },
+        "OnlyTitle" => false,
+        "Value" => "",
+        "CurrentSearchTab" => 1,
+        "KendoFilter" => {
+          "filters" => [
+            {
+              "logic" => "or",
+              "filters" => [
+                { "field" => "Legislative", "operator" => "eq", "value" => term.to_s }
+              ]
+            }
+          ],
+          "logic" => "and"
+        },
+        "KendoSort" => [
+          {"field" => "PublicDate", "dir" => "desc"}
+        ]
+      }.to_json
+
+      mechanize_agent.post(OVERVIEW_URL, body, headers).body
+    end
   end
 
   def self.extract_entries(mp)
-    v = mp.search('//input[@type="hidden"]')
-      .find { |i| i.attributes['name'].to_s.ends_with? '$documentListInput' }
-      .attributes['value']
-    v = v.to_s.gsub(/\&qu?o?t?quot;/, '"')
-    JSON.parse(v)
+    JSON.parse(mp)["FilteredResult"]
   end
 
   def self.extract_doc_link(entry)
-    Addressable::URI.parse(BASE_URL).join(entry['URL']).normalize.to_s
+    Addressable::URI.parse(BASE_URL).join(entry['FilePath']).normalize.to_s
   end
 
   def self.extract_date(entry)
-    Date.parse(entry['Dokumentdatum'])
+    dotnet_serialized_date = entry['PublicDate']
+    seconds_since_epoch = dotnet_serialized_date.scan(/[0-9]+/)[0].to_i / 1000.0
+
+    Time.at(seconds_since_epoch).utc.to_date
   end
 
   def self.extract_title(entry)
-    entry['Titel'].strip
+    entry['Title'].strip
   end
 
   def self.extract_paper(entry)
     return nil if entry.nil? || !extract_is_answer(entry)
     url = extract_doc_link(entry)
-    full_reference = entry['Dokumentnummer']
+    full_reference = entry['DocumentNumber']
     reference = full_reference.split('/').last
     legislative_term = full_reference.split('/').first
     title = extract_title(entry)
@@ -101,6 +166,6 @@ def self.extract_paper(entry)
   end
 
   def self.extract_is_answer(entry)
-    entry['Dokumentname'].starts_with? 'Aw'
+    entry['FileName'].starts_with? 'Aw'
   end
-end
+end