diff --git a/build/referendum/4/supporting/index.json b/build/referendum/4/supporting/index.json index 83610c7e8..76f0b5b22 100644 --- a/build/referendum/4/supporting/index.json +++ b/build/referendum/4/supporting/index.json @@ -10,7 +10,7 @@ "id": 2, "name": "Causa Justa :: Just Cause (nonprofit 501(c)(3))", "payee": "Causa Justa :: Just Cause (nonprofit 501(c)(3))", - "amount": 33930.57 + "amount": 94470.86 } ], "total_contributions": 456, diff --git a/calculators/candidate_expenditures_by_type.rb b/calculators/candidate_expenditures_by_type.rb index c053f8dba..eaba65f83 100644 --- a/calculators/candidate_expenditures_by_type.rb +++ b/calculators/candidate_expenditures_by_type.rb @@ -65,7 +65,21 @@ def expenditures_by_candidate_by_type ORDER BY "Expn_Code", "Filer_ID", "Report_Num" DESC SQL - results.each do |result| + # 497 does not contain "Expn_Code" making this calculator pretty useless + # for those contributions. + # To make the numbers line up closer, we'll bucket those all under "Not + # Stated". + late_expenditures = ActiveRecord::Base.connection.execute(<<-SQL) + SELECT DISTINCT ON ("Filer_ID") + "Filer_ID", '' AS "Expn_Code", SUM("Amount") AS "Total" + FROM "efile_COAK_2016_497" + WHERE "Filer_ID" IN ('#{@candidates_by_filer_id.keys.join "','"}') + AND "Form_Type" = 'F497P2' + GROUP BY "Filer_ID", "Report_Num" + ORDER BY "Filer_ID", "Report_Num" DESC + SQL + + (results.to_a + late_expenditures.to_a).each do |result| hash[result['Filer_ID']] ||= {} hash[result['Filer_ID']][result['Expn_Code']] = result['Total'] end diff --git a/calculators/referendum_supporters_calculator.rb b/calculators/referendum_supporters_calculator.rb index 42cc79ea3..52436abe9 100644 --- a/calculators/referendum_supporters_calculator.rb +++ b/calculators/referendum_supporters_calculator.rb @@ -16,6 +16,16 @@ def fetch ORDER BY "Filer_ID", "Filer_NamL", "Bal_Name", "Sup_Opp_Cd", "Report_Num" DESC SQL + late_expenditures = ActiveRecord::Base.connection.execute(<<-SQL) + SELECT DISTINCT ON ("Filer_ID", "Filer_NamL", "Bal_Name") + "Filer_ID", "Filer_NamL", "Bal_Name", SUM("Amount") AS "Total_Amount" + FROM "efile_COAK_2016_497" + WHERE "Bal_Name" IS NOT NULL + AND "Form_Type" = 'F497P2' + GROUP BY "Filer_ID", "Filer_NamL", "Bal_Name", "Report_Num" + ORDER BY "Filer_ID", "Filer_NamL", "Bal_Name", "Report_Num" DESC + SQL + supporting_by_measure_name = {} opposing_by_measure_name = {} @@ -29,6 +39,35 @@ def fetch end end + late_expenditures.each do |row| + sup_opp_cd = guess_whether_committee_supports_measure(row['Filer_ID'], row['Bal_Name']) + if sup_opp_cd == 'S' + supporting_by_measure_name[row['Bal_Name']] ||= [] + existing_idx = supporting_by_measure_name[row['Bal_Name']].find_index do |existing_row| + existing_row['Filer_ID'].to_s == row['Filer_ID'].to_s + end + + if existing_idx + supporting_by_measure_name[row['Bal_Name']][existing_idx]['Total_Amount'] += + row['Total_Amount'] + else + supporting_by_measure_name[row['Bal_Name']] << row + end + elsif sup_opp_cd == 'O' + opposing_by_measure_name[row['Bal_Name']] ||= [] + existing_idx = opposing_by_measure_name[row['Bal_Name']].find_index do |existing_row| + existing_row['Filer_ID'].to_s == row['Filer_ID'].to_s + end + + if existing_idx + opposing_by_measure_name[row['Bal_Name']][existing_idx]['Total_Amount'] += + row['Total_Amount'] + else + opposing_by_measure_name[row['Bal_Name']] << row + end + end + end + [ # { bal_name => rows } , calculation name [supporting_by_measure_name, :supporting_organizations], @@ -77,4 +116,29 @@ def committee_from_expenditure(expenditure) committee end + + # Form 497 Page 2 (Late Expenditures) includes the ballot measure name and + # committee ID, but does not indicate whether that expenditure was in support + # or opposition of the ballot measure. + # + # This is not perfect, but it should get us pretty close. + def guess_whether_committee_supports_measure(committee_id, bal_name) + @_guess_cache ||= + begin + guesses = ActiveRecord::Base.connection.execute(<<-SQL) + SELECT "Filer_ID", "Bal_Name", "Sup_Opp_Cd" + FROM "efile_COAK_2016_E-Expenditure" + WHERE "Bal_Name" IS NOT NULL + GROUP BY "Filer_ID", "Bal_Name", "Sup_Opp_Cd" + SQL + + guesses.index_by do |row| + row.values_at('Filer_ID', 'Bal_Name').map(&:to_s) + end + end + + if row = @_guess_cache[[committee_id.to_s, bal_name]] + row['Sup_Opp_Cd'] + end + end end diff --git a/calculators/total_expenditures_calculator.rb b/calculators/total_expenditures_calculator.rb index b884da9d4..0c195fe40 100644 --- a/calculators/total_expenditures_calculator.rb +++ b/calculators/total_expenditures_calculator.rb @@ -5,7 +5,7 @@ def initialize(candidates: [], ballot_measures: [], committees: []) end def fetch - @results = ActiveRecord::Base.connection.execute <<-SQL + results = ActiveRecord::Base.connection.execute <<-SQL SELECT DISTINCT ON ("Filer_ID", "Amount_A") "Filer_ID", "Amount_A" FROM "efile_COAK_2016_Summary" @@ -16,7 +16,17 @@ def fetch ORDER BY "Filer_ID", "Amount_A", "Report_Num" DESC SQL - @results.each do |result| + late_expenditures = ActiveRecord::Base.connection.execute <<-SQL + SELECT DISTINCT ON ("Filer_ID") + "Filer_ID", SUM("Amount") AS "Amount_A" + FROM "efile_COAK_2016_497" + WHERE "Filer_ID" IN ('#{@candidates_by_filer_id.keys.join "', '"}') + AND "Form_Type" = 'F497P2' + GROUP BY "Filer_ID", "Report_Num" + ORDER BY "Filer_ID", "Report_Num" DESC + SQL + + (results.to_a + late_expenditures.to_a).each do |result| candidate = @candidates_by_filer_id[result['Filer_ID'].to_i] candidate.save_calculation(:total_expenditures, result['Amount_A']) end