diff --git a/Gemfile b/Gemfile index 5e04bd4..81ee8ea 100644 --- a/Gemfile +++ b/Gemfile @@ -81,4 +81,6 @@ gem "sassc-rails" gem 'activeadmin' gem 'money-rails', '~> 1.12' -gem 'capitalize-names' \ No newline at end of file +gem 'capitalize-names' + +gem "chartkick" \ No newline at end of file diff --git a/Gemfile.lock b/Gemfile.lock index a175949..b1894de 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -89,6 +89,7 @@ GEM byebug (11.1.3) capitalize-names (1.2.0) cgi (0.3.6) + chartkick (5.0.7) coderay (1.1.3) concurrent-ruby (1.2.2) crass (1.0.6) @@ -318,6 +319,7 @@ DEPENDENCIES bootsnap bullet capitalize-names + chartkick debug devise dockerfile-rails (>= 1.2) diff --git a/app/controllers/lazy_load_groups_controller.rb b/app/controllers/lazy_load_groups_controller.rb index d2ea967..8499bc4 100644 --- a/app/controllers/lazy_load_groups_controller.rb +++ b/app/controllers/lazy_load_groups_controller.rb @@ -1,5 +1,68 @@ class LazyLoadGroupsController < ApplicationController def show @group = Group.find(params[:id]) + + # call the methods to save the instance variables + @giver_colors = colors(transfers_as_giver, giver: true) + @transfers_as_giver_by_year = group_by_year(transfers_as_giver) + @transfers_as_giver_by_name = group_by_name(transfers_as_giver, giver: true) + @taker_colors = colors(transfers_as_taker) + @transfers_as_taker_by_year = group_by_year(transfers_as_taker) + @transfers_as_taker_by_name = group_by_name(transfers_as_taker) + end + + def transfers_as_giver + @transfers_as_giver ||= Transfer.where(giver_type: @group.class.name, giver_id: @group.id) + end + + def transfers_as_taker + @transfers_as_taker ||= Transfer.where(taker_type: @group.class.name, taker_id: @group.id) + end + + def group_by_year(query) + query.group(:effective_date) + .sum(:amount) + .sort_by{|k, _v| k } + .to_h + .transform_keys{ |key| key.year } + end + + def group_by_name(query, giver: false) + if giver + all_the_groups = query.group(:taker_id, :taker_type) + .sum(:amount) + .transform_keys{ |key| key[1].constantize.find(key[0]).name } + .sort_by{|k, v| v} + else + all_the_groups = query.group(:giver_id, :giver_type) + .sum(:amount) + .transform_keys{ |key| key[1].constantize.find(key[0]).name } + .sort_by{|k, v| v} + end + + + last_five = all_the_groups.last(5) + sum_others = (all_the_groups - last_five).map{|a| a.last}.sum + + if sum_others.zero? + last_five.to_h + else + last_five.to_h.merge('Others' => sum_others).sort_by { |_k, value| value } + end + end + + def colors(query, giver: false) + if giver + query.group(:taker_id, :taker_type) + .sum(:amount) + .transform_keys{ |key| key[1].constantize.find(key[0]).name } + .map{|name, v| "#" + Digest::MD5.hexdigest(name)[0..5]} + else + query.group(:giver_id, :giver_type) + .sum(:amount) + .transform_keys{ |key| key[1].constantize.find(key[0]).name } + .map{|name, v| "#" + Digest::MD5.hexdigest(name)[0..5]} + end + end end \ No newline at end of file diff --git a/app/javascript/application.js b/app/javascript/application.js index 1a571e4..fbcbe60 100644 --- a/app/javascript/application.js +++ b/app/javascript/application.js @@ -4,3 +4,6 @@ import 'bootstrap' // Configure your import map in config/importmap.rb. Read more: https://github.com/rails/importmap-rails import "@hotwired/turbo-rails" import "controllers" + +import "chartkick" +import "Chart.bundle" \ No newline at end of file diff --git a/app/models/group.rb b/app/models/group.rb index 614bdfd..e853b35 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -71,7 +71,7 @@ class Group < ApplicationRecord # these are a bit weird, hence the transfers method below has_many :outgoing_transfers, class_name: 'Transfer', foreign_key: 'giver_id', as: :giver - has_many :incoming_transfers, class_name: 'Transfer', foreign_key: 'taker_id' + has_many :incoming_transfers, class_name: 'Transfer', foreign_key: 'taker_id', as: :taker accepts_nested_attributes_for :memberships, allow_destroy: true diff --git a/app/models/person.rb b/app/models/person.rb index d279536..2764f93 100644 --- a/app/models/person.rb +++ b/app/models/person.rb @@ -8,6 +8,7 @@ class Person < ApplicationRecord has_many :groups, through: :memberships has_many :outgoing_transfers, class_name: 'Transfer', foreign_key: 'giver_id', as: :giver + has_many :incoming_transfers, class_name: 'Transfer', foreign_key: 'taker_id', as: :taker accepts_nested_attributes_for :memberships, allow_destroy: true diff --git a/app/models/transfer.rb b/app/models/transfer.rb index 192e6c7..3b30693 100644 --- a/app/models/transfer.rb +++ b/app/models/transfer.rb @@ -2,12 +2,12 @@ class Transfer < ApplicationRecord belongs_to :giver, polymorphic: true # could be a Person or a Group - belongs_to :taker, class_name: 'Group' # Taker MUST also become polymorphic + belongs_to :taker, polymorphic: true # could be a Person or a Group validates :amount, presence: true validates :effective_date, presence: true - validates :giver, presence: true - validates :taker, presence: true + validates :giver_id, presence: true + validates :taker_id, presence: true validates :giver_type, uniqueness: { scope: [:giver_id, :taker_id, :amount, :effective_date], message: "should have unique combination of giver_type, giver_id, taker_id, amount, and effective_date" diff --git a/app/services/file_ingestor.rb b/app/services/file_ingestor.rb index 6a6eb59..11687a2 100644 --- a/app/services/file_ingestor.rb +++ b/app/services/file_ingestor.rb @@ -6,11 +6,15 @@ def annual_donor_ingest(file) csv.each do |row| donation_date = Date.new( "20#{row['Financial Year'].last(2)}".to_i, 6, 30) # saves bothering about the date format financial_year = Dates::FinancialYear.new(donation_date) + giver = RecordPersonOrGroup.call(row["Donor Name"]) + taker = RecordPersonOrGroup.call(row["Donation Made To"]) # TODO: Transfer.taker can be a group or a person. Use RecordGroup.call transfer = Transfer.find_or_create_by( - giver: RecordDonation.call(row["Donor Name"]), - taker: RecordGroup.call(row["Donation Made To"]), + giver_id: giver.id, + giver_type: giver.class.name, + taker_id: taker.id, + taker_type: taker.class.name, effective_date: financial_year.last_day, # group all donations for a financial year. There are too many otherwise. transfer_type: 'donations', evidence: 'https://transparency.aec.gov.au/AnnualDonor', @@ -20,9 +24,10 @@ def annual_donor_ingest(file) transfer.donations ||= [] # TODO: Transfer.taker can be a group or a person. Use RecordGroup.call + # this is for the JSON data field, recording each individual donation transfer.donations << { - giver: RecordDonation.call(row["Donor Name"])&.name, - taker: RecordGroup.call(row["Donation Made To"])&.name, + giver: giver.name, + taker: taker.name, effective_date: financial_year.last_day, donation_date: row['Date'], transfer_type: 'donation', @@ -82,7 +87,8 @@ def federal_parliamentarians_upload(file) # this is where we can add preselectors who choose the candidates for the electorate # for major parties the branch is a subgroup of the state party # for minor parties the branch is a subgroup of the federal party - unless senator + # add this back later if wanted + unless senator || true # it is the || true which prevents this from running. remove if needed branch_name = "#{federal_party.less_level} Branch for #{row['electorate']} Electorate" branch = RecordGroup.call(branch_name) @@ -101,14 +107,14 @@ def federal_parliamentarians_upload(file) member_type: "Group", member_id: branch.id, group: state_party, - ) unless senator + ) unless senator || true else # affiliate the branch with the federal party Membership.find_or_create_by( member_type: "Group", member_id: branch.id, group: federal_party, - ) unless senator + ) unless senator || true end end end @@ -120,7 +126,7 @@ def ministries_upload(file) ministry_group = RecordGroup.call(row['group']) person = RecordPerson.call(row['person']) - title = row['title'].strip + title = row['title'].strip if row['title'] start_date = parse_date(row['start_date']) end_date = parse_date(row['end_date']) @@ -134,7 +140,7 @@ def ministries_upload(file) Position.find_or_create_by(membership:, title:, start_date:, end_date:) rescue => e - p "Error: #{e}" + p "Error: #{e} | row#{row.inspect}" end end @@ -148,9 +154,9 @@ def general_upload(file) person = RecordPerson.call(row['person']) - title = row['title'].strip - start_date = parse_date(row['start_date']) - end_date = parse_date(row['end_date']) + title = row['title'].strip if row['title'].present? + start_date = parse_date(row['start_date']) if row['start_date'].present? + end_date = parse_date(row['end_date']) if row['end_date'].present? # the membership may not exist, if so, we need to create it membership = Membership.find_or_create_by( @@ -164,7 +170,7 @@ def general_upload(file) end rescue => e - p "Error: #{e}" + p "General Upload | Error: #{e} | row#{row.inspect}" end end diff --git a/app/services/map_group_names.rb b/app/services/map_group_names.rb index b55be1b..916720e 100644 --- a/app/services/map_group_names.rb +++ b/app/services/map_group_names.rb @@ -15,7 +15,6 @@ def map_or_return_name(name) return 'Get Up Limited' if name.match?(/(GetUp|Get Up)/i) return 'Australian Hotels Association' if name.match?(/Australian Hotels Association/i) return 'Advance Australia' if name.match?(/Advance Aus/i) - return 'Advance Australia' if name.match?(/Advanced Aus/i) return "It's Not a Race Limited" if name.match?(/(Not A Race|Note a Race)/i) return 'Australian Council of Trade Unions' if name.match?(/ACTU/i) return 'Climate 200' if name.match?(/(Climate 200|Climate200)/i) @@ -38,6 +37,7 @@ def map_or_return_name(name) return 'Katter Australia Party' if name.match?(/(Katter|KAP)/i) return 'Australian Conservatives' if name.match?(/Australian Conservatives/i) return 'Federal Independents' if name.match?(/Independent Fed/i) + return 'Waringah Independents' if name.match?(/[Warringah|Waringah].+independent/i) return 'Lambie Network' if name.match?(/Lambie/i) return "Pauline Hanson's One Nation" if name.match?(/Pauline Hanson|One Nation/i) @@ -139,7 +139,8 @@ def cleaned_up_name(name) regex_for_titleize = /\bPty\b|\bLtd\b|\bBus\b|\bInc\b|\bCo\b|\bTel\b|\bVan\b|\bAus\b/i regex_for_titleize_2 = /\bMud\b\bWeb\b|\bNow\b|\bNo\b|\bTen\b|Eli lilly\b|\bNew\b|\bJob\b/i regex_for_titleize_3 = /\bDot\b|\bRex\b|\bTan\b|\bUmi\b|\bBig\b|\bDr\b|\bGas\b/i - regex_for_titleize_4 = /\bTax\b|\bAid\b|\bBay\b|/i + regex_for_titleize_4 = /\bTax\b|\bAid\b|\bBay\b/i + regex_for_titleize_5 = /\bAmazon Web Services\b|\bAce Gutters\b/i regex_for_downcase = /\bthe\b|\bof\b|\band\b|\bas\b|\bfor\b/i @@ -150,6 +151,7 @@ def cleaned_up_name(name) .gsub(regex_for_titleize_2) { |word| word.titleize } .gsub(regex_for_titleize_3) { |word| word.titleize } .gsub(regex_for_titleize_4) { |word| word.titleize } + .gsub(regex_for_titleize_5) { |word| word.titleize } .gsub(regex_for_downcase) { |word| word.downcase } .gsub(/^the/) { |word| word.titleize } .gsub(/australia/) { |word| word.titleize } diff --git a/app/services/record_group.rb b/app/services/record_group.rb index 09d2cce..9963ec3 100644 --- a/app/services/record_group.rb +++ b/app/services/record_group.rb @@ -1,6 +1,4 @@ class RecordGroup - - attr_reader :name def initialize(name) diff --git a/app/services/record_person.rb b/app/services/record_person.rb index 615ecc0..e7c47af 100644 --- a/app/services/record_person.rb +++ b/app/services/record_person.rb @@ -19,14 +19,19 @@ def call def cleaned_up_name(name) regex_for_removal_elected = /\bMP\b|\bSenator\b/i - regex_for_removal_honours = /\bOAM\b|\bAO\b|\bAM\b/i + regex_for_removal_honours = /\bOAM\b|\bAO\b|\bAM\b|\bCSC\b|\bCBE\b/i regex_for_removal_titles = /\bQC\b|\bProf\b|\bDr\b/i + regex_for_removal_normal_titles = /\bMr\b|\bMs\b|\bMs\b|\bMiss\b/i name = CapitalizeNames.capitalize(name) + return 'David Pocock' if name.match?(/David Pocock/i) + return 'Nicholas Fairfax' if name.match?(/Nicholas John Fairfax/i) + name.gsub(regex_for_removal_elected, '') .gsub(regex_for_removal_honours, '') .gsub(regex_for_removal_titles, '') + .gsub(regex_for_removal_normal_titles, '') .strip end end diff --git a/app/services/record_person_or_group.rb b/app/services/record_person_or_group.rb new file mode 100644 index 0000000..3280c34 --- /dev/null +++ b/app/services/record_person_or_group.rb @@ -0,0 +1,88 @@ +class RecordPersonOrGroup + def self.call(name) + new(name).call + end + + def call + return nil unless name + + if person_or_group == 'person' + RecordPerson.call(first_name_last_name) + elsif person_or_group == 'group' + RecordGroup.call(name) + elsif person_or_group == 'couple' + RecordGroup.call(name) + # TODO: create memberships for each person in the couple + else + RecordGroup.call(name) + end + end + + private + + attr_reader :name + + def initialize(name) + @name = name.strip + end + + def person_or_group + regex_for_3_or_4_capitals = /HCF|INPEX|CMAX|SDA|ONA|SPP|ACCI|ACTU/i + regex_for_company_words_1 = /Corporation|Transport|Tax Aid|Outcomes|Lifestyle/i + regex_for_company_words_2 = /business|technology|shopping|toyota|bank|promotions|publications/i + regex_for_company_words_3 = /institute|horticultural|cleaning|technologies|centre/i + regex_for_company_words_4 = /Services|investments|entertainment|Insurance|Commerce/i + regex_for_company_words_5 = /Public|affairs|nimbin hemp|company|workpac|wren oil/i + regex_for_company_words_6 = /plumbing|division|federal|office|advisory|deloitte touche/i + regex_for_company_words_7 = /company|events|commerce|webdrill|private|restaurant/i + regex_for_company_words_8 = /enterprise|lendlease|party|healthcare|agency|team|lawyers/i + regex_for_company_words_9 = /national/i + + return 'group' if name.match?(regex_for_3_or_4_capitals) # Check for acronyms + return 'group' if name.match?(regex_for_company_words_1) # Check for company names + return 'group' if name.match?(regex_for_company_words_2) # Check for company names + return 'group' if name.match?(regex_for_company_words_3) # Check for company names + return 'group' if name.match?(regex_for_company_words_4) # Check for company names + return 'group' if name.match?(regex_for_company_words_5) # Check for company names + return 'group' if name.match?(regex_for_company_words_6) # Check for company names + return 'group' if name.match?(regex_for_company_words_7) # Check for company names + return 'group' if name.match?(regex_for_company_words_8) # Check for company names + return 'group' if name.match?(regex_for_company_words_9) # Check for company names + + + return 'group' if name.match?(/(PricewaterhouseCoopers|MSD)/) + return 'group' if name.match?(/Democratic Labour Party/i) + return 'group' if name.match?(/One Nation/i) + return 'group' if name.match?(/Kim For Canberra/i) + return 'group' if name.match?(/Get Up|Getup/i) + return 'group' if name.match?(/ALP-|ALP -|Alp Bruce Fea/i) + return 'group' if name.match?(/\bGrn\b/i) + return 'group' if name.match?(/\bKap\b/i) + return 'group' if name.match?(/\bWa-Alp\b/i) + return 'group' if name.match?(/ACP-VIC/i) + return 'group' if name.match?(/\bThe Nationals\b/i) + return 'group' if name.match?(/Independents/i) + return 'group' if name.match?(/Develco|Ecovis Clark Jacobs|Rapidplas|Rendition Homes/i) + return 'person' if name.match?(/(?:MP|OAM|AO)$/) # Check for individuals with MP or OAM + return 'person' if name.match?(/\bMP\b|\bDr\b/) # Check for individuals with MP or OAM + return 'group' if name.match?(/(limited|incorporated|ltd|government|associat|management|group|trust)/i) # Check for company names + return 'group' if name.match?(/(australia|management|capital|windfarm|engineering|energy)/i) # Check for company names + return 'group' if name.match?(/(guild|foundation|trust|retail|council|union|club|alliance)/i) # Check for company names + return 'group' if name.match?(/(nsw|queensland|state|tasmania|south|northern|territory|western)/i) # Check for states names + return 'group' if name.match?(/(n\.s\.w|qld|s\.a\.|n\.t\.|w\.a\.)/i) # Check for states abbreviations + return 'group' if name.match?(/( pl$|t\/as|trading as| p\/l)/i) # Check for company endings + return 'goup' if name.match?(/&|\(/) # Check for entries with ampersands (considered as companies) + return 'goup' if name.match?(/\d/) # Check for entries with numbers (considered as companies) + return 'couple' if name.match(/ and /) # Check for couples + return 'person' if name.match?(/^[A-Z][a-z]+, [A-Z][a-z]+$/) # Check for names in the format "Lastname, Firstname" + + 'person' # default + end + + def first_name_last_name + # handle last_name, first_name if in that format + name.include?(',') ? name.split(',').reverse.join(' ') : name + end +end + + diff --git a/app/views/application_view.rb b/app/views/application_view.rb index d64acf3..44856ea 100644 --- a/app/views/application_view.rb +++ b/app/views/application_view.rb @@ -33,8 +33,7 @@ def button_styles(instance, depth = 0) end def background_color(item) - background_color = safe_name(item) - "#" + text_to_hex(background_color) + "#" + text_to_hex(safe_name(item)) end def color(item) diff --git a/app/views/components/common/money_summary.rb b/app/views/components/common/money_summary.rb index b6ae19b..517c03c 100644 --- a/app/views/components/common/money_summary.rb +++ b/app/views/components/common/money_summary.rb @@ -3,7 +3,7 @@ class Common::MoneySummary < ApplicationView attr_reader :entity - def initialize(entity:) + def initialize(entity:) @entity = entity end @@ -15,24 +15,44 @@ def template div(class: 'col') do h3 { 'Money In' } p { money_in } + + # TODO: make this work for people as well + if money_in.present? && entity.is_a?(Group) # charts only work for groups for now (need to refactor those controllers) + turbo_frame(id: 'money_in_charts', src: lazy_load_group_path, loading: :lazy) do + p do + p { 'Loading Chart...'} + hr + end + end + end end div(class: 'col') do h3 { 'Money Out' } p { money_out } + + # TODO: make this work for people as well + if money_out.present? && entity.is_a?(Group) # charts only work for groups for now (need to refactor those controllers) + turbo_frame(id: 'money_out_charts', src: lazy_load_group_path, loading: :lazy) do + p do + p { 'Loading Chart...'} + hr + end + end + end end end end end def money_in - amount = Transfer.where(taker: entity).sum(:amount) + amount = Transfer.where(taker_type: entity.class.name, taker_id: entity.id).sum(:amount) return unless amount.positive? number_to_currency amount, precision: 0 end def money_out - amount = Transfer.where(giver: entity).sum(:amount) + amount = Transfer.where(giver_type: entity.class.name, giver_id: entity.id).sum(:amount) return unless amount.positive? number_to_currency amount, precision: 0 diff --git a/app/views/groups/show.html.erb b/app/views/groups/show.html.erb index 6568a46..7138697 100644 --- a/app/views/groups/show.html.erb +++ b/app/views/groups/show.html.erb @@ -1,4 +1,4 @@ -<%= turbo_stream_from(:updates) %> +
Loading More Transfer Records...
-