Skip to content

Commit

Permalink
Merge pull request #10 from johnofsydney/june-new-version
Browse files Browse the repository at this point in the history
June new version
  • Loading branch information
johnofsydney authored Jun 14, 2024
2 parents a2d5b89 + 9b6e154 commit b092555
Show file tree
Hide file tree
Showing 23 changed files with 255 additions and 45 deletions.
4 changes: 3 additions & 1 deletion Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -81,4 +81,6 @@ gem "sassc-rails"
gem 'activeadmin'

gem 'money-rails', '~> 1.12'
gem 'capitalize-names'
gem 'capitalize-names'

gem "chartkick"
2 changes: 2 additions & 0 deletions Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ GEM
byebug (11.1.3)
capitalize-names (1.2.0)
cgi (0.3.6)
chartkick (5.0.7)
coderay (1.1.3)
concurrent-ruby (1.2.2)
crass (1.0.6)
Expand Down Expand Up @@ -318,6 +319,7 @@ DEPENDENCIES
bootsnap
bullet
capitalize-names
chartkick
debug
devise
dockerfile-rails (>= 1.2)
Expand Down
63 changes: 63 additions & 0 deletions app/controllers/lazy_load_groups_controller.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,68 @@
class LazyLoadGroupsController < ApplicationController
def show
@group = Group.find(params[:id])

# call the methods to save the instance variables
@giver_colors = colors(transfers_as_giver, giver: true)
@transfers_as_giver_by_year = group_by_year(transfers_as_giver)
@transfers_as_giver_by_name = group_by_name(transfers_as_giver, giver: true)
@taker_colors = colors(transfers_as_taker)
@transfers_as_taker_by_year = group_by_year(transfers_as_taker)
@transfers_as_taker_by_name = group_by_name(transfers_as_taker)
end

def transfers_as_giver
@transfers_as_giver ||= Transfer.where(giver_type: @group.class.name, giver_id: @group.id)
end

def transfers_as_taker
@transfers_as_taker ||= Transfer.where(taker_type: @group.class.name, taker_id: @group.id)
end

def group_by_year(query)
query.group(:effective_date)
.sum(:amount)
.sort_by{|k, _v| k }
.to_h
.transform_keys{ |key| key.year }
end

def group_by_name(query, giver: false)
if giver
all_the_groups = query.group(:taker_id, :taker_type)
.sum(:amount)
.transform_keys{ |key| key[1].constantize.find(key[0]).name }
.sort_by{|k, v| v}
else
all_the_groups = query.group(:giver_id, :giver_type)
.sum(:amount)
.transform_keys{ |key| key[1].constantize.find(key[0]).name }
.sort_by{|k, v| v}
end


last_five = all_the_groups.last(5)
sum_others = (all_the_groups - last_five).map{|a| a.last}.sum

if sum_others.zero?
last_five.to_h
else
last_five.to_h.merge('Others' => sum_others).sort_by { |_k, value| value }
end
end

def colors(query, giver: false)
if giver
query.group(:taker_id, :taker_type)
.sum(:amount)
.transform_keys{ |key| key[1].constantize.find(key[0]).name }
.map{|name, v| "#" + Digest::MD5.hexdigest(name)[0..5]}
else
query.group(:giver_id, :giver_type)
.sum(:amount)
.transform_keys{ |key| key[1].constantize.find(key[0]).name }
.map{|name, v| "#" + Digest::MD5.hexdigest(name)[0..5]}
end

end
end
3 changes: 3 additions & 0 deletions app/javascript/application.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@ import 'bootstrap'
// Configure your import map in config/importmap.rb. Read more: https://github.com/rails/importmap-rails
import "@hotwired/turbo-rails"
import "controllers"

import "chartkick"
import "Chart.bundle"
2 changes: 1 addition & 1 deletion app/models/group.rb
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ class Group < ApplicationRecord

# these are a bit weird, hence the transfers method below
has_many :outgoing_transfers, class_name: 'Transfer', foreign_key: 'giver_id', as: :giver
has_many :incoming_transfers, class_name: 'Transfer', foreign_key: 'taker_id'
has_many :incoming_transfers, class_name: 'Transfer', foreign_key: 'taker_id', as: :taker

accepts_nested_attributes_for :memberships, allow_destroy: true

Expand Down
1 change: 1 addition & 0 deletions app/models/person.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ class Person < ApplicationRecord
has_many :groups, through: :memberships

has_many :outgoing_transfers, class_name: 'Transfer', foreign_key: 'giver_id', as: :giver
has_many :incoming_transfers, class_name: 'Transfer', foreign_key: 'taker_id', as: :taker

accepts_nested_attributes_for :memberships, allow_destroy: true

Expand Down
6 changes: 3 additions & 3 deletions app/models/transfer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@

class Transfer < ApplicationRecord
belongs_to :giver, polymorphic: true # could be a Person or a Group
belongs_to :taker, class_name: 'Group' # Taker MUST also become polymorphic
belongs_to :taker, polymorphic: true # could be a Person or a Group

validates :amount, presence: true
validates :effective_date, presence: true
validates :giver, presence: true
validates :taker, presence: true
validates :giver_id, presence: true
validates :taker_id, presence: true
validates :giver_type, uniqueness: {
scope: [:giver_id, :taker_id, :amount, :effective_date],
message: "should have unique combination of giver_type, giver_id, taker_id, amount, and effective_date"
Expand Down
32 changes: 19 additions & 13 deletions app/services/file_ingestor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,15 @@ def annual_donor_ingest(file)
csv.each do |row|
donation_date = Date.new( "20#{row['Financial Year'].last(2)}".to_i, 6, 30) # saves bothering about the date format
financial_year = Dates::FinancialYear.new(donation_date)
giver = RecordPersonOrGroup.call(row["Donor Name"])
taker = RecordPersonOrGroup.call(row["Donation Made To"])

# TODO: Transfer.taker can be a group or a person. Use RecordGroup.call
transfer = Transfer.find_or_create_by(
giver: RecordDonation.call(row["Donor Name"]),
taker: RecordGroup.call(row["Donation Made To"]),
giver_id: giver.id,
giver_type: giver.class.name,
taker_id: taker.id,
taker_type: taker.class.name,
effective_date: financial_year.last_day, # group all donations for a financial year. There are too many otherwise.
transfer_type: 'donations',
evidence: 'https://transparency.aec.gov.au/AnnualDonor',
Expand All @@ -20,9 +24,10 @@ def annual_donor_ingest(file)
transfer.donations ||= []

# TODO: Transfer.taker can be a group or a person. Use RecordGroup.call
# this is for the JSON data field, recording each individual donation
transfer.donations << {
giver: RecordDonation.call(row["Donor Name"])&.name,
taker: RecordGroup.call(row["Donation Made To"])&.name,
giver: giver.name,
taker: taker.name,
effective_date: financial_year.last_day,
donation_date: row['Date'],
transfer_type: 'donation',
Expand Down Expand Up @@ -82,7 +87,8 @@ def federal_parliamentarians_upload(file)
# this is where we can add preselectors who choose the candidates for the electorate
# for major parties the branch is a subgroup of the state party
# for minor parties the branch is a subgroup of the federal party
unless senator
# add this back later if wanted
unless senator || true # it is the || true which prevents this from running. remove if needed
branch_name = "#{federal_party.less_level} Branch for #{row['electorate']} Electorate"
branch = RecordGroup.call(branch_name)

Expand All @@ -101,14 +107,14 @@ def federal_parliamentarians_upload(file)
member_type: "Group",
member_id: branch.id,
group: state_party,
) unless senator
) unless senator || true
else
# affiliate the branch with the federal party
Membership.find_or_create_by(
member_type: "Group",
member_id: branch.id,
group: federal_party,
) unless senator
) unless senator || true
end
end
end
Expand All @@ -120,7 +126,7 @@ def ministries_upload(file)
ministry_group = RecordGroup.call(row['group'])

person = RecordPerson.call(row['person'])
title = row['title'].strip
title = row['title'].strip if row['title']
start_date = parse_date(row['start_date'])
end_date = parse_date(row['end_date'])

Expand All @@ -134,7 +140,7 @@ def ministries_upload(file)
Position.find_or_create_by(membership:, title:, start_date:, end_date:)
rescue => e

p "Error: #{e}"
p "Error: #{e} | row#{row.inspect}"
end
end

Expand All @@ -148,9 +154,9 @@ def general_upload(file)

person = RecordPerson.call(row['person'])

title = row['title'].strip
start_date = parse_date(row['start_date'])
end_date = parse_date(row['end_date'])
title = row['title'].strip if row['title'].present?
start_date = parse_date(row['start_date']) if row['start_date'].present?
end_date = parse_date(row['end_date']) if row['end_date'].present?

# the membership may not exist, if so, we need to create it
membership = Membership.find_or_create_by(
Expand All @@ -164,7 +170,7 @@ def general_upload(file)
end
rescue => e

p "Error: #{e}"
p "General Upload | Error: #{e} | row#{row.inspect}"
end
end

Expand Down
6 changes: 4 additions & 2 deletions app/services/map_group_names.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ def map_or_return_name(name)
return 'Get Up Limited' if name.match?(/(GetUp|Get Up)/i)
return 'Australian Hotels Association' if name.match?(/Australian Hotels Association/i)
return 'Advance Australia' if name.match?(/Advance Aus/i)
return 'Advance Australia' if name.match?(/Advanced Aus/i)
return "It's Not a Race Limited" if name.match?(/(Not A Race|Note a Race)/i)
return 'Australian Council of Trade Unions' if name.match?(/ACTU/i)
return 'Climate 200' if name.match?(/(Climate 200|Climate200)/i)
Expand All @@ -38,6 +37,7 @@ def map_or_return_name(name)
return 'Katter Australia Party' if name.match?(/(Katter|KAP)/i)
return 'Australian Conservatives' if name.match?(/Australian Conservatives/i)
return 'Federal Independents' if name.match?(/Independent Fed/i)
return 'Waringah Independents' if name.match?(/[Warringah|Waringah].+independent/i)
return 'Lambie Network' if name.match?(/Lambie/i)

return "Pauline Hanson's One Nation" if name.match?(/Pauline Hanson|One Nation/i)
Expand Down Expand Up @@ -139,7 +139,8 @@ def cleaned_up_name(name)
regex_for_titleize = /\bPty\b|\bLtd\b|\bBus\b|\bInc\b|\bCo\b|\bTel\b|\bVan\b|\bAus\b/i
regex_for_titleize_2 = /\bMud\b\bWeb\b|\bNow\b|\bNo\b|\bTen\b|Eli lilly\b|\bNew\b|\bJob\b/i
regex_for_titleize_3 = /\bDot\b|\bRex\b|\bTan\b|\bUmi\b|\bBig\b|\bDr\b|\bGas\b/i
regex_for_titleize_4 = /\bTax\b|\bAid\b|\bBay\b|/i
regex_for_titleize_4 = /\bTax\b|\bAid\b|\bBay\b/i
regex_for_titleize_5 = /\bAmazon Web Services\b|\bAce Gutters\b/i

regex_for_downcase = /\bthe\b|\bof\b|\band\b|\bas\b|\bfor\b/i

Expand All @@ -150,6 +151,7 @@ def cleaned_up_name(name)
.gsub(regex_for_titleize_2) { |word| word.titleize }
.gsub(regex_for_titleize_3) { |word| word.titleize }
.gsub(regex_for_titleize_4) { |word| word.titleize }
.gsub(regex_for_titleize_5) { |word| word.titleize }
.gsub(regex_for_downcase) { |word| word.downcase }
.gsub(/^the/) { |word| word.titleize }
.gsub(/australia/) { |word| word.titleize }
Expand Down
2 changes: 0 additions & 2 deletions app/services/record_group.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
class RecordGroup


attr_reader :name

def initialize(name)
Expand Down
7 changes: 6 additions & 1 deletion app/services/record_person.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,19 @@ def call

def cleaned_up_name(name)
regex_for_removal_elected = /\bMP\b|\bSenator\b/i
regex_for_removal_honours = /\bOAM\b|\bAO\b|\bAM\b/i
regex_for_removal_honours = /\bOAM\b|\bAO\b|\bAM\b|\bCSC\b|\bCBE\b/i
regex_for_removal_titles = /\bQC\b|\bProf\b|\bDr\b/i
regex_for_removal_normal_titles = /\bMr\b|\bMs\b|\bMs\b|\bMiss\b/i

name = CapitalizeNames.capitalize(name)

return 'David Pocock' if name.match?(/David Pocock/i)
return 'Nicholas Fairfax' if name.match?(/Nicholas John Fairfax/i)

name.gsub(regex_for_removal_elected, '')
.gsub(regex_for_removal_honours, '')
.gsub(regex_for_removal_titles, '')
.gsub(regex_for_removal_normal_titles, '')
.strip
end
end
88 changes: 88 additions & 0 deletions app/services/record_person_or_group.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
class RecordPersonOrGroup
def self.call(name)
new(name).call
end

def call
return nil unless name

if person_or_group == 'person'
RecordPerson.call(first_name_last_name)
elsif person_or_group == 'group'
RecordGroup.call(name)
elsif person_or_group == 'couple'
RecordGroup.call(name)
# TODO: create memberships for each person in the couple
else
RecordGroup.call(name)
end
end

private

attr_reader :name

def initialize(name)
@name = name.strip
end

def person_or_group
regex_for_3_or_4_capitals = /HCF|INPEX|CMAX|SDA|ONA|SPP|ACCI|ACTU/i
regex_for_company_words_1 = /Corporation|Transport|Tax Aid|Outcomes|Lifestyle/i
regex_for_company_words_2 = /business|technology|shopping|toyota|bank|promotions|publications/i
regex_for_company_words_3 = /institute|horticultural|cleaning|technologies|centre/i
regex_for_company_words_4 = /Services|investments|entertainment|Insurance|Commerce/i
regex_for_company_words_5 = /Public|affairs|nimbin hemp|company|workpac|wren oil/i
regex_for_company_words_6 = /plumbing|division|federal|office|advisory|deloitte touche/i
regex_for_company_words_7 = /company|events|commerce|webdrill|private|restaurant/i
regex_for_company_words_8 = /enterprise|lendlease|party|healthcare|agency|team|lawyers/i
regex_for_company_words_9 = /national/i

return 'group' if name.match?(regex_for_3_or_4_capitals) # Check for acronyms
return 'group' if name.match?(regex_for_company_words_1) # Check for company names
return 'group' if name.match?(regex_for_company_words_2) # Check for company names
return 'group' if name.match?(regex_for_company_words_3) # Check for company names
return 'group' if name.match?(regex_for_company_words_4) # Check for company names
return 'group' if name.match?(regex_for_company_words_5) # Check for company names
return 'group' if name.match?(regex_for_company_words_6) # Check for company names
return 'group' if name.match?(regex_for_company_words_7) # Check for company names
return 'group' if name.match?(regex_for_company_words_8) # Check for company names
return 'group' if name.match?(regex_for_company_words_9) # Check for company names


return 'group' if name.match?(/(PricewaterhouseCoopers|MSD)/)
return 'group' if name.match?(/Democratic Labour Party/i)
return 'group' if name.match?(/One Nation/i)
return 'group' if name.match?(/Kim For Canberra/i)
return 'group' if name.match?(/Get Up|Getup/i)
return 'group' if name.match?(/ALP-|ALP -|Alp Bruce Fea/i)
return 'group' if name.match?(/\bGrn\b/i)
return 'group' if name.match?(/\bKap\b/i)
return 'group' if name.match?(/\bWa-Alp\b/i)
return 'group' if name.match?(/ACP-VIC/i)
return 'group' if name.match?(/\bThe Nationals\b/i)
return 'group' if name.match?(/Independents/i)
return 'group' if name.match?(/Develco|Ecovis Clark Jacobs|Rapidplas|Rendition Homes/i)
return 'person' if name.match?(/(?:MP|OAM|AO)$/) # Check for individuals with MP or OAM
return 'person' if name.match?(/\bMP\b|\bDr\b/) # Check for individuals with MP or OAM
return 'group' if name.match?(/(limited|incorporated|ltd|government|associat|management|group|trust)/i) # Check for company names
return 'group' if name.match?(/(australia|management|capital|windfarm|engineering|energy)/i) # Check for company names
return 'group' if name.match?(/(guild|foundation|trust|retail|council|union|club|alliance)/i) # Check for company names
return 'group' if name.match?(/(nsw|queensland|state|tasmania|south|northern|territory|western)/i) # Check for states names
return 'group' if name.match?(/(n\.s\.w|qld|s\.a\.|n\.t\.|w\.a\.)/i) # Check for states abbreviations
return 'group' if name.match?(/( pl$|t\/as|trading as| p\/l)/i) # Check for company endings
return 'goup' if name.match?(/&|\(/) # Check for entries with ampersands (considered as companies)
return 'goup' if name.match?(/\d/) # Check for entries with numbers (considered as companies)
return 'couple' if name.match(/ and /) # Check for couples
return 'person' if name.match?(/^[A-Z][a-z]+, [A-Z][a-z]+$/) # Check for names in the format "Lastname, Firstname"

'person' # default
end

def first_name_last_name
# handle last_name, first_name if in that format
name.include?(',') ? name.split(',').reverse.join(' ') : name
end
end


3 changes: 1 addition & 2 deletions app/views/application_view.rb
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,7 @@ def button_styles(instance, depth = 0)
end

def background_color(item)
background_color = safe_name(item)
"#" + text_to_hex(background_color)
"#" + text_to_hex(safe_name(item))
end

def color(item)
Expand Down
Loading

0 comments on commit b092555

Please sign in to comment.