From 716e1e2dea4d2ebcfbee402980c59daa43df8aaa Mon Sep 17 00:00:00 2001 From: Blake Astley <46544374+astley92@users.noreply.github.com> Date: Wed, 2 Oct 2024 10:13:17 +1000 Subject: [PATCH] Add rake task to sync using AusPayNet API (#61) * Get something working * End to end test * Split bsb tasks into two tasks * Small random cleanup * Generate latest update json * Rubocop * Use LEADER_WIDTH to calc outputparam width * Extract output param width * Add README section * Misc cleanup * Fixup README * Add unit test for aus pay net api client * fix: update bank list Signed-off-by: David Looi * fix: typo Signed-off-by: David Looi --------- Signed-off-by: David Looi Co-authored-by: David Looi --- Gemfile | 3 + README.md | 41 ++++- Rakefile | 1 + lib/bsb.rb | 4 +- lib/bsb/aus_pay_net/client.rb | 37 +++++ lib/bsb/base_generator.rb | 2 + lib/bsb/database_generator.rb | 20 +++ lib/tasks/bsb_tasks.rake | 28 ++-- lib/tasks/sync_bsb_db.rake | 34 +++++ test/bsb/aus_pay_net/client_test.rb | 19 +++ test/fixtures/bsb_db.json | 20 +++ .../auspaynet_fetch_all_bsbs.yml | 80 ++++++++++ test/tasks/sync_bsb_db_test.rb | 141 ++++++++++++++++++ test/test_helper.rb | 14 ++ test/tmp/.keep | 0 15 files changed, 428 insertions(+), 16 deletions(-) create mode 100644 lib/bsb/aus_pay_net/client.rb create mode 100644 lib/tasks/sync_bsb_db.rake create mode 100644 test/bsb/aus_pay_net/client_test.rb create mode 100644 test/fixtures/bsb_db.json create mode 100644 test/fixtures/vcr_cassettes/auspaynet_fetch_all_bsbs.yml create mode 100644 test/tasks/sync_bsb_db_test.rb create mode 100644 test/tmp/.keep diff --git a/Gemfile b/Gemfile index 99aebac..8881305 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,9 @@ source 'https://rubygems.org' gemspec gem 'bundler', '~> 2.0' +gem 'faraday' +gem 'minitest-stub-const' gem 'net-ftp', '~> 0.1.3' gem 'rake', '~> 13.0' gem 'rubocop' +gem 'vcr' diff --git a/README.md b/README.md index 9cb2ec5..21eb4fd 100644 --- a/README.md +++ b/README.md @@ -68,18 +68,53 @@ Two data sources are used: Other formats of APCA BSB data is available from http://bsb.apca.com.au. -## Update source +## Update BSB Bank List At the moment BSB data is a manual download from the Auspaynet site [here](https://bsb.auspaynet.com.au/). -You will need to download two files, place them in `tmp/`: +You will need to download the Key to Abbreviations and BSB Number file and place it in `tmp/`: - `Reference Documents` button > `Key to Abbreviations and BSB Number` in CSV format. + +Run the sync task with the files to complete sync of the latest data: + +```sh +rake bsb:sync_bank_list['tmp/key to abbreviations and bsb numbers (august 2024).csv'] +``` + +Browse the list of database changes, make a few queries on the website to ensure the results are the same. + +## Update BSB DB + +`config/bsb_db.json` can be updated by running the `bsb:sync_bsb_db` rake task. +This task depends on you having you having an Aus Pay Net API subscription and key and that key being available in the +`AUSPAYNET_SUB_KEY` environment variable. + +You can apply for an API subscription and key by visiting [AusPayNet's bsb page](https://bsb.auspaynet.com.au/), +clicking the `API Registration` button and following the prompts. + +Once you have a key, the task can be run as follows + +```sh +AUSPAYNET_SUB_KEY="your_key_here" rake bsb:sync_bsb_db +``` + +This will update the `config/bsb_db.json` file with the latest information and will produce a +`config/latest_update.json` file that contains a breakdown of additions, deletions and modifications to make spot +checking results easier. + +Browse the list of database changes, make a few queries on the website to ensure the results are the same. + +## Update BSB DB (Manual) + +BSB DB data can be downloaded manually from the Auspaynet site [here](https://bsb.auspaynet.com.au/). + +You will need to download the BSB directory and place it in `tmp/`: - `Download BSB Files` button > `BSB Directory (Full Version)` in TEXT format. Run the sync task with the files to complete sync of the latest data: ```sh -rake bsb:sync['tmp/key to abbreviations and bsb numbers (august 2024).csv','tmp/BSBDirectoryAug24-341.txt'] +rake bsb:sync_bsb_db_manual['tmp/BSBDirectoryAug24-341.txt'] ``` Browse the list of database changes, make a few queries on the website to ensure the results are the same. diff --git a/Rakefile b/Rakefile index b153386..46e3323 100644 --- a/Rakefile +++ b/Rakefile @@ -2,6 +2,7 @@ require 'bundler/gem_tasks' load 'lib/tasks/bsb_tasks.rake' +load 'lib/tasks/sync_bsb_db.rake' require 'rake/testtask' diff --git a/lib/bsb.rb b/lib/bsb.rb index ed36ed4..360256a 100644 --- a/lib/bsb.rb +++ b/lib/bsb.rb @@ -5,6 +5,8 @@ require 'bsb_number_validator' module BSB + DB_FILEPATH = 'config/bsb_db.json' + CHANGES_FILEPATH = 'config/latest_update.json' class << self def lookup(number) bsb = normalize(number) @@ -42,7 +44,7 @@ def normalize(str) protected def data_hash - @data_hash ||= JSON.parse(File.read(File.expand_path('../config/bsb_db.json', __dir__))) + @data_hash ||= JSON.parse(File.read(File.expand_path("../#{DB_FILEPATH}", __dir__))) end def bank_list diff --git a/lib/bsb/aus_pay_net/client.rb b/lib/bsb/aus_pay_net/client.rb new file mode 100644 index 0000000..cfd385c --- /dev/null +++ b/lib/bsb/aus_pay_net/client.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module BSB + module AusPayNet + module Client + class MissingSubscriptionKeyError < StandardError; end + + OUTPUT_PARAM_WIDTH = 30 + LEADER_WIDTH = OUTPUT_PARAM_WIDTH + 11 + + Response = Struct.new(:body, keyword_init: true) + + def self.fetch_all_bsbs + subscription_key = ENV.fetch('AUSPAYNET_SUB_KEY', nil) + if subscription_key.nil? + raise MissingSubscriptionKeyError, "the environment variable 'AUSPAYNET_SUB_KEY' must be present" + end + + conn = Faraday.new( + url: 'https://auspaynet-bicbsb-api-prod.azure-api.net', + headers: { + 'Content-Type': 'application/json', + 'Ocp-Apim-Subscription-Key': subscription_key + } + ) do |faraday| + faraday.response :raise_error + end + + response = conn.post('/bsbquery/manual/paths/invoke') do |req| + req.body = { outputparam: ' ' * OUTPUT_PARAM_WIDTH }.to_json + end + + Response.new(body: response.body[LEADER_WIDTH..]) + end + end + end +end diff --git a/lib/bsb/base_generator.rb b/lib/bsb/base_generator.rb index 67d7357..46bd34a 100644 --- a/lib/bsb/base_generator.rb +++ b/lib/bsb/base_generator.rb @@ -4,6 +4,8 @@ module BSB class BaseGenerator + attr_reader :hash + def initialize(hash) @hash = hash end diff --git a/lib/bsb/database_generator.rb b/lib/bsb/database_generator.rb index 55505c3..43d6c15 100644 --- a/lib/bsb/database_generator.rb +++ b/lib/bsb/database_generator.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'bsb/base_generator' +require 'bsb/aus_pay_net/client' module BSB class DatabaseGenerator < BaseGenerator @@ -16,5 +17,24 @@ def self.load_file(filename) end new(hash) end + + def self.fetch_latest + response = BSB::AusPayNet::Client.fetch_all_bsbs + + hash = {} + JSON.parse(response.body).each do |bsb_config| + bsb = bsb_config.fetch('BSBCode').delete('-') + hash[bsb] = [ + bsb_config.fetch('FiMnemonic'), + bsb_config.fetch('BSBName'), + bsb_config.fetch('Address'), + bsb_config.fetch('Suburb'), + bsb_config.fetch('State'), + bsb_config.fetch('Postcode'), + 'PEH'.chars.map { bsb_config.fetch('StreamCode').include?(_1) ? _1 : ' ' }.join + ] + end + new(hash) + end end end diff --git a/lib/tasks/bsb_tasks.rake b/lib/tasks/bsb_tasks.rake index 706b35d..21ecc26 100644 --- a/lib/tasks/bsb_tasks.rake +++ b/lib/tasks/bsb_tasks.rake @@ -1,26 +1,30 @@ # frozen_string_literal: true +require 'bsb' +require 'bsb/database_generator' +require 'bsb/bank_list_generator' + namespace :bsb do desc 'Sync config/*.json.' - task :sync, [:keyfile, :bsbfile] do |_t, args| - require 'bsb/base_generator' - bank_list_filename = args[:keyfile] + task :sync_bsb_db_manual, [:bsbfile] do |_t, args| db_list_filename = args[:bsbfile] + if db_list_filename + bsb_db_gen = BSB::DatabaseGenerator.load_file(db_list_filename) + File.write(BSB::DB_FILEPATH, bsb_db_gen.json) + else + warn 'Missing bsb db "BSBDirectory"' + end + end + + task :sync_bank_list, [:keyfile] do |_t, args| + bank_list_filename = args[:keyfile] + if bank_list_filename - require 'bsb/bank_list_generator' bsb_bl_gen = BSB::BankListGenerator.load_file(bank_list_filename) File.write('config/bsb_bank_list.json', bsb_bl_gen.json) else warn 'Missing bank list "KEY TO ABBREVIATIONS AND BSB NUMBERS"' end - - if db_list_filename - require 'bsb/database_generator' - bsb_db_gen = BSB::DatabaseGenerator.load_file(db_list_filename) - File.write('config/bsb_db.json', bsb_db_gen.json) - else - warn 'Missing bsb db "BSBDirectory"' - end end end diff --git a/lib/tasks/sync_bsb_db.rake b/lib/tasks/sync_bsb_db.rake new file mode 100644 index 0000000..c1d1971 --- /dev/null +++ b/lib/tasks/sync_bsb_db.rake @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'bsb' +require 'bsb/database_generator' + +namespace :bsb do + desc 'Sync config/bsb_db.json with data provided by AusPayNet' + task :sync_bsb_db do + latest_db = BSB::DatabaseGenerator.fetch_latest + existing_db_hash = JSON.parse(File.read(BSB::DB_FILEPATH)) + latest_db_hash = latest_db.hash + + deletions = existing_db_hash.reject { |bsb, _| latest_db_hash.key?(bsb) } + additions = latest_db_hash.reject { |bsb, _| existing_db_hash.key?(bsb) } + modifications = {} + + latest_db_hash.each do |bsb, data| + next unless existing_db_hash.key?(bsb) && existing_db_hash[bsb] != data + + modifications[bsb] = data + end + + changes_json = JSON.pretty_generate( + { + additions: additions, + deletions: deletions, + modifications: modifications + } + ) + + File.write(BSB::DB_FILEPATH, latest_db.json) + File.write(BSB::CHANGES_FILEPATH, changes_json) + end +end diff --git a/test/bsb/aus_pay_net/client_test.rb b/test/bsb/aus_pay_net/client_test.rb new file mode 100644 index 0000000..419f68d --- /dev/null +++ b/test/bsb/aus_pay_net/client_test.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'bsb/aus_pay_net/client' +require 'test_helper' + +describe BSB::AusPayNet::Client do + describe '.fetch_all_bsbs' do + before { ENV['AUSPAYNET_SUB_KEY'] = 'something' } + + it 'returns the expected response' do + VCR.use_cassette('auspaynet_fetch_all_bsbs') do + response = BSB::AusPayNet::Client.fetch_all_bsbs + assert_equal(response.class, BSB::AusPayNet::Client::Response) + assert_equal(response.body.class, String) + assert_equal(JSON.parse(response.body).count, 2) + end + end + end +end diff --git a/test/fixtures/bsb_db.json b/test/fixtures/bsb_db.json new file mode 100644 index 0000000..d36e9c8 --- /dev/null +++ b/test/fixtures/bsb_db.json @@ -0,0 +1,20 @@ +{ + "333333": [ + "AAA", + "Aviato3", + "123 Fakest Street", + "Canberra", + "ACT", + "1234", + "P H" + ], + "987654": [ + "EST", + "Aviato2", + "123 Old Faker Street", + "Ballina", + "NSW", + "1234", + "P " + ] +} diff --git a/test/fixtures/vcr_cassettes/auspaynet_fetch_all_bsbs.yml b/test/fixtures/vcr_cassettes/auspaynet_fetch_all_bsbs.yml new file mode 100644 index 0000000..e4262e5 --- /dev/null +++ b/test/fixtures/vcr_cassettes/auspaynet_fetch_all_bsbs.yml @@ -0,0 +1,80 @@ +--- +http_interactions: +- request: + method: post + uri: https://auspaynet-bicbsb-api-prod.azure-api.net/bsbquery/manual/paths/invoke + body: + encoding: UTF-8 + string: '{"outputparam":" "}' + headers: + Content-type: + - application/json + Ocp-apim-subscription-key: + - "" + User-Agent: + - Faraday v2.10.1 + response: + status: + code: 200 + message: OK + headers: + cache-control: + - no-cache + pragma: + - no-cache + transfer-encoding: + - chunked + content-type: + - text/plain; charset=utf-8 + content-encoding: + - gzip + expires: + - "-1" + vary: + - Accept-Encoding + set-cookie: + - ARRAffinity=0aa69915266871205a67096b40953eafb333722c9d662666b4ee1cbd3af96c28;Path=/;HttpOnly;Secure;Domain=prod-07.australiasoutheast.logic.azure.com, + ARRAffinitySameSite=0aa69915266871205a67096b40953eafb333722c9d662666b4ee1cbd3af96c28;Path=/;HttpOnly;SameSite=None;Secure;Domain=prod-07.australiasoutheast.logic.azure.com + strict-transport-security: + - max-age=31536000; includeSubDomains + x-ms-workflow-run-id: + - '08584759810780029358383571724CU01' + x-ms-correlation-id: + - 8d349630-46fe-44ae-9d3f-0dc53071b48e + x-ms-client-tracking-id: + - '08584759810780029358383571724CU01' + x-ms-trigger-history-name: + - '08584759810780029358383571724CU01' + x-ms-execution-location: + - australiasoutheast + x-ms-workflow-system-id: + - "/locations/australiasoutheast/scaleunits/prod-07/workflows/55b07bc2b01c48cea922b9673bb16c54" + x-ms-workflow-id: + - 55b07bc2b01c48cea922b9673bb16c54 + x-ms-workflow-version: + - '08584793971872858901' + x-ms-workflow-name: + - logicapp-all-wildcardquery + x-ms-tracking-id: + - 8d349630-46fe-44ae-9d3f-0dc53071b48e + x-ms-ratelimit-burst-remaining-workflow-writes: + - '2999' + x-ms-ratelimit-remaining-workflow-download-contentsize: + - '210888995' + x-ms-ratelimit-remaining-workflow-upload-contentsize: + - '214748112' + x-ms-ratelimit-time-remaining-directapirequests: + - '19998471' + x-ms-request-id: + - australiasoutheast:8d349630-46fe-44ae-9d3f-0dc53071b48e + request-context: + - appId=cid-v1:62fce8e1-7550-4725-83ec-f11232c5768f + date: + - Fri, 06 Sep 2024 12:30:08 GMT + body: + encoding: UTF-8 + string: '{"outputparam":"Query results returned "}[{"BSBCode":"012-002","BSBName":"ANZ + Smart Choice","FiMnemonic":"ANZ","Address":"115 Pitt Street","Suburb":"Sydney","State":"NSW","Postcode":"2000","StreamCode":"PEH","lastmodified":null,"BIC":"ANZBAU3R","BICINT":"","repair":"00"},{"BSBCode":"012-003","BSBName":"Merged","FiMnemonic":"ANZ","Address":"Refer + to BSB 012-019","Suburb":"Sydney","State":"NSW","Postcode":"2000","StreamCode":"PEH","lastmodified":null,"BIC":"ANZBAU3R","BICINT":"","repair":"00"}]' + recorded_at: Fri, 06 Sep 2024 12:30:09 GMT +recorded_with: VCR 6.3.1 diff --git a/test/tasks/sync_bsb_db_test.rb b/test/tasks/sync_bsb_db_test.rb new file mode 100644 index 0000000..a800ba5 --- /dev/null +++ b/test/tasks/sync_bsb_db_test.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'rake' +require 'bsb/database_generator' + +describe 'sync_bsb_db rake task' do # rubocop:disable Metrics/BlockLength + before do + ENV.update('AUSPAYNET_SUB_KEY' => 'something') + Rake.application.rake_require('../lib/tasks/sync_bsb_db') + Rake::Task['bsb:sync_bsb_db'].reenable + File.write('test/tmp/bsb_db.json', File.read('test/fixtures/bsb_db.json')) + end + + let(:auspaynet_bsb_client_response) do + BSB::AusPayNet::Client::Response.new( + body: JSON.dump( + [ + { + BSBCode: '123-456', + BSBName: 'Aviato', + FiMnemonic: 'TST', + Address: '123 Fake Street', + Suburb: 'Dubbo', + State: 'NSW', + Postcode: '1234', + StreamCode: 'EH', + lastmodified: nil, + BIC: 'TSTAAU2SSYD', + BICINT: '', + repair: '00' + }, + { + BSBCode: '987-654', + BSBName: 'Aviato2', + FiMnemonic: 'EST', + Address: '123 Faker Street', + Suburb: 'Ballina', + State: 'NSW', + Postcode: '1234', + StreamCode: 'P', + lastmodified: nil, + BIC: 'TSTAAU2SSYD', + BICINT: '', + repair: '00' + } + ] + ) + ) + end + + let(:expected_db) do + JSON.pretty_generate( + { + '123456': [ + 'TST', + 'Aviato', + '123 Fake Street', + 'Dubbo', + 'NSW', + '1234', + ' EH' + ], + '987654': [ + 'EST', + 'Aviato2', + '123 Faker Street', + 'Ballina', + 'NSW', + '1234', + 'P ' + ] + } + ) + end + + let(:expected_changes) do + JSON.pretty_generate( + { + additions: { + '123456': [ + 'TST', + 'Aviato', + '123 Fake Street', + 'Dubbo', + 'NSW', + '1234', + ' EH' + ] + }, + deletions: { + '333333': [ + 'AAA', + 'Aviato3', + '123 Fakest Street', + 'Canberra', + 'ACT', + '1234', + 'P H' + ] + }, + modifications: { + '987654': [ + 'EST', + 'Aviato2', + '123 Faker Street', + 'Ballina', + 'NSW', + '1234', + 'P ' + ] + } + } + ) + end + + it 'generates the expected bsb_db and changes file' do + BSB.stub_consts(DB_FILEPATH: 'test/tmp/bsb_db.json', CHANGES_FILEPATH: 'test/tmp/latest_update.json') do + BSB::AusPayNet::Client.stub(:fetch_all_bsbs, auspaynet_bsb_client_response) do + Rake::Task['bsb:sync_bsb_db'].invoke + end + + resultant_db = File.read(BSB::DB_FILEPATH).strip + resultant_changes = File.read(BSB::CHANGES_FILEPATH).strip + assert_equal(resultant_db, expected_db) + assert_equal(resultant_changes, expected_changes) + end + end + + describe 'when the AUSPAYNET_SUB_KEY env var is not set' do + before do + ENV.delete('AUSPAYNET_SUB_KEY') + end + + it 'raises the expected error' do + assert_raises BSB::AusPayNet::Client::MissingSubscriptionKeyError do + Rake::Task['bsb:sync_bsb_db'].invoke + end + end + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index 808cb99..3766383 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -2,6 +2,20 @@ require 'bsb' require 'minitest/autorun' +require 'minitest/stub_const' +require 'vcr' + +Minitest.after_run do + Dir.glob('test/tmp/**/*.json').each { File.delete(_1) } +end + +VCR.configure do |config| + config.cassette_library_dir = 'test/fixtures/vcr_cassettes' + config.hook_into :faraday + config.filter_sensitive_data('') do |interaction| + interaction.request.headers['Ocp-apim-subscription-key'][0] + end +end class Account include ActiveModel::API diff --git a/test/tmp/.keep b/test/tmp/.keep new file mode 100644 index 0000000..e69de29