diff --git a/data-raw/refresh_data.R b/data-raw/refresh_data.R index 55e0880f..43897e1c 100644 --- a/data-raw/refresh_data.R +++ b/data-raw/refresh_data.R @@ -459,166 +459,6 @@ abs_lfs <- abs_lfs %>% bind_rows(vac) -# Get JSA Internet vacancy index -tryCatch({ - - httr::set_config( - httr::user_agent( - "Mozilla/5.0 (Windows NT 5.1; rv:66.0) Gecko/20100101 Firefox/66.0" - ) - ) - - - ## GET IVI LINKS TO DATA - if (any(grepl('IVI', Sys.getenv()))) { - - # GET LINKS FROM ENV (GITHUB ACTIONS) - ivi_link_region = Sys.getenv('IVI_REGIONS') - ivi_link_anzsco4 <- Sys.getenv('IVI_ANZSCO4') - ivi_link_skills <- Sys.getenv('IVI_SKILLS') - - } else { - - # GET LINKS FROM WEBSITE (LOCAL MACHINE) - links <- read_html("https://www.jobsandskills.gov.au/work/internet-vacancy-index") %>% - html_elements("a.downloadLink") %>% - html_attr("href") %>% - stringr::str_subset("xlsx|XLSX") %>% - stringr::str_subset("(?i)anzsco4|region|skill") %>% - paste0("https://www.jobsandskills.gov.au", .) - - for (link in links) { - name <- case_when( - grepl('region', link, ignore.case = TRUE) ~ 'region', - grepl('anzsco4', link, ignore.case = TRUE) ~ 'anzsco4', - grepl('skill', link, ignore.case = TRUE) ~ 'skills' - ) - - assign( - x = paste0('ivi_link_', name), - value = link - ) - - } - - } - - - ## IVI REGIONS - # Link for regional ivi information - - stopifnot(tools::file_ext(ivi_link_region) %in% c("xlsx", "XLSX")) - - - # Download regional ivi to temporary diretory - ivi_region_tmp_xlsx <- tempfile(fileext = ".xlsx") - #download.file(ivi_link_region, ivi_region_tmp_xlsx, mode = "wb") - resp <- httr::GET(ivi_link_region, - httr::write_disk(ivi_region_tmp_xlsx, overwrite=TRUE), - httr::config(verbose=TRUE, http_version=1, forbid_reuse=F)) - status <- httr::http_status(resp) - print(status) - - # Clean and save regional ivi information - ivi_region <- readxl::read_excel(ivi_region_tmp_xlsx, sheet = "Averaged") %>% - filter(State == 'VIC') %>% - unite(series, Level, State, region, ANZSCO_CODE, ANZSCO_TITLE) %>% - pivot_longer( - -series, - names_to = "date", - values_to = "value" - ) %>% - mutate( - date = as.Date(as.numeric(date), origin = "1899-12-30"), - series_id = sapply(series, rlang::hash), - value = as.numeric(value), - table_no = "ivi_region", - series_type = "Original", - data_type = "FLOW", - frequency = "Month", - unit = "Advertisements" - ) - - - abs_lfs <- abs_lfs %>% - bind_rows(ivi_region) - - - ## IVI ANZSCO4 - # Get-you a link for anzsco 4 IVI - - stopifnot(tools::file_ext(ivi_link_anzsco4) %in% c("xlsx", "XLSX")) - - # Download anzsco 4 ivi to temporary diretory - ivi_anzsco4_tmp_xlsx <- tempfile(fileext = ".xlsx") - download.file(ivi_link_anzsco4, ivi_anzsco4_tmp_xlsx, mode = "wb") - - - # Clean and save anzsco 4 ivi information - ivi_anzsco4 <- readxl::read_excel( - ivi_anzsco4_tmp_xlsx, - sheet = "4 digit 3 month average") %>% - filter(state == 'VIC') %>% - unite(series, state, ANZSCO_CODE, ANZSCO_TITLE) %>% - pivot_longer( - -series, - names_to = "date", - values_to = "value" - ) %>% - mutate( - date = as.Date(as.numeric(date), origin = "1899-12-30"), - series_id = sapply(series, rlang::hash), - value = as.numeric(value), - table_no = "ivi_anzsco4", - series_type = "Original", - data_type = "FLOW", - frequency = "Month", - unit = "Advertisements" - ) - - abs_lfs <- abs_lfs %>% - bind_rows(ivi_anzsco4) - - - ### IVI SKILLS - - # Download ivi skills to temporary diretory - ivi_skills_tmp_xlsx <- tempfile(fileext = ".xlsx") - download.file(ivi_link_skills, ivi_skills_tmp_xlsx, mode = "wb") - - skillz <- readxl::read_excel(ivi_skills_tmp_xlsx, sheet = "Seasonally Adjusted") %>% - filter(State == "VIC") %>% - unite(series, Level, State, Title, Skill_level) %>% - pivot_longer( - -series, - names_to = "date", - values_to = "value" - ) %>% - mutate(date = as.Date(as.numeric(date), origin = "1899-12-30"), - series_id = sapply(series, rlang::hash), - value = as.numeric(value), - table_no = "ivi_skills", - series_type = "Original", - data_type = "FLOW", - frequency = "Month", - unit = "Advertisements") - - abs_lfs <- abs_lfs %>% - bind_rows(skillz) - - -}, -error = function(e){ - abs_lfs <- abs_lfs %>% - bind_rows( - old_data %>% - filter(table_no %in% c("ivi_anzsco4", "ivi_region", "ivi_skills")) - ) - - message("IVI data did not parse. Original error:\n", e) - } -) - # Check if data updated -----