diff --git a/datasets/census_bureau_acs/infra/cbsa_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/cbsa_2019_1yr_pipeline.tf index aebf40c1a..2e44be364 100644 --- a/datasets/census_bureau_acs/infra/cbsa_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/cbsa_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_cbsa_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "cbsa_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2019_1yr" description = "CBSA 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/cbsa_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/cbsa_2019_5yr_pipeline.tf index 4dc30e1cf..0f2d07dce 100644 --- a/datasets/census_bureau_acs/infra/cbsa_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/cbsa_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_cbsa_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "cbsa_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2019_5yr" description = "CBSA 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/census_bureau_acs_dataset.tf b/datasets/census_bureau_acs/infra/census_bureau_acs_dataset.tf index bd914ce61..21be9523e 100644 --- a/datasets/census_bureau_acs/infra/census_bureau_acs_dataset.tf +++ b/datasets/census_bureau_acs/infra/census_bureau_acs_dataset.tf @@ -24,19 +24,3 @@ resource "google_bigquery_dataset" "census_bureau_acs" { output "bigquery_dataset-census_bureau_acs-dataset_id" { value = google_bigquery_dataset.census_bureau_acs.dataset_id } - -resource "google_storage_bucket" "census-bureau-acs" { - name = "${var.bucket_name_prefix}-census-bureau-acs" - force_destroy = true - location = "US" - uniform_bucket_level_access = true - lifecycle { - ignore_changes = [ - logging, - ] - } -} - -output "storage_bucket-census-bureau-acs-name" { - value = google_storage_bucket.census-bureau-acs.name -} diff --git a/datasets/census_bureau_acs/infra/census_bureau_acs_pipeline.tf b/datasets/census_bureau_acs/infra/census_bureau_acs_pipeline.tf new file mode 100644 index 000000000..b4f714a5e --- /dev/null +++ b/datasets/census_bureau_acs/infra/census_bureau_acs_pipeline.tf @@ -0,0 +1,1420 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +resource "google_bigquery_table" "census_bureau_acs_cbsa_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2019_1yr" + description = "CBSA 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_cbsa_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_cbsa_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_cbsa_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2020_1yr" + description = "CBSA 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_cbsa_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_cbsa_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_cbsa_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2021_1yr" + description = "CBSA 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_cbsa_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_cbsa_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_cbsa_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2019_5yr" + description = "CBSA 2019 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_cbsa_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_cbsa_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_cbsa_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2020_5yr" + description = "CBSA 2020 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_cbsa_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_cbsa_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_cbsa_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "cbsa_2021_5yr" + description = "CBSA 2021 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_cbsa_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_cbsa_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_cbsa_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_censustract_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "censustract_2019_5yr" + description = "Census tract 2019 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_censustract_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_censustract_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_censustract_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_censustract_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_censustract_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "censustract_2020_5yr" + description = "Census tract 2020 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_censustract_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_censustract_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_censustract_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_censustract_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_censustract_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "censustract_2021_5yr" + description = "Census tract 2021 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_censustract_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_censustract_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_censustract_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_censustract_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2019_1yr" + description = "Congressional district 2019 1 year table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2020_1yr" + description = "Congressional district 2020 1 year table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2021_1yr" + description = "Congressional district 2021 1 year table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2019_5yr" + description = "Congressional district 2019 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2020_5yr" + description = "Congressional district 2020 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2020_5yr" + description = "Congressional district 2020 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_congressionaldistrict_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_congressionaldistrict_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_county_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2019_1yr" + description = "County 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_county_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_county_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_county_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_county_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_county_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2020_1yr" + description = "County 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_county_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_county_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_county_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_county_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_county_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2021_1yr" + description = "County 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_county_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_county_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_county_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_county_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_county_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2019_5yr" + description = "County 2019 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_county_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_county_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_county_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_county_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_county_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2020_5yr" + description = "County 2020 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_county_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_county_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_county_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_county_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_county_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2021_5yr" + description = "County 2021 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_county_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_county_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_county_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_county_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_place_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2019_1yr" + description = "Place 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_place_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_place_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_place_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_place_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_place_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2020_1yr" + description = "Place 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_place_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_place_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_place_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_place_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_place_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2021_1yr" + description = "Place 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_place_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_place_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_place_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_place_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_place_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2019_5yr" + description = "Place 2019 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_place_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_place_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_place_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_place_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_place_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2020_5yr" + description = "Place 2020 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_place_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_place_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_place_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_place_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_place_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2021_5yr" + description = "Place 2021 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_place_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_place_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_place_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_place_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_puma_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2019_1yr" + description = "PUMA 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_puma_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_puma_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_puma_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_puma_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_puma_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2020_1yr" + description = "PUMA 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_puma_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_puma_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_puma_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_puma_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_puma_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2021_1yr" + description = "PUMA 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_puma_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_puma_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_puma_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_puma_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_puma_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2019_5yr" + description = "PUMA 2019 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_puma_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_puma_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_puma_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_puma_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_puma_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2020_5yr" + description = "PUMA 2020 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_puma_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_puma_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_puma_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_puma_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_puma_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2021_5yr" + description = "PUMA 2021 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_puma_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_puma_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_puma_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_puma_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2016_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2016_1yr" + description = "School district elementary 2016 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2016_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2016_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2016_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2016_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2017_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2017_1yr" + description = "School district elementary 2017 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2017_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2017_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2017_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2017_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2018_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2018_1yr" + description = "School district elementary 2018 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2018_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2018_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2018_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2018_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2019_1yr" + description = "School district elementary 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2020_1yr" + description = "School district elementary 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2021_1yr" + description = "School district elementary 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2016_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2016_5yr" + description = "School district elementary 2016 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2016_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2016_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2016_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2016_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2017_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2017_5yr" + description = "School district elementary 2017 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2017_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2017_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2017_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2017_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2018_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2018_5yr" + description = "School district elementary 2018 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2018_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2018_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2018_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2018_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2019_5yr" + description = "School district elementary 2019 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2020_5yr" + description = "School district elementary 2020 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2021_5yr" + description = "School district elementary 2021 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictelementary_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictelementary_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2016_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2016_1yr" + description = "School district secondary 2016 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2016_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2016_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2016_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2016_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2017_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2017_1yr" + description = "School district secondary 2017 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2017_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2017_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2017_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2017_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2018_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2018_1yr" + description = "School district secondary 2018 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2018_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2018_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2018_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2018_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2019_1yr" + description = "School district secondary 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2020_1yr" + description = "School district secondary 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2021_1yr" + description = "School district secondary 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2016_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2016_5yr" + description = "School district secondary 2016 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2016_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2016_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2016_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2016_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2017_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2017_5yr" + description = "School district secondary 2017 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2017_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2017_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2017_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2017_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2018_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2018_5yr" + description = "School district secondary 2018 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2018_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2018_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2018_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2018_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2019_5yr" + description = "School district secondary 2019 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2020_5yr" + description = "School district secondary 2020 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2021_5yr" + description = "School district secondary 2021 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictsecondary_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictsecondary_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2016_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2016_1yr" + description = "School district unified 2016 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2016_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2016_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2016_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2016_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2017_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2017_1yr" + description = "School district unified 2017 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2017_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2017_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2017_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2017_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2018_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2018_1yr" + description = "School district unified 2018 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2018_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2018_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2018_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2018_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2019_1yr" + description = "School district unified 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2020_1yr" + description = "School district unified 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2021_1yr" + description = "School district unified 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2016_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2016_5yr" + description = "School district unified 2016 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2016_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2016_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2016_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2016_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2017_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2017_5yr" + description = "School district unified 2017 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2017_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2017_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2017_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2017_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2018_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2018_5yr" + description = "School district unified 2018 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2018_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2018_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2018_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2018_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2019_5yr" + description = "School district unified 2019 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2020_5yr" + description = "School district unified 2020 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2021_5yr" + description = "School district unified 2021 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_schooldistrictunified_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_schooldistrictunified_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_state_2019_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2019_1yr" + description = "State 2019 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_state_2019_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_state_2019_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_state_2019_1yr-id" { + value = google_bigquery_table.census_bureau_acs_state_2019_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_state_2020_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2020_1yr" + description = "State 2020 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_state_2020_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_state_2020_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_state_2020_1yr-id" { + value = google_bigquery_table.census_bureau_acs_state_2020_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_state_2021_1yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2021_1yr" + description = "State 2021 1 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_state_2021_1yr-table_id" { + value = google_bigquery_table.census_bureau_acs_state_2021_1yr.table_id +} + +output "bigquery_table-census_bureau_acs_state_2021_1yr-id" { + value = google_bigquery_table.census_bureau_acs_state_2021_1yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_state_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2019_5yr" + description = "State 2019 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_state_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_state_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_state_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_state_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_state_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2020_5yr" + description = "State 2020 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_state_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_state_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_state_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_state_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_state_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2021_5yr" + description = "State 2021 5 year report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_state_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_state_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_state_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_state_2021_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_zcta_2019_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "zcta_2019_5yr" + description = "ZCTA 2019 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_zcta_2019_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_zcta_2019_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_zcta_2019_5yr-id" { + value = google_bigquery_table.census_bureau_acs_zcta_2019_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_zcta_2020_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "zcta_2020_5yr" + description = "ZCTA 2020 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_zcta_2020_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_zcta_2020_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_zcta_2020_5yr-id" { + value = google_bigquery_table.census_bureau_acs_zcta_2020_5yr.id +} + +resource "google_bigquery_table" "census_bureau_acs_zcta_2021_5yr" { + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "zcta_2021_5yr" + description = "ZCTA 2021 5 years report table" + depends_on = [ + google_bigquery_dataset.census_bureau_acs + ] +} + +output "bigquery_table-census_bureau_acs_zcta_2021_5yr-table_id" { + value = google_bigquery_table.census_bureau_acs_zcta_2021_5yr.table_id +} + +output "bigquery_table-census_bureau_acs_zcta_2021_5yr-id" { + value = google_bigquery_table.census_bureau_acs_zcta_2021_5yr.id +} diff --git a/datasets/census_bureau_acs/infra/censustract_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/censustract_2019_5yr_pipeline.tf index faa93863f..d50b84def 100644 --- a/datasets/census_bureau_acs/infra/censustract_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/censustract_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_censustract_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "censustract_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "censustract_2019_5yr" description = "Census tract 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/congressionaldistrict_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/congressionaldistrict_2019_1yr_pipeline.tf index e98a87a3f..adbe673f9 100644 --- a/datasets/census_bureau_acs/infra/congressionaldistrict_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/congressionaldistrict_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "congressionaldistrict_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2019_1yr" description = "Congressional district 2019 1 year table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/congressionaldistrict_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/congressionaldistrict_2019_5yr_pipeline.tf index 189da3596..a9c79b60f 100644 --- a/datasets/census_bureau_acs/infra/congressionaldistrict_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/congressionaldistrict_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_congressionaldistrict_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "congressionaldistrict_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "congressionaldistrict_2019_5yr" description = "Congressional district 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/county_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/county_2019_1yr_pipeline.tf index 0174b6be5..1df6040bc 100644 --- a/datasets/census_bureau_acs/infra/county_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/county_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_county_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "county_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2019_1yr" description = "County 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/county_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/county_2019_5yr_pipeline.tf index aa7409520..16aaabad9 100644 --- a/datasets/census_bureau_acs/infra/county_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/county_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_county_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "county_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "county_2019_5yr" description = "County 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/place_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/place_2019_1yr_pipeline.tf index 62f11d13e..f5bd6f238 100644 --- a/datasets/census_bureau_acs/infra/place_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/place_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_place_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "place_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2019_1yr" description = "Place 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/place_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/place_2019_5yr_pipeline.tf index 27d17d19c..663c0664b 100644 --- a/datasets/census_bureau_acs/infra/place_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/place_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_place_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "place_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "place_2019_5yr" description = "Place 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/puma_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/puma_2019_1yr_pipeline.tf index ea2dfc43f..9394f5481 100644 --- a/datasets/census_bureau_acs/infra/puma_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/puma_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_puma_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "puma_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2019_1yr" description = "PUMA 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/puma_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/puma_2019_5yr_pipeline.tf index a6946806d..ab797df9a 100644 --- a/datasets/census_bureau_acs/infra/puma_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/puma_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_puma_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "puma_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "puma_2019_5yr" description = "PUMA 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_1yr_pipeline.tf index 18d7b4d27..284a16104 100644 --- a/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "schooldistrictelementary_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2019_1yr" description = "School district elementary 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_5yr_pipeline.tf index d6959a5ca..abb4367fe 100644 --- a/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/schooldistrictelementary_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_schooldistrictelementary_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "schooldistrictelementary_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictelementary_2019_5yr" description = "School district elementary 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_1yr_pipeline.tf index 97db61017..860f1f807 100644 --- a/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "schooldistrictsecondary_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2019_1yr" description = "School district secondary 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_5yr_pipeline.tf index 039feaf2c..3a7817f0c 100644 --- a/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/schooldistrictsecondary_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_schooldistrictsecondary_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "schooldistrictsecondary_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictsecondary_2019_5yr" description = "School district secondary 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/schooldistrictunified_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/schooldistrictunified_2019_1yr_pipeline.tf index ad8a23c0c..328f907d1 100644 --- a/datasets/census_bureau_acs/infra/schooldistrictunified_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/schooldistrictunified_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "schooldistrictunified_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2019_1yr" description = "School district unified 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/schooldistrictunified_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/schooldistrictunified_2019_5yr_pipeline.tf index 47e7f9f49..453623aa9 100644 --- a/datasets/census_bureau_acs/infra/schooldistrictunified_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/schooldistrictunified_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_schooldistrictunified_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "schooldistrictunified_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "schooldistrictunified_2019_5yr" description = "School district unified 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/state_2019_1yr_pipeline.tf b/datasets/census_bureau_acs/infra/state_2019_1yr_pipeline.tf index 0b1a4f5d1..a39028ee5 100644 --- a/datasets/census_bureau_acs/infra/state_2019_1yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/state_2019_1yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_state_2019_1yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "state_2019_1yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2019_1yr" description = "State 2019 1 year report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/state_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/state_2019_5yr_pipeline.tf index 8e92e7290..0f35e4dd6 100644 --- a/datasets/census_bureau_acs/infra/state_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/state_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_state_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "state_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "state_2019_5yr" description = "State 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/infra/variables.tf b/datasets/census_bureau_acs/infra/variables.tf index c3ec7c506..53f483735 100644 --- a/datasets/census_bureau_acs/infra/variables.tf +++ b/datasets/census_bureau_acs/infra/variables.tf @@ -20,4 +20,7 @@ variable "bucket_name_prefix" {} variable "impersonating_acct" {} variable "region" {} variable "env" {} +variable "iam_policies" { + default = {} +} diff --git a/datasets/census_bureau_acs/infra/zcta_2019_5yr_pipeline.tf b/datasets/census_bureau_acs/infra/zcta_2019_5yr_pipeline.tf index 279c312c6..c779347ae 100644 --- a/datasets/census_bureau_acs/infra/zcta_2019_5yr_pipeline.tf +++ b/datasets/census_bureau_acs/infra/zcta_2019_5yr_pipeline.tf @@ -16,15 +16,10 @@ resource "google_bigquery_table" "census_bureau_acs_zcta_2019_5yr" { - project = var.project_id - dataset_id = "census_bureau_acs" - table_id = "zcta_2019_5yr" - + project = var.project_id + dataset_id = "census_bureau_acs" + table_id = "zcta_2019_5yr" description = "ZCTA 2019 5 years report table" - - - - depends_on = [ google_bigquery_dataset.census_bureau_acs ] diff --git a/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/csv_transform.py b/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/csv_transform.py index 1fca4898f..843b1b523 100644 --- a/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/csv_transform.py +++ b/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/csv_transform.py @@ -12,97 +12,426 @@ # See the License for the specific language governing permissions and # limitations under the License. - -import datetime +import csv import json import logging import os import pathlib +import sys import typing import numpy as np import pandas as pd import requests -from google.cloud import storage +from google.api_core.exceptions import NotFound +from google.cloud import bigquery, storage def main( source_url: str, - year_report: str, - api_naming_convention: str, + source_file: pathlib.Path, target_file: pathlib.Path, + chunksize: str, + project_id: str, + dataset_id: str, + table_id: str, + schema_path: str, + pipeline_name: str, target_gcs_bucket: str, target_gcs_path: str, - headers: typing.List[str], - rename_mappings: dict, - pipeline_name: str, + year_report: str, + api_naming_convention: str, geography: str, report_level: str, - concat_col: typing.List[str], + concat_col_list: typing.List[str], + data_dtypes: str, + rename_mappings_list: dict, + input_csv_headers: typing.List[str], + output_csv_headers: typing.List[str], ) -> None: - - logging.info( - f"ACS {pipeline_name} process started at " - + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - ) - logging.info("Creating 'files' folder") pathlib.Path("./files").mkdir(parents=True, exist_ok=True) + logging.info(f"{pipeline_name} process started") + execute_pipeline( + source_file=source_file, + chunksize=chunksize, + report_level=report_level, + year_report=year_report, + api_naming_convention=api_naming_convention, + source_url=source_url, + data_dtypes=data_dtypes, + rename_mappings_list=rename_mappings_list, + geography=geography, + concat_col_list=concat_col_list, + input_csv_headers=input_csv_headers, + output_csv_headers=output_csv_headers, + target_file=target_file, + target_gcs_bucket=target_gcs_bucket, + target_gcs_path=target_gcs_path, + project_id=project_id, + dataset_id=dataset_id, + destination_table=table_id, + schema_path=schema_path, + ) + logging.info(f"{pipeline_name} --> ETL process completed") + +def execute_pipeline( + source_file: str, + chunksize: str, + report_level: str, + year_report: str, + api_naming_convention: str, + source_url: str, + data_dtypes: dict, + rename_mappings_list: dict, + geography: str, + concat_col_list: typing.List[str], + input_csv_headers: typing.List[str], + output_csv_headers: typing.List[str], + target_file: str, + target_gcs_bucket: str, + target_gcs_path: str, + project_id: str, + dataset_id: str, + destination_table: str, + schema_path: str, +) -> None: json_obj_group_id = open("group_ids.json") group_id = json.load(json_obj_group_id) - json_obj_state_code = open("state_codes.json") state_code = json.load(json_obj_state_code) - logging.info("Extracting the data from API and loading into dataframe...") if report_level == "national_level": df = extract_data_and_convert_to_df_national_level( - group_id, year_report, api_naming_convention, source_url + group_id, year_report, api_naming_convention, source_url, destination_table ) elif report_level == "state_level": df = extract_data_and_convert_to_df_state_level( - group_id, state_code, year_report, api_naming_convention, source_url + group_id, + state_code, + year_report, + api_naming_convention, + source_url, + destination_table, + ) + save_to_new_file(df, source_file, sep=",") + process_source_file( + source_file=source_file, + target_file=target_file, + chunksize=chunksize, + input_headers=input_csv_headers, + data_dtypes=data_dtypes, + geography=geography, + rename_mappings_list=rename_mappings_list, + concat_col_list=concat_col_list, + input_csv_headers=input_csv_headers, + output_csv_headers=output_csv_headers, + group_id=group_id, + state_code=state_code, + ) + if os.path.exists(target_file): + upload_file_to_gcs( + file_path=target_file, + target_gcs_bucket=target_gcs_bucket, + target_gcs_path=target_gcs_path, + project_id=project_id, + ) + table_exists = create_dest_table( + project_id=project_id, + dataset_id=dataset_id, + table_id=destination_table, + schema_filepath=schema_path, + bucket_name=target_gcs_bucket, + drop_table="N", + ) + if table_exists: + load_data_to_bq( + project_id=project_id, + dataset_id=dataset_id, + table_id=destination_table, + file_path=target_file, + truncate_table=True, + field_delimiter="|", + ) + else: + error_msg = f"Error: Data was not loaded because the destination table {project_id}.{dataset_id}.{destination_table} does not exist and/or could not be created." + raise ValueError(error_msg) + else: + logging.info( + f"Informational: The data file {target_file} was not generated because no data file was available. Continuing." ) - logging.info("Replacing values...") - df = df.replace(to_replace={"KPI_Name": group_id}) - logging.info("Renaming headers...") - rename_headers(df, rename_mappings) +def process_source_file( + source_file: str, + target_file: str, + chunksize: str, + input_headers: str, + data_dtypes: dict, + geography: str, + rename_mappings_list: dict, + concat_col_list: typing.List[str], + input_csv_headers: typing.List[str], + output_csv_headers: typing.List[str], + group_id: str, + state_code: str, +) -> None: + logging.info(f"Opening source file {source_file}") + csv.field_size_limit(512 << 10) + csv.register_dialect("TabDialect", quotechar='"', delimiter=",", strict=True) + with open(source_file) as reader: + data = [] + chunk_number = 1 + for index, line in enumerate(csv.reader(reader, "TabDialect"), 0): + data.append(line) + if index % int(chunksize) == 0 and index > 0: + process_dataframe_chunk( + data=data, + input_headers=input_headers, + target_file=target_file, + chunk_number=chunk_number, + data_dtypes=data_dtypes, + geography=geography, + rename_mappings_list=rename_mappings_list, + concat_col_list=concat_col_list, + input_csv_headers=input_csv_headers, + output_csv_headers=output_csv_headers, + group_id=group_id, + state_code=state_code, + ) + data = [] + chunk_number += 1 + if index % int(chunksize) != 0 and index > 0: + process_dataframe_chunk( + data=data, + input_headers=input_headers, + target_file=target_file, + chunk_number=chunk_number, + data_dtypes=data_dtypes, + geography=geography, + rename_mappings_list=rename_mappings_list, + concat_col_list=concat_col_list, + input_csv_headers=input_csv_headers, + output_csv_headers=output_csv_headers, + group_id=group_id, + state_code=state_code, + ) + dfcolumns = list(target_df.columns) + i = 0 + while i < (len(output_csv_headers)): + if output_csv_headers[i] not in dfcolumns: + output_csv_headers.pop(i) + i -= 1 + i += 1 + logging.info("Reordering headers...") + final_df = target_df[output_csv_headers] + save_to_new_file(final_df, target_file, sep="|") + print("final df -->") + print(final_df) + print("Opening final file -->") + print(pd.read_csv(target_file, sep="|")) + + +def process_dataframe_chunk( + data: typing.List[str], + input_headers: typing.List[str], + target_file: str, + chunk_number: int, + data_dtypes: dict, + geography: str, + rename_mappings_list: dict, + concat_col_list: typing.List[str], + input_csv_headers: typing.List[str], + output_csv_headers: typing.List[str], + group_id: str, + state_code: str, +) -> None: + df = pd.DataFrame(data, columns=input_headers) + target_file_batch = str(target_file).replace( + ".csv", "-" + str(chunk_number) + ".csv" + ) + process_chunk( + df=df, + chunk_number=chunk_number, + target_file_batch=target_file_batch, + target_file=target_file, + skip_header=0, + geography=geography, + rename_mappings_list=rename_mappings_list, + concat_col_list=concat_col_list, + input_csv_headers=input_csv_headers, + output_csv_headers=output_csv_headers, + group_id=group_id, + state_code=state_code, + ) - logging.info("Creating column geo_id...") + +def set_df_datatypes(df: pd.DataFrame, data_dtypes: dict) -> pd.DataFrame: + logging.info("Setting data types") + for key, item in data_dtypes.items(): + df[key] = df[key].astype(item) + return df + + +def process_chunk( + df: pd.DataFrame, + chunk_number: int, + target_file_batch: str, + target_file: str, + skip_header: bool, + geography: str, + rename_mappings_list: dict, + concat_col_list: typing.List[str], + input_csv_headers: typing.List[str], + output_csv_headers: typing.List[str], + group_id: str, + state_code: str, +) -> None: + logging.info(f"Processing batch file {target_file_batch}") + logging.info("Replacing values...") + df = df.replace(to_replace={"KPI_Name": group_id}) + rename_headers(df, rename_mappings_list) if geography == "censustract" or geography == "blockgroup": df["tract"] = df["tract"].apply(pad_zeroes_to_the_left, args=(6,)) df["state"] = df["state"].apply(pad_zeroes_to_the_left, args=(2,)) df["county"] = df["county"].apply(pad_zeroes_to_the_left, args=(3,)) + df = create_geo_id(df, concat_col_list) + df = pivot_dataframe(df) + save_to_new_file(df, file_path=str(target_file_batch), sep="|") + df = append_batch_file( + df, chunk_number, target_file_batch, target_file, skip_header, not (skip_header) + ) + logging.info(f"Processing batch file {target_file_batch} completed") - df = create_geo_id(df, concat_col) - logging.info("Pivoting the dataframe...") - df = df[["geo_id", "KPI_Name", "KPI_Value"]] - df = df.pivot_table( - index="geo_id", columns="KPI_Name", values="KPI_Value", aggfunc=np.sum - ).reset_index() +def load_data_to_bq( + project_id: str, + dataset_id: str, + table_id: str, + file_path: str, + truncate_table: bool, + field_delimiter: str = "|", +) -> None: + logging.info( + f"Loading data from {file_path} into {project_id}.{dataset_id}.{table_id} started" + ) + client = bigquery.Client(project=project_id) + table_ref = client.dataset(dataset_id).table(table_id) + job_config = bigquery.LoadJobConfig() + job_config.source_format = bigquery.SourceFormat.CSV + job_config.field_delimiter = field_delimiter + if truncate_table: + job_config.write_disposition = "WRITE_TRUNCATE" + else: + job_config.write_disposition = "WRITE_APPEND" + job_config.skip_leading_rows = 1 + job_config.autodetect = False + with open(file_path, "rb") as source_file: + job = client.load_table_from_file(source_file, table_ref, job_config=job_config) + job.result() + logging.info( + f"Loading data from {file_path} into {project_id}.{dataset_id}.{table_id} completed" + ) - logging.info("Reordering headers...") - df = df[headers] - logging.info(f"Saving to output file.. {target_file}") +def create_dest_table( + project_id: str, + dataset_id: str, + table_id: str, + schema_filepath: list, + bucket_name: str, + drop_table: bool = False, +) -> bool: + table_ref = f"{project_id}.{dataset_id}.{table_id}" + logging.info(f"Attempting to create table {table_ref} if it doesn't already exist") + client = bigquery.Client() + table_exists = False try: - save_to_new_file(df, file_path=str(target_file)) - except Exception as e: - logging.error(f"Error saving output file: {e}.") + table = client.get_table(table_ref) + table_exists_id = table.table_id + logging.info(f"Table {table_exists_id} currently exists.") + if drop_table: + logging.info("Dropping existing table") + client.delete_table(table) + table = None + except NotFound: + table = None + if not table: + logging.info( + ( + f"Table {table_ref} currently does not exist. Attempting to create table." + ) + ) + if check_gcs_file_exists(schema_filepath, bucket_name): + schema = create_table_schema([], bucket_name, schema_filepath) + table = bigquery.Table(table_ref, schema=schema) + client.create_table(table) + print(f"Table {table_ref} was created".format(table_id)) + table_exists = True + else: + file_name = os.path.split(schema_filepath)[1] + file_path = os.path.split(schema_filepath)[0] + logging.info( + f"Error: Unable to create table {table_ref} because schema file {file_name} does not exist in location {file_path} in bucket {bucket_name}" + ) + table_exists = False + else: + table_exists = True + return table_exists - logging.info( - f"Uploading output file to.. gs://{target_gcs_bucket}/{target_gcs_path}" - ) - upload_file_to_gcs(target_file, target_gcs_bucket, target_gcs_path) - logging.info( - f"ACS {pipeline_name} process completed at " - + str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - ) +def check_gcs_file_exists(file_path: str, bucket_name: str) -> bool: + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + exists = storage.Blob(bucket=bucket, name=file_path).exists(storage_client) + return exists + + +def create_table_schema( + schema_structure: list, + bucket_name: str = "", + schema_filepath: str = "", +) -> list: + logging.info(f"Defining table schema... {bucket_name} ... {schema_filepath}") + schema = [] + if not (schema_filepath): + schema_struct = schema_structure + else: + storage_client = storage.Client() + bucket = storage_client.get_bucket(bucket_name) + blob = bucket.blob(schema_filepath) + schema_struct = json.loads(blob.download_as_string(client=None)) + i = 0 + dfcolumns = list(target_df.columns) + while i < len(schema_struct): + if schema_struct[i].get("name") not in dfcolumns: + schema_struct.pop(i) + i -= 1 + i += 1 + for schema_field in schema_struct: + fld_name = schema_field["name"] + fld_type = schema_field["type"] + try: + fld_descr = schema_field["description"] + except KeyError: + fld_descr = "" + fld_mode = schema_field["mode"] + schema.append( + bigquery.SchemaField( + name=fld_name, field_type=fld_type, mode=fld_mode, description=fld_descr + ) + ) + return schema + + +def pivot_dataframe(df: pd.DataFrame) -> pd.DataFrame: + logging.info("Pivoting the dataframe...") + df = df[["geo_id", "KPI_Name", "KPI_Value"]] + df = df.pivot_table( + index="geo_id", columns="KPI_Name", values="KPI_Value", aggfunc=np.sum + ).reset_index() + return df def string_replace(source_url, replace: dict) -> str: @@ -112,9 +441,14 @@ def string_replace(source_url, replace: dict) -> str: def extract_data_and_convert_to_df_national_level( - group_id: dict, year_report: str, api_naming_convention: str, source_url: str + group_id: dict, + year_report: str, + api_naming_convention: str, + source_url: str, + destination_table: str, ) -> pd.DataFrame: list_temp = [] + flag = 0 for key in group_id: logging.info(f"reading data from API for KPI {key}...") str1 = source_url.replace("~year_report~", year_report) @@ -122,17 +456,24 @@ def extract_data_and_convert_to_df_national_level( str3 = str2.replace("~row_position~", key[-3:]) source_url_new = str3.replace("~api_naming_convention~", api_naming_convention) try: - r = requests.get(source_url_new, stream=True) - logging.info(f"Source url : {source_url_new}") - logging.info(f"status code : {r.status_code}") + r = requests.get(source_url_new, stream=True, verify=False, timeout=200) if r.status_code == 200: + logging.info("Data source valid for at least one entity") + flag = 1 text = r.json() frame = load_nested_list_into_df_without_headers(text) frame["KPI_Name"] = key list_temp.append(frame) + elif 400 >= r.status_code <= 499: + logging.info(r.status_code) + else: + logging.info(f"Source url : {source_url_new}") + logging.info(f"status code : {r.status_code}") except OSError as e: logging.info(f"error : {e}") - pass + if flag == 0: + logging.info(f"Data not available for {destination_table} yet") + sys.exit(0) logging.info("creating the dataframe...") df = pd.concat(list_temp) return df @@ -150,8 +491,10 @@ def extract_data_and_convert_to_df_state_level( year_report: str, api_naming_convention: str, source_url: str, + destination_table: str, ) -> pd.DataFrame: list_temp = [] + flag = 0 for key in group_id: for sc in state_code: logging.info(f"reading data from API for KPI {key}...") @@ -162,24 +505,33 @@ def extract_data_and_convert_to_df_state_level( str4 = str3.replace("~api_naming_convention~", api_naming_convention) source_url_new = str4.replace("~state_code~", sc) try: - r = requests.get(source_url_new, stream=True) - logging.info(f"Source url : {source_url_new}") - logging.info(f"status code : {r.status_code}") + r = requests.get(source_url_new, stream=True, verify=False, timeout=200) if r.status_code == 200: + logging.info("Data source valid for at least one entity") + flag = 1 text = r.json() frame = load_nested_list_into_df_without_headers(text) frame["KPI_Name"] = key list_temp.append(frame) + elif 400 >= r.status_code <= 499: + logging.info(r.status_code) + else: + logging.info(f"Source url : {source_url_new}") + logging.info(f"status code : {r.status_code}") + except OSError as e: logging.info(f"error : {e}") - pass - + if flag == 0: + logging.info(f"Data not available for {destination_table} yet") + sys.exit(0) logging.info("creating the dataframe...") df = pd.concat(list_temp) return df def create_geo_id(df: pd.DataFrame, concat_col: str) -> pd.DataFrame: + logging.info("Creating column geo_id...") + df = df.drop(0, axis=0) df["geo_id"] = "" for col in concat_col: df["geo_id"] = df["geo_id"] + df[col] @@ -194,35 +546,96 @@ def pad_zeroes_to_the_left(val: str, length: int) -> str: def rename_headers(df: pd.DataFrame, rename_mappings: dict) -> None: - rename_mappings = {int(k): str(v) for k, v in rename_mappings.items()} + logging.info("Renaming headers...") df.rename(columns=rename_mappings, inplace=True) -def save_to_new_file(df: pd.DataFrame, file_path: str) -> None: - df.to_csv(file_path, index=False) +def save_to_new_file(df: pd.DataFrame, file_path: str, sep: str = ",") -> None: + logging.info(f"Saving data to target file.. {file_path} ...") + df.to_csv(file_path, index=False, sep=sep) -def upload_file_to_gcs(file_path: pathlib.Path, gcs_bucket: str, gcs_path: str) -> None: - storage_client = storage.Client() - bucket = storage_client.bucket(gcs_bucket) - blob = bucket.blob(gcs_path) - blob.upload_from_filename(file_path) +def append_batch_file( + df: pd.DataFrame, + chunk_number: int, + batch_file_path: str, + target_file_path: str, + skip_header: bool, + truncate_file: bool, +) -> pd.DataFrame: + global target_df + truncate_file = True + with open(batch_file_path, "r") as data_file: + if truncate_file: + open(target_file_path, "w+").close() + with open(target_file_path, "w"): + if skip_header: + logging.info( + f"Appending batch file {batch_file_path} to {target_file_path} with skip header" + ) + next(data_file) + else: + logging.info( + f"Appending batch file {batch_file_path} to {target_file_path}" + ) + batch_df = pd.read_csv(data_file, sep="|") + logging.info("Making batch df -- >") + if chunk_number == 1: + target_df = batch_df + target_df.to_csv(target_file_path, index=False, sep="|") + else: + batchdfcols = list(batch_df.columns) + targetdfcols = dict.fromkeys(list(target_df.columns)) + logging.info(f"Removing common columns from batch : {chunk_number}") + for columns in batchdfcols: + if columns in targetdfcols: + batch_df.drop(columns, axis=1, inplace=True) + target_df = pd.concat([target_df, batch_df], axis=1, ignore_index=False) + target_df.to_csv(target_file_path, index=False, sep="|") + return target_df + + +def upload_file_to_gcs( + file_path: pathlib.Path, + target_gcs_bucket: str, + target_gcs_path: str, + project_id: str, +) -> None: + if os.path.exists(file_path): + logging.info( + f"Uploading output file to gs://{target_gcs_bucket}/{target_gcs_path}" + ) + storage_client = storage.Client(project=project_id) + bucket = storage_client.bucket(target_gcs_bucket) + blob = bucket.blob(target_gcs_path) + blob.upload_from_filename(file_path) + else: + logging.info( + f"Cannot upload file to gs://{target_gcs_bucket}/{target_gcs_path} as it does not exist." + ) if __name__ == "__main__": logging.getLogger().setLevel(logging.INFO) - main( - source_url=os.environ["SOURCE_URL"], - year_report=os.environ["YEAR_REPORT"], - api_naming_convention=os.environ["API_NAMING_CONVENTION"], - target_file=pathlib.Path(os.environ["TARGET_FILE"]).expanduser(), - target_gcs_bucket=os.environ["TARGET_GCS_BUCKET"], - target_gcs_path=os.environ["TARGET_GCS_PATH"], - headers=json.loads(os.environ["CSV_HEADERS"]), - rename_mappings=json.loads(os.environ["RENAME_MAPPINGS"]), - pipeline_name=os.environ["PIPELINE_NAME"], - geography=os.environ["GEOGRAPHY"], - report_level=os.environ["REPORT_LEVEL"], - concat_col=json.loads(os.environ["CONCAT_COL"]), + source_url=os.environ.get("SOURCE_URL", ""), + chunksize=os.environ.get("CHUNKSIZE", ""), + source_file=pathlib.Path(os.environ.get("SOURCE_FILE", "")).expanduser(), + target_file=pathlib.Path(os.environ.get("TARGET_FILE", "")).expanduser(), + project_id=os.environ.get("PROJECT_ID", ""), + dataset_id=os.environ.get("DATASET_ID", ""), + table_id=os.environ.get("TABLE_ID", ""), + schema_path=os.environ.get("SCHEMA_PATH", ""), + pipeline_name=os.environ.get("PIPELINE_NAME", ""), + target_gcs_bucket=os.environ.get("TARGET_GCS_BUCKET", ""), + target_gcs_path=os.environ.get("TARGET_GCS_PATH", ""), + year_report=os.environ.get("YEAR_REPORT", ""), + api_naming_convention=os.environ.get("API_NAMING_CONVENTION", ""), + geography=os.environ.get("GEOGRAPHY", ""), + report_level=os.environ.get("REPORT_LEVEL", ""), + concat_col_list=json.loads(os.environ.get("CONCAT_COL_LIST", r"[]")), + data_dtypes=json.loads(os.environ.get("DATA_DTYPES", r"{}")), + rename_mappings_list=json.loads(os.environ.get("RENAME_MAPPINGS_LIST", r"{}")), + input_csv_headers=json.loads(os.environ.get("INPUT_CSV_HEADERS", r"[]")), + output_csv_headers=json.loads(os.environ.get("OUTPUT_CSV_HEADERS", r"[]")), ) diff --git a/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/requirements.txt b/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/requirements.txt index b17e29e5e..b8d99f1d5 100644 --- a/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/requirements.txt +++ b/datasets/census_bureau_acs/pipelines/_images/run_csv_transform_kub/requirements.txt @@ -1,5 +1,6 @@ requests pandas google-cloud-storage +google-cloud-bigquery numpy diff --git a/datasets/census_bureau_acs/pipelines/census_bureau_acs/census_bureau_acs_dag.py b/datasets/census_bureau_acs/pipelines/census_bureau_acs/census_bureau_acs_dag.py new file mode 100644 index 000000000..9f5da3c1b --- /dev/null +++ b/datasets/census_bureau_acs/pipelines/census_bureau_acs/census_bureau_acs_dag.py @@ -0,0 +1,2949 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from airflow import DAG +from airflow.providers.google.cloud.operators import kubernetes_engine + +default_args = { + "owner": "Google", + "depends_on_past": False, + "start_date": "2022-06-30", +} + + +with DAG( + dag_id="census_bureau_acs.census_bureau_acs", + default_args=default_args, + max_active_runs=1, + schedule_interval="@monthly", + catchup=False, + default_view="graph", +) as dag: + create_cluster = kubernetes_engine.GKECreateClusterOperator( + task_id="create_cluster", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + body={ + "name": "census-bureau-acs", + "initial_node_count": 30, + "network": "{{ var.value.vpc_network }}", + "node_config": { + "machine_type": "e2-standard-4", + "oauth_scopes": [ + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/cloud-platform", + ], + }, + }, + ) + + # Run CSV transform within kubernetes pod + cbsa_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="cbsa_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "cbsa_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "metropolitan%20statistical%20area/micropolitan%20statistical%20area", + "GEOGRAPHY": "cbsa", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["combined_statistical_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + cbsa_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="cbsa_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.cbsa_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.cbsa_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.cbsa_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.cbsa_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.cbsa_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "cbsa_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.cbsa_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "metropolitan%20statistical%20area/micropolitan%20statistical%20area", + "GEOGRAPHY": "cbsa", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["combined_statistical_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + cbsa_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="cbsa_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "cbsa_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "metropolitan%20statistical%20area/micropolitan%20statistical%20area", + "GEOGRAPHY": "cbsa", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["combined_statistical_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + cbsa_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="cbsa_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.cbsa_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.cbsa_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.cbsa_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.cbsa_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.cbsa_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "cbsa_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "metropolitan%20statistical%20area/micropolitan%20statistical%20area", + "GEOGRAPHY": "cbsa", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["combined_statistical_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + cbsa_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="cbsa_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.cbsa_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.cbsa_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.cbsa_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.cbsa_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.cbsa_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "cbsa_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.cbsa_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "metropolitan%20statistical%20area/micropolitan%20statistical%20area", + "GEOGRAPHY": "cbsa", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["combined_statistical_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + cbsa_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="cbsa_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.cbsa_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.cbsa_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.cbsa_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.cbsa_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.cbsa_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "cbsa_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.cbsa_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "metropolitan%20statistical%20area/micropolitan%20statistical%20area", + "GEOGRAPHY": "cbsa", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["combined_statistical_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + censustract_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="censustract_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.censustract_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.censustract_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.censustract_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.censustract_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.censustract_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "censustract_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.censustract_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "tract", + "GEOGRAPHY": "censustract", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","county","tract"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county","4":"tract" }', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "4", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "12G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + censustract_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="censustract_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.censustract_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.censustract_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.censustract_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.censustract_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.censustract_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "censustract_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.censustract_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "tract", + "GEOGRAPHY": "censustract", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","county","tract"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county","4":"tract" }', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "4", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "12G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + censustract_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="censustract_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.censustract_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.censustract_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.censustract_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.censustract_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.censustract_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "censustract_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.censustract_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "tract", + "GEOGRAPHY": "censustract", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","county","tract"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county","4":"tract" }', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "4", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "12G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + congressionaldistrict_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="congressionaldistrict_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "congressionaldistrict_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "congressional%20district", + "GEOGRAPHY": "congressionaldistrict", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","congressional_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + congressionaldistrict_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="congressionaldistrict_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "congressionaldistrict_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "congressional%20district", + "GEOGRAPHY": "congressionaldistrict", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","congressional_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + congressionaldistrict_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="congressionaldistrict_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "congressionaldistrict_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "congressional%20district", + "GEOGRAPHY": "congressionaldistrict", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","congressional_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + congressionaldistrict_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="congressionaldistrict_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "congressionaldistrict_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "congressional%20district", + "GEOGRAPHY": "congressionaldistrict", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","congressional_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + congressionaldistrict_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="congressionaldistrict_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "congressionaldistrict_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "congressional%20district", + "GEOGRAPHY": "congressionaldistrict", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","congressional_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + congressionaldistrict_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="congressionaldistrict_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "congressionaldistrict_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "congressional%20district", + "GEOGRAPHY": "congressionaldistrict", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","congressional_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + county_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="county_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.county_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.county_2019_1yr.target_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.county_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.county_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.county_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "county_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.county_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "county", + "GEOGRAPHY": "county", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","county"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + county_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="county_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.county_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.county_2020_1yr.target_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.county_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.county_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.county_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "county_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.county_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "county", + "GEOGRAPHY": "county", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","county"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + county_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="county_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.county_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.county_2021_1yr.target_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.county_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.county_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.county_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "county_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.county_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "county", + "GEOGRAPHY": "county", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","county"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + county_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="county_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.county_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.county_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.county_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.county_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.county_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "county_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.county_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "county", + "GEOGRAPHY": "county", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","county"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + county_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="county_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.county_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.county_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.county_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.county_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.county_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "county_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.county_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "county", + "GEOGRAPHY": "county", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","county"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + county_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="county_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.county_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.county_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.county_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.county_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.county_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "county_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.county_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "county", + "GEOGRAPHY": "county", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","county"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"county"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + place_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="place_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.place_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.place_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.place_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.place_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.place_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "place_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.place_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "place", + "GEOGRAPHY": "place", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","place"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"place"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + place_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="place_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.place_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.place_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.place_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.place_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.place_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "place_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.place_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "place", + "GEOGRAPHY": "place", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","place"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"place"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + place_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="place_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.place_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.place_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.place_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.place_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.place_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "place_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.place_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "place", + "GEOGRAPHY": "place", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","place"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"place"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + place_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="place_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.place_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.place_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.place_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.place_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.place_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "place_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.place_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "place", + "GEOGRAPHY": "place", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","place"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"place"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + place_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="place_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.place_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.place_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.place_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.place_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.place_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "place_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.place_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "place", + "GEOGRAPHY": "place", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","place"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"place"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + place_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="place_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.place_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.place_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.place_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.place_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.place_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "place_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.place_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "place", + "GEOGRAPHY": "place", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","place"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"place"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + puma_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="puma_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.puma_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.puma_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.puma_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.puma_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.puma_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.puma_2019_1yr.schema_path }}", + "PIPELINE_NAME": "puma_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.puma_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "public%20use%20microdata%20area", + "GEOGRAPHY": "puma", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","public_use_microdata_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + puma_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="puma_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.puma_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.puma_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.puma_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.puma_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.puma_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.puma_2020_1yr.schema_path }}", + "PIPELINE_NAME": "puma_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.puma_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "public%20use%20microdata%20area", + "GEOGRAPHY": "puma", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","public_use_microdata_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + puma_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="puma_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.puma_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.puma_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.puma_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.puma_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.puma_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.puma_2021_1yr.schema_path }}", + "PIPELINE_NAME": "puma_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.puma_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "public%20use%20microdata%20area", + "GEOGRAPHY": "puma", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","public_use_microdata_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + puma_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="puma_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.puma_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.puma_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.puma_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.puma_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.puma_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.puma_2019_5yr.schema_path }}", + "PIPELINE_NAME": "puma_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.puma_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "public%20use%20microdata%20area", + "GEOGRAPHY": "puma", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","public_use_microdata_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + puma_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="puma_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.puma_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.puma_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.puma_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.puma_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.puma_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.puma_2020_5yr.schema_path }}", + "PIPELINE_NAME": "puma_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.puma_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "public%20use%20microdata%20area", + "GEOGRAPHY": "puma", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","public_use_microdata_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + puma_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="puma_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.puma_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.puma_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.puma_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.puma_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.puma_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.puma_2021_5yr.schema_path }}", + "PIPELINE_NAME": "puma_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.puma_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "public%20use%20microdata%20area", + "GEOGRAPHY": "puma", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state","public_use_microdata_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2016_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2016_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2016_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2017_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2017_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2017_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2018_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2018_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2018_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2016_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2016_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2016_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2017_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2017_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2017_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2018_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2018_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2018_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictelementary_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictelementary_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictelementary_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(elementary)", + "GEOGRAPHY": "schooldistrictelementary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2016_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2016_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2016_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2017_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2017_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2017_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2018_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2018_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2018_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2016_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2016_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2016_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2017_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2017_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2017_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2018_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2018_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2018_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictsecondary_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictsecondary_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictsecondary_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(secondary)", + "GEOGRAPHY": "schooldistrictsecondary", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2016_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2016_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2016_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2017_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2017_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2017_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2018_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2018_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2018_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2016_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2016_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2016_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2017_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2017_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2017_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2018_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2018_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2018_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + schooldistrictunified_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="schooldistrictunified_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.schema_path }}", + "PIPELINE_NAME": "schooldistrictunified_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "school%20district%20(unified)", + "GEOGRAPHY": "schooldistrictunified", + "REPORT_LEVEL": "state_level", + "CONCAT_COL_LIST": '["state","school_district"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + state_2019_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="state_2019_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.state_2019_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.state_2019_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.state_2019_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.state_2019_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.state_2019_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "state_2019_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.state_2019_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "state", + "GEOGRAPHY": "state", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + state_2020_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="state_2020_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.state_2020_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.state_2020_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.state_2020_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.state_2020_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.state_2020_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "state_2020_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.state_2020_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "state", + "GEOGRAPHY": "state", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + state_2021_1yr = kubernetes_engine.GKEStartPodOperator( + task_id="state_2021_1yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.state_2021_1yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.state_2021_1yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.state_2021_1yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.state_2021_1yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.state_2021_1yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2021_1yr.schema_path }}", + "PIPELINE_NAME": "state_2021_1yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.state_2021_1yr.target_gcs_path }}", + "YEAR_REPORT": "1", + "API_NAMING_CONVENTION": "state", + "GEOGRAPHY": "state", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + state_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="state_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.state_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.state_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.state_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.state_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.state_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.state_2019_5yr.schema_path }}", + "PIPELINE_NAME": "state_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.state_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "state", + "GEOGRAPHY": "state", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + state_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="state_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.state_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.state_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.state_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.state_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.state_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "state_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.state_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "state", + "GEOGRAPHY": "state", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + state_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="state_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.state_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.state_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.state_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.state_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.state_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.state_2021_5yr.schema_path }}", + "PIPELINE_NAME": "state_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.state_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "state", + "GEOGRAPHY": "state", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["state"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + zcta_2019_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="zcta_2019_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.zcta_2019_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.zcta_2019_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.zcta_2019_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.zcta_2019_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.zcta_2019_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.zcta_2019_5yr.schema_path }}", + "PIPELINE_NAME": "zcta_2019_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.zcta_2019_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "zip%20code%20tabulation%20area", + "GEOGRAPHY": "zcta", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["zip_code_tabulation_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"zip_code_tabulation_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + zcta_2020_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="zcta_2020_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.zcta_2020_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.zcta_2020_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.zcta_2020_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.zcta_2020_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.zcta_2020_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}", + "PIPELINE_NAME": "zcta_2020_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.zcta_2020_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "zip%20code%20tabulation%20area", + "GEOGRAPHY": "zcta", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["zip_code_tabulation_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"zip_code_tabulation_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + + # Run CSV transform within kubernetes pod + zcta_2021_5yr = kubernetes_engine.GKEStartPodOperator( + task_id="zcta_2021_5yr", + startup_timeout_seconds=1000, + name="load_data", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + cluster_name="census-bureau-acs", + namespace="default", + image_pull_policy="Always", + image="{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}", + env_vars={ + "SOURCE_URL": "{{ var.json.census_bureau_acs.zcta_2021_5yr.source_url }}", + "SOURCE_FILE": "{{ var.json.census_bureau_acs.zcta_2021_5yr.source_file }}", + "TARGET_FILE": "{{ var.json.census_bureau_acs.zcta_2021_5yr.target_file }}", + "CHUNKSIZE": "{{ var.json.census_bureau_acs.chunk_size }}", + "PROJECT_ID": "{{ var.value.gcp_project }}", + "DATASET_ID": "{{ var.json.census_bureau_acs.zcta_2021_5yr.dataset_id }}", + "TABLE_ID": "{{ var.json.census_bureau_acs.zcta_2021_5yr.destination_table }}", + "SCHEMA_PATH": "{{ var.json.census_bureau_acs.zcta_2021_5yr.schema_path }}", + "PIPELINE_NAME": "zcta_2021_5yr", + "TARGET_GCS_BUCKET": "{{ var.value.composer_bucket }}", + "TARGET_GCS_PATH": "{{ var.json.census_bureau_acs.zcta_2021_5yr.target_gcs_path }}", + "YEAR_REPORT": "5", + "API_NAMING_CONVENTION": "zip%20code%20tabulation%20area", + "GEOGRAPHY": "zcta", + "REPORT_LEVEL": "national_level", + "CONCAT_COL_LIST": '["zip_code_tabulation_area"]', + "RENAME_MAPPINGS_LIST": '{"0":"name", "1":"KPI_Value", "2":"state", "3":"zip_code_tabulation_area"}', + "INPUT_CSV_HEADERS": '["0", "1", "2", "3","KPI_Name"]', + "DATA_DTYPES": '{"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"}', + "OUTPUT_CSV_HEADERS": '["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"]', + }, + resources={"request_ephemeral_storage": "8G", "request_cpu": "1"}, + ) + delete_cluster = kubernetes_engine.GKEDeleteClusterOperator( + task_id="delete_cluster", + project_id="{{ var.value.gcp_project }}", + location="us-central1-c", + name="census_bureau_acs", + ) + + ( + create_cluster + >> [ + cbsa_2019_1yr, + cbsa_2020_1yr, + cbsa_2021_1yr, + cbsa_2019_5yr, + cbsa_2020_5yr, + cbsa_2021_5yr, + censustract_2019_5yr, + censustract_2020_5yr, + censustract_2021_5yr, + congressionaldistrict_2019_1yr, + congressionaldistrict_2020_1yr, + congressionaldistrict_2021_1yr, + congressionaldistrict_2019_5yr, + congressionaldistrict_2020_5yr, + congressionaldistrict_2021_5yr, + county_2019_1yr, + county_2020_1yr, + county_2021_1yr, + county_2019_5yr, + county_2020_5yr, + county_2021_5yr, + place_2019_1yr, + place_2020_1yr, + place_2021_1yr, + place_2019_5yr, + place_2020_5yr, + place_2021_5yr, + puma_2019_1yr, + puma_2020_1yr, + puma_2021_1yr, + puma_2019_5yr, + puma_2020_5yr, + puma_2021_5yr, + schooldistrictelementary_2016_1yr, + schooldistrictelementary_2017_1yr, + schooldistrictelementary_2018_1yr, + schooldistrictelementary_2019_1yr, + schooldistrictelementary_2020_1yr, + schooldistrictelementary_2021_1yr, + schooldistrictelementary_2016_5yr, + schooldistrictelementary_2017_5yr, + schooldistrictelementary_2018_5yr, + schooldistrictelementary_2019_5yr, + schooldistrictelementary_2020_5yr, + schooldistrictelementary_2021_5yr, + schooldistrictsecondary_2016_1yr, + schooldistrictsecondary_2017_1yr, + schooldistrictsecondary_2018_1yr, + schooldistrictsecondary_2019_1yr, + schooldistrictsecondary_2020_1yr, + schooldistrictsecondary_2021_1yr, + schooldistrictsecondary_2016_5yr, + schooldistrictsecondary_2017_5yr, + schooldistrictsecondary_2018_5yr, + schooldistrictsecondary_2019_5yr, + schooldistrictsecondary_2020_5yr, + schooldistrictsecondary_2021_5yr, + schooldistrictunified_2016_1yr, + schooldistrictunified_2017_1yr, + schooldistrictunified_2018_1yr, + schooldistrictunified_2019_1yr, + schooldistrictunified_2020_1yr, + schooldistrictunified_2021_1yr, + schooldistrictunified_2016_5yr, + schooldistrictunified_2017_5yr, + schooldistrictunified_2018_5yr, + schooldistrictunified_2019_5yr, + schooldistrictunified_2020_5yr, + schooldistrictunified_2021_5yr, + state_2019_1yr, + state_2020_1yr, + state_2021_1yr, + state_2019_5yr, + state_2020_5yr, + state_2021_5yr, + zcta_2019_5yr, + zcta_2020_5yr, + zcta_2021_5yr, + ] + >> delete_cluster + ) diff --git a/datasets/census_bureau_acs/pipelines/census_bureau_acs/pipeline.yaml b/datasets/census_bureau_acs/pipelines/census_bureau_acs/pipeline.yaml new file mode 100644 index 000000000..982b2ab86 --- /dev/null +++ b/datasets/census_bureau_acs/pipelines/census_bureau_acs/pipeline.yaml @@ -0,0 +1,3568 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# requestations under the License. + +--- +resources: + + - type: bigquery_table + table_id: cbsa_2019_1yr + description: "CBSA 2019 1 year report table" + - type: bigquery_table + table_id: cbsa_2020_1yr + description: "CBSA 2020 1 year report table" + - type: bigquery_table + table_id: cbsa_2021_1yr + description: "CBSA 2021 1 year report table" + - type: bigquery_table + table_id: cbsa_2019_5yr + description: "CBSA 2019 5 year report table" + - type: bigquery_table + table_id: cbsa_2020_5yr + description: "CBSA 2020 5 year report table" + - type: bigquery_table + table_id: cbsa_2021_5yr + description: "CBSA 2021 5 year report table" + - type: bigquery_table + table_id: censustract_2019_5yr + description: "Census tract 2019 5 years report table" + - type: bigquery_table + table_id: censustract_2020_5yr + description: "Census tract 2020 5 years report table" + - type: bigquery_table + table_id: censustract_2021_5yr + description: "Census tract 2021 5 years report table" + - type: bigquery_table + table_id: congressionaldistrict_2019_1yr + description: "Congressional district 2019 1 year table" + - type: bigquery_table + table_id: congressionaldistrict_2020_1yr + description: "Congressional district 2020 1 year table" + - type: bigquery_table + table_id: congressionaldistrict_2021_1yr + description: "Congressional district 2021 1 year table" + - type: bigquery_table + table_id: congressionaldistrict_2019_5yr + description: "Congressional district 2019 5 years report table" + - type: bigquery_table + table_id: congressionaldistrict_2020_5yr + description: "Congressional district 2020 5 years report table" + - type: bigquery_table + table_id: congressionaldistrict_2020_5yr + description: "Congressional district 2020 5 years report table" + - type: bigquery_table + table_id: county_2019_1yr + description: "County 2019 1 year report table" + - type: bigquery_table + table_id: county_2020_1yr + description: "County 2020 1 year report table" + - type: bigquery_table + table_id: county_2021_1yr + description: "County 2021 1 year report table" + - type: bigquery_table + table_id: county_2019_5yr + description: "County 2019 5 years report table" + - type: bigquery_table + table_id: county_2020_5yr + description: "County 2020 5 years report table" + - type: bigquery_table + table_id: county_2021_5yr + description: "County 2021 5 years report table" + - type: bigquery_table + table_id: place_2019_1yr + description: "Place 2019 1 year report table" + - type: bigquery_table + table_id: place_2020_1yr + description: "Place 2020 1 year report table" + - type: bigquery_table + table_id: place_2021_1yr + description: "Place 2021 1 year report table" + - type: bigquery_table + table_id: place_2019_5yr + description: "Place 2019 5 year report table" + - type: bigquery_table + table_id: place_2020_5yr + description: "Place 2020 5 year report table" + - type: bigquery_table + table_id: place_2021_5yr + description: "Place 2021 5 year report table" + - type: bigquery_table + table_id: puma_2019_1yr + description: "PUMA 2019 1 year report table" + - type: bigquery_table + table_id: puma_2020_1yr + description: "PUMA 2020 1 year report table" + - type: bigquery_table + table_id: puma_2021_1yr + description: "PUMA 2021 1 year report table" + - type: bigquery_table + table_id: puma_2019_5yr + description: "PUMA 2019 5 year report table" + - type: bigquery_table + table_id: puma_2020_5yr + description: "PUMA 2020 5 year report table" + - type: bigquery_table + table_id: puma_2021_5yr + description: "PUMA 2021 5 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2016_1yr + description: "School district elementary 2016 1 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2017_1yr + description: "School district elementary 2017 1 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2018_1yr + description: "School district elementary 2018 1 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2019_1yr + description: "School district elementary 2019 1 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2020_1yr + description: "School district elementary 2020 1 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2021_1yr + description: "School district elementary 2021 1 year report table" + - type: bigquery_table + table_id: schooldistrictelementary_2016_5yr + description: "School district elementary 2016 5 years report table" + - type: bigquery_table + table_id: schooldistrictelementary_2017_5yr + description: "School district elementary 2017 5 years report table" + - type: bigquery_table + table_id: schooldistrictelementary_2018_5yr + description: "School district elementary 2018 5 years report table" + - type: bigquery_table + table_id: schooldistrictelementary_2019_5yr + description: "School district elementary 2019 5 years report table" + - type: bigquery_table + table_id: schooldistrictelementary_2020_5yr + description: "School district elementary 2020 5 years report table" + - type: bigquery_table + table_id: schooldistrictelementary_2021_5yr + description: "School district elementary 2021 5 years report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2016_1yr + description: "School district secondary 2016 1 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2017_1yr + description: "School district secondary 2017 1 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2018_1yr + description: "School district secondary 2018 1 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2019_1yr + description: "School district secondary 2019 1 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2020_1yr + description: "School district secondary 2020 1 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2021_1yr + description: "School district secondary 2021 1 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2016_5yr + description: "School district secondary 2016 5 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2017_5yr + description: "School district secondary 2017 5 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2018_5yr + description: "School district secondary 2018 5 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2019_5yr + description: "School district secondary 2019 5 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2020_5yr + description: "School district secondary 2020 5 year report table" + - type: bigquery_table + table_id: schooldistrictsecondary_2021_5yr + description: "School district secondary 2021 5 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2016_1yr + description: "School district unified 2016 1 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2017_1yr + description: "School district unified 2017 1 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2018_1yr + description: "School district unified 2018 1 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2019_1yr + description: "School district unified 2019 1 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2020_1yr + description: "School district unified 2020 1 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2021_1yr + description: "School district unified 2021 1 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2016_5yr + description: "School district unified 2016 5 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2017_5yr + description: "School district unified 2017 5 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2018_5yr + description: "School district unified 2018 5 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2019_5yr + description: "School district unified 2019 5 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2020_5yr + description: "School district unified 2020 5 year report table" + - type: bigquery_table + table_id: schooldistrictunified_2021_5yr + description: "School district unified 2021 5 year report table" + - type: bigquery_table + table_id: state_2019_1yr + description: "State 2019 1 year report table" + - type: bigquery_table + table_id: state_2020_1yr + description: "State 2020 1 year report table" + - type: bigquery_table + table_id: state_2021_1yr + description: "State 2021 1 year report table" + - type: bigquery_table + table_id: state_2019_5yr + description: "State 2019 5 year report table" + - type: bigquery_table + table_id: state_2020_5yr + description: "State 2020 5 year report table" + - type: bigquery_table + table_id: state_2021_5yr + description: "State 2021 5 year report table" + - type: bigquery_table + table_id: zcta_2019_5yr + description: "ZCTA 2019 5 years report table" + - type: bigquery_table + table_id: zcta_2020_5yr + description: "ZCTA 2020 5 years report table" + - type: bigquery_table + table_id: zcta_2021_5yr + description: "ZCTA 2021 5 years report table" + +dag: + airflow_version: 2 + initialize: + dag_id: census_bureau_acs + default_args: + owner: "Google" + depends_on_past: False + start_date: "2022-06-30" + max_active_runs: 1 + schedule_interval: "@monthly" + catchup: False + default_view: graph + + tasks: + - operator: "GKECreateClusterOperator" + args: + task_id: "create_cluster" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + body: + name: census-bureau-acs + initial_node_count: 30 + network: "{{ var.value.vpc_network }}" + node_config: + machine_type: e2-standard-4 + oauth_scopes: + - https://www.googleapis.com/auth/devstorage.read_write + - https://www.googleapis.com/auth/cloud-platform + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "cbsa_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "cbsa_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "metropolitan%20statistical%20area/micropolitan%20statistical%20area" + GEOGRAPHY: "cbsa" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["combined_statistical_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "cbsa_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.cbsa_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.cbsa_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.cbsa_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.cbsa_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.cbsa_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "cbsa_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.cbsa_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "metropolitan%20statistical%20area/micropolitan%20statistical%20area" + GEOGRAPHY: "cbsa" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["combined_statistical_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "cbsa_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "cbsa_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "metropolitan%20statistical%20area/micropolitan%20statistical%20area" + GEOGRAPHY: "cbsa" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["combined_statistical_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "cbsa_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.cbsa_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.cbsa_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.cbsa_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.cbsa_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.cbsa_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "cbsa_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "metropolitan%20statistical%20area/micropolitan%20statistical%20area" + GEOGRAPHY: "cbsa" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["combined_statistical_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "cbsa_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.cbsa_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.cbsa_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.cbsa_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.cbsa_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.cbsa_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "cbsa_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.cbsa_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "metropolitan%20statistical%20area/micropolitan%20statistical%20area" + GEOGRAPHY: "cbsa" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["combined_statistical_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "cbsa_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.cbsa_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.cbsa_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.cbsa_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.cbsa_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.cbsa_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "cbsa_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.cbsa_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "metropolitan%20statistical%20area/micropolitan%20statistical%20area" + GEOGRAPHY: "cbsa" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["combined_statistical_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"combined_statistical_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "censustract_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.censustract_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.censustract_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.censustract_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.censustract_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.censustract_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "censustract_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.censustract_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "tract" + GEOGRAPHY: "censustract" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","county","tract"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county","4":"tract" } + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "4", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "12G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "censustract_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.censustract_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.censustract_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.censustract_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.censustract_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.censustract_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "censustract_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.censustract_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "tract" + GEOGRAPHY: "censustract" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","county","tract"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county","4":"tract" } + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "4", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "12G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "censustract_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.censustract_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.censustract_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.censustract_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.censustract_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.censustract_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "censustract_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.censustract_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "tract" + GEOGRAPHY: "censustract" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","county","tract"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county","4":"tract" } + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "4", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "12G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "congressionaldistrict_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "congressionaldistrict_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "congressional%20district" + GEOGRAPHY: "congressionaldistrict" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","congressional_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "congressionaldistrict_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "congressionaldistrict_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "congressional%20district" + GEOGRAPHY: "congressionaldistrict" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","congressional_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "congressionaldistrict_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "congressionaldistrict_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "congressional%20district" + GEOGRAPHY: "congressionaldistrict" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","congressional_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "congressionaldistrict_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "congressionaldistrict_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.congressionaldistrict_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "congressional%20district" + GEOGRAPHY: "congressionaldistrict" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","congressional_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "congressionaldistrict_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "congressionaldistrict_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.congressionaldistrict_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "congressional%20district" + GEOGRAPHY: "congressionaldistrict" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","congressional_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "congressionaldistrict_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "congressionaldistrict_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.congressionaldistrict_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "congressional%20district" + GEOGRAPHY: "congressionaldistrict" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","congressional_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"congressional_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "county_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.county_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.county_2019_1yr.target_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.county_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.county_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.county_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "county_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.county_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "county" + GEOGRAPHY: "county" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","county"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "county_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.county_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.county_2020_1yr.target_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.county_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.county_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.county_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "county_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.county_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "county" + GEOGRAPHY: "county" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","county"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "county_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.county_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.county_2021_1yr.target_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.county_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.county_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.county_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "county_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.county_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "county" + GEOGRAPHY: "county" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","county"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "county_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.county_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.county_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.county_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.county_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.county_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "county_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.county_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "county" + GEOGRAPHY: "county" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","county"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "county_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.county_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.county_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.county_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.county_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.county_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "county_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.county_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "county" + GEOGRAPHY: "county" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","county"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "county_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.county_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.county_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.county_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.county_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.county_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "county_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.county_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "county" + GEOGRAPHY: "county" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","county"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"county"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "place_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.place_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.place_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.place_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.place_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.place_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "place_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.place_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "place" + GEOGRAPHY: "place" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","place"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"place"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "place_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.place_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.place_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.place_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.place_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.place_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "place_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.place_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "place" + GEOGRAPHY: "place" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","place"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"place"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "place_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.place_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.place_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.place_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.place_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.place_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "place_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.place_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "place" + GEOGRAPHY: "place" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","place"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"place"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "place_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.place_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.place_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.place_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.place_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.place_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "place_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.place_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "place" + GEOGRAPHY: "place" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","place"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"place"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "place_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.place_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.place_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.place_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.place_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.place_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "place_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.place_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "place" + GEOGRAPHY: "place" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","place"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"place"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "place_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.place_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.place_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.place_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.place_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.place_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "place_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.place_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "place" + GEOGRAPHY: "place" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","place"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"place"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "puma_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.puma_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.puma_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.puma_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.puma_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.puma_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.puma_2019_1yr.schema_path }}" + PIPELINE_NAME: "puma_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.puma_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "public%20use%20microdata%20area" + GEOGRAPHY: "puma" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","public_use_microdata_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "puma_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.puma_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.puma_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.puma_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.puma_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.puma_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.puma_2020_1yr.schema_path }}" + PIPELINE_NAME: "puma_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.puma_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "public%20use%20microdata%20area" + GEOGRAPHY: "puma" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","public_use_microdata_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "puma_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.puma_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.puma_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.puma_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.puma_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.puma_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.puma_2021_1yr.schema_path }}" + PIPELINE_NAME: "puma_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.puma_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "public%20use%20microdata%20area" + GEOGRAPHY: "puma" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","public_use_microdata_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "puma_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.puma_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.puma_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.puma_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.puma_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.puma_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.puma_2019_5yr.schema_path }}" + PIPELINE_NAME: "puma_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.puma_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "public%20use%20microdata%20area" + GEOGRAPHY: "puma" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","public_use_microdata_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "puma_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.puma_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.puma_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.puma_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.puma_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.puma_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.puma_2020_5yr.schema_path }}" + PIPELINE_NAME: "puma_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.puma_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "public%20use%20microdata%20area" + GEOGRAPHY: "puma" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","public_use_microdata_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "puma_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.puma_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.puma_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.puma_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.puma_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.puma_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.puma_2021_5yr.schema_path }}" + PIPELINE_NAME: "puma_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.puma_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "public%20use%20microdata%20area" + GEOGRAPHY: "puma" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state","public_use_microdata_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"public_use_microdata_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2016_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2016_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2017_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2017_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2018_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2018_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2016_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2016_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2016_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2017_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2017_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2017_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2018_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2018_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2018_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictelementary_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictelementary_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictelementary_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(elementary)" + GEOGRAPHY: "schooldistrictelementary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2016_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2016_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2017_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2017_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2018_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2018_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2016_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2016_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2016_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2017_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2017_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2017_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2018_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2018_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2018_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2019_5yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictsecondary_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictsecondary_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictsecondary_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(secondary)" + GEOGRAPHY: "schooldistrictsecondary" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2016_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2016_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2017_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2017_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2018_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2018_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2016_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2016_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2016_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2017_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2017_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2017_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2018_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2018_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2018_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "schooldistrictunified_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.schema_path }}" + PIPELINE_NAME: "schooldistrictunified_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.schooldistrictunified_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "school%20district%20(unified)" + GEOGRAPHY: "schooldistrictunified" + REPORT_LEVEL: "state_level" + CONCAT_COL_LIST: >- + ["state","school_district"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"school_district"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "state_2019_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.state_2019_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.state_2019_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.state_2019_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.state_2019_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.state_2019_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "state_2019_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.state_2019_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "state" + GEOGRAPHY: "state" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "state_2020_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.state_2020_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.state_2020_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.state_2020_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.state_2020_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.state_2020_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "state_2020_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.state_2020_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "state" + GEOGRAPHY: "state" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "state_2021_1yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.state_2021_1yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.state_2021_1yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.state_2021_1yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.state_2021_1yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.state_2021_1yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2021_1yr.schema_path }}" + PIPELINE_NAME: "state_2021_1yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.state_2021_1yr.target_gcs_path }}" + YEAR_REPORT: "1" + API_NAMING_CONVENTION: "state" + GEOGRAPHY: "state" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "state_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.state_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.state_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.state_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.state_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.state_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.state_2019_5yr.schema_path }}" + PIPELINE_NAME: "state_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.state_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "state" + GEOGRAPHY: "state" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "state_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.state_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.state_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.state_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.state_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.state_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "state_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.state_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "state" + GEOGRAPHY: "state" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "state_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.state_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.state_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.state_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.state_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.state_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.state_2021_5yr.schema_path }}" + PIPELINE_NAME: "state_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.state_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "state" + GEOGRAPHY: "state" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["state"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "zcta_2019_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.zcta_2019_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.zcta_2019_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.zcta_2019_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.zcta_2019_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.zcta_2019_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.zcta_2019_5yr.schema_path }}" + PIPELINE_NAME: "zcta_2019_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.zcta_2019_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "zip%20code%20tabulation%20area" + GEOGRAPHY: "zcta" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["zip_code_tabulation_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"zip_code_tabulation_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "zcta_2020_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.zcta_2020_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.zcta_2020_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.zcta_2020_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.zcta_2020_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.zcta_2020_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.cbsa_2019_1yr.schema_path }}" + PIPELINE_NAME: "zcta_2020_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.zcta_2020_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "zip%20code%20tabulation%20area" + GEOGRAPHY: "zcta" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["zip_code_tabulation_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"zip_code_tabulation_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEStartPodOperator" + description: "Run CSV transform within kubernetes pod" + args: + task_id: "zcta_2021_5yr" + startup_timeout_seconds: 1000 + name: "load_data" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + cluster_name: census-bureau-acs + namespace: "default" + image_pull_policy: "Always" + image: "{{ var.json.census_bureau_acs.container_registry.run_csv_transform_kub }}" + env_vars: + SOURCE_URL: "{{ var.json.census_bureau_acs.zcta_2021_5yr.source_url }}" + SOURCE_FILE: "{{ var.json.census_bureau_acs.zcta_2021_5yr.source_file }}" + TARGET_FILE: "{{ var.json.census_bureau_acs.zcta_2021_5yr.target_file }}" + CHUNKSIZE: "{{ var.json.census_bureau_acs.chunk_size }}" + PROJECT_ID: "{{ var.value.gcp_project }}" + DATASET_ID: "{{ var.json.census_bureau_acs.zcta_2021_5yr.dataset_id }}" + TABLE_ID: "{{ var.json.census_bureau_acs.zcta_2021_5yr.destination_table }}" + SCHEMA_PATH: "{{ var.json.census_bureau_acs.zcta_2021_5yr.schema_path }}" + PIPELINE_NAME: "zcta_2021_5yr" + TARGET_GCS_BUCKET: "{{ var.value.composer_bucket }}" + TARGET_GCS_PATH: "{{ var.json.census_bureau_acs.zcta_2021_5yr.target_gcs_path }}" + YEAR_REPORT: "5" + API_NAMING_CONVENTION: "zip%20code%20tabulation%20area" + GEOGRAPHY: "zcta" + REPORT_LEVEL: "national_level" + CONCAT_COL_LIST: >- + ["zip_code_tabulation_area"] + RENAME_MAPPINGS_LIST: >- + {"0":"name", "1":"KPI_Value", "2":"state", "3":"zip_code_tabulation_area"} + INPUT_CSV_HEADERS: >- + ["0", "1", "2", "3","KPI_Name"] + DATA_DTYPES: >- + {"geo_id": "str", "aggregate_travel_time_to_work": "str", "amerindian_including_hispanic": "str", "amerindian_pop": "str", "armed_forces": "str", "asian_including_hispanic": "str", "asian_male_45_54": "str", "asian_male_55_64": "str", "asian_pop": "str", "associates_degree": "str", "bachelors_degree": "str", "bachelors_degree_2": "str", "bachelors_degree_or_higher_25_64": "str", "black_including_hispanic": "str", "black_male_45_54": "str", "black_male_55_64": "str", "black_pop": "str", "children": "str", "children_in_single_female_hh": "str", "civilian_labor_force": "str", "commute_10_14_mins": "str", "commute_15_19_mins": "str", "commute_20_24_mins": "str", "commute_25_29_mins": "str", "commute_30_34_mins": "str", "commute_35_39_mins": "str", "commute_35_44_mins": "str", "commute_40_44_mins": "str", "commute_45_59_mins": "str", "commute_5_9_mins": "str", "commute_60_89_mins": "str", "commute_60_more_mins": "str", "commute_90_more_mins": "str", "commute_less_10_mins": "str", "commuters_16_over": "str", "commuters_by_bus": "str", "commuters_by_car_truck_van": "str", "commuters_by_carpool": "str", "commuters_by_public_transportation": "str", "commuters_by_subway_or_elevated": "str", "commuters_drove_alone": "str", "different_house_year_ago_different_city": "str", "different_house_year_ago_same_city": "str", "dwellings_10_to_19_units": "str", "dwellings_1_units_attached": "str", "dwellings_1_units_detached": "str", "dwellings_20_to_49_units": "str", "dwellings_2_units": "str", "dwellings_3_to_4_units": "str", "dwellings_50_or_more_units": "str", "dwellings_5_to_9_units": "str", "employed_agriculture_forestry_fishing_hunting_mining": "str", "employed_arts_entertainment_recreation_accommodation_food": "str", "employed_construction": "str", "employed_education_health_social": "str", "employed_finance_insurance_real_estate": "str", "employed_information": "str", "employed_manufacturing": "str", "employed_other_services_not_public_admin": "str", "employed_pop": "str", "employed_public_administration": "str", "employed_retail_trade": "str", "employed_science_management_admin_waste": "str", "employed_transportation_warehousing_utilities": "str", "employed_wholesale_trade": "str", "families_with_young_children": "str", "family_households": "str", "father_in_labor_force_one_parent_families_with_young_children": "str", "father_one_parent_families_with_young_children": "str", "female_10_to_14": "str", "female_15_to_17": "str", "female_18_to_19": "str", "female_20": "str", "female_21": "str", "female_22_to_24": "str", "female_25_to_29": "str", "female_30_to_34": "str", "female_35_to_39": "str", "female_40_to_44": "str", "female_45_to_49": "str", "female_50_to_54": "str", "female_55_to_59": "str", "female_5_to_9": "str", "female_60_to_61": "str", "female_62_to_64": "str", "female_65_to_66": "str", "female_67_to_69": "str", "female_70_to_74": "str", "female_75_to_79": "str", "female_80_to_84": "str", "female_85_and_over": "str", "female_female_households": "str", "female_pop": "str", "female_under_5": "str", "four_more_cars": "str", "gini_index": "str", "graduate_professional_degree": "str", "group_quarters": "str", "high_school_diploma": "str", "high_school_including_ged": "str", "hispanic_any_race": "str", "hispanic_male_45_54": "str", "hispanic_male_55_64": "str", "hispanic_pop": "str", "households": "str", "households_public_asst_or_food_stamps": "str", "households_retirement_income": "str", "housing_built_1939_or_earlier": "str", "housing_built_2000_to_2004": "str", "housing_built_2005_or_later": "str", "housing_units": "str", "housing_units_renter_occupied": "str", "in_grades_1_to_4": "str", "in_grades_5_to_8": "str", "in_grades_9_to_12": "str", "in_school": "str", "in_undergrad_college": "str", "income_100000_124999": "str", "income_10000_14999": "str", "income_125000_149999": "str", "income_150000_199999": "str", "income_15000_19999": "str", "income_200000_or_more": "str", "income_20000_24999": "str", "income_25000_29999": "str", "income_30000_34999": "str", "income_35000_39999": "str", "income_40000_44999": "str", "income_45000_49999": "str", "income_50000_59999": "str", "income_60000_74999": "str", "income_75000_99999": "str", "income_less_10000": "str", "income_per_capita": "str", "less_one_year_college": "str", "less_than_high_school_graduate": "str", "male_10_to_14": "str", "male_15_to_17": "str", "male_18_to_19": "str", "male_20": "str", "male_21": "str", "male_22_to_24": "str", "male_25_to_29": "str", "male_30_to_34": "str", "male_35_to_39": "str", "male_40_to_44": "str", "male_45_64_associates_degree": "str", "male_45_64_bachelors_degree": "str", "male_45_64_grade_9_12": "str", "male_45_64_graduate_degree": "str", "male_45_64_high_school": "str", "male_45_64_less_than_9_grade": "str", "male_45_64_some_college": "str", "male_45_to_49": "str", "male_45_to_64": "str", "male_50_to_54": "str", "male_55_to_59": "str", "male_5_to_9": "str", "male_60_to_61": "str", "male_62_to_64": "str", "male_65_to_66": "str", "male_67_to_69": "str", "male_70_to_74": "str", "male_75_to_79": "str", "male_80_to_84": "str", "male_85_and_over": "str", "male_male_households": "str", "male_pop": "str", "male_under_5": "str", "management_business_sci_arts_employed": "str", "married_households": "str", "masters_degree": "str", "median_age": "str", "median_income": "str", "median_rent": "str", "median_year_structure_built": "str", "million_dollar_housing_units": "str", "mobile_homes": "str", "mortgaged_housing_units": "str", "no_car": "str", "no_cars": "str", "nonfamily_households": "str", "not_hispanic_pop": "str", "not_in_labor_force": "str", "not_us_citizen_pop": "str", "occupation_management_arts": "str", "occupation_natural_resources_construction_maintenance": "str", "occupation_production_transportation_material": "str", "occupation_sales_office": "str", "occupation_services": "str", "occupied_housing_units": "str", "one_car": "str", "one_parent_families_with_young_children": "str", "one_year_more_college": "str", "other_race_pop": "str", "owner_occupied_housing_units": "str", "owner_occupied_housing_units_lower_value_quartile": "str", "owner_occupied_housing_units_median_value": "str", "owner_occupied_housing_units_upper_value_quartile": "str", "percent_income_spent_on_rent": "str", "pop_16_over": "str", "pop_25_64": "str", "pop_25_years_over": "str", "pop_5_years_over": "str", "pop_determined_poverty_status": "str", "pop_in_labor_force": "str", "population_1_year_and_over": "str", "population_3_years_over": "str", "poverty": "str", "rent_10_to_15_percent": "str", "rent_15_to_20_percent": "str", "rent_20_to_25_percent": "str", "rent_25_to_30_percent": "str", "rent_30_to_35_percent": "str", "rent_35_to_40_percent": "str", "rent_40_to_50_percent": "str", "rent_burden_not_computed": "str", "rent_over_50_percent": "str", "rent_under_10_percent": "str", "renter_occupied_housing_units_paying_cash_median_gross_rent": "str", "sales_office_employed": "str", "some_college_and_associates_degree": "str", "speak_only_english_at_home": "str", "speak_spanish_at_home": "str", "speak_spanish_at_home_low_english": "str", "three_cars": "str", "total_pop": "str", "two_cars": "str", "two_or_more_races_pop": "str", "two_parent_families_with_young_children": "str", "two_parents_father_in_labor_force_families_with_young_children": "str", "two_parents_in_labor_force_families_with_young_children": "str", "two_parents_mother_in_labor_force_families_with_young_children": "str", "two_parents_not_in_labor_force_families_with_young_children": "str", "unemployed_pop": "str", "vacant_housing_units": "str", "vacant_housing_units_for_rent": "str", "vacant_housing_units_for_sale": "str", "walked_to_work": "str", "white_including_hispanic": "str", "white_male_45_54": "str", "white_male_55_64": "str", "white_pop": "str", "worked_at_home": "str", "workers_16_and_over": "str"} + OUTPUT_CSV_HEADERS: >- + ["geo_id", "aggregate_travel_time_to_work", "amerindian_including_hispanic", "amerindian_pop", "armed_forces", "asian_including_hispanic", "asian_male_45_54", "asian_male_55_64", "asian_pop", "associates_degree", "bachelors_degree", "bachelors_degree_2", "bachelors_degree_or_higher_25_64", "black_including_hispanic", "black_male_45_54", "black_male_55_64", "black_pop", "children", "children_in_single_female_hh", "civilian_labor_force", "commute_10_14_mins", "commute_15_19_mins", "commute_20_24_mins", "commute_25_29_mins", "commute_30_34_mins", "commute_35_39_mins", "commute_35_44_mins", "commute_40_44_mins", "commute_45_59_mins", "commute_5_9_mins", "commute_60_89_mins", "commute_60_more_mins", "commute_90_more_mins", "commute_less_10_mins", "commuters_16_over", "commuters_by_bus", "commuters_by_car_truck_van", "commuters_by_carpool", "commuters_by_public_transportation", "commuters_by_subway_or_elevated", "commuters_drove_alone", "different_house_year_ago_different_city", "different_house_year_ago_same_city", "dwellings_10_to_19_units", "dwellings_1_units_attached", "dwellings_1_units_detached", "dwellings_20_to_49_units", "dwellings_2_units", "dwellings_3_to_4_units", "dwellings_50_or_more_units", "dwellings_5_to_9_units", "employed_agriculture_forestry_fishing_hunting_mining", "employed_arts_entertainment_recreation_accommodation_food", "employed_construction", "employed_education_health_social", "employed_finance_insurance_real_estate", "employed_information", "employed_manufacturing", "employed_other_services_not_public_admin", "employed_pop", "employed_public_administration", "employed_retail_trade", "employed_science_management_admin_waste", "employed_transportation_warehousing_utilities", "employed_wholesale_trade", "families_with_young_children", "family_households", "father_in_labor_force_one_parent_families_with_young_children", "father_one_parent_families_with_young_children", "female_10_to_14", "female_15_to_17", "female_18_to_19", "female_20", "female_21", "female_22_to_24", "female_25_to_29", "female_30_to_34", "female_35_to_39", "female_40_to_44", "female_45_to_49", "female_50_to_54", "female_55_to_59", "female_5_to_9", "female_60_to_61", "female_62_to_64", "female_65_to_66", "female_67_to_69", "female_70_to_74", "female_75_to_79", "female_80_to_84", "female_85_and_over", "female_female_households", "female_pop", "female_under_5", "four_more_cars", "gini_index", "graduate_professional_degree", "group_quarters", "high_school_diploma", "high_school_including_ged", "hispanic_any_race", "hispanic_male_45_54", "hispanic_male_55_64", "hispanic_pop", "households", "households_public_asst_or_food_stamps", "households_retirement_income", "housing_built_1939_or_earlier", "housing_built_2000_to_2004", "housing_built_2005_or_later", "housing_units", "housing_units_renter_occupied", "in_grades_1_to_4", "in_grades_5_to_8", "in_grades_9_to_12", "in_school", "in_undergrad_college", "income_100000_124999", "income_10000_14999", "income_125000_149999", "income_150000_199999", "income_15000_19999", "income_200000_or_more", "income_20000_24999", "income_25000_29999", "income_30000_34999", "income_35000_39999", "income_40000_44999", "income_45000_49999", "income_50000_59999", "income_60000_74999", "income_75000_99999", "income_less_10000", "income_per_capita", "less_one_year_college", "less_than_high_school_graduate", "male_10_to_14", "male_15_to_17", "male_18_to_19", "male_20", "male_21", "male_22_to_24", "male_25_to_29", "male_30_to_34", "male_35_to_39", "male_40_to_44", "male_45_64_associates_degree", "male_45_64_bachelors_degree", "male_45_64_grade_9_12", "male_45_64_graduate_degree", "male_45_64_high_school", "male_45_64_less_than_9_grade", "male_45_64_some_college", "male_45_to_49", "male_45_to_64", "male_50_to_54", "male_55_to_59", "male_5_to_9", "male_60_to_61", "male_62_to_64", "male_65_to_66", "male_67_to_69", "male_70_to_74", "male_75_to_79", "male_80_to_84", "male_85_and_over", "male_male_households", "male_pop", "male_under_5", "management_business_sci_arts_employed", "married_households", "masters_degree", "median_age", "median_income", "median_rent", "median_year_structure_built", "million_dollar_housing_units", "mobile_homes", "mortgaged_housing_units", "no_car", "no_cars", "nonfamily_households", "not_hispanic_pop", "not_in_labor_force", "not_us_citizen_pop", "occupation_management_arts", "occupation_natural_resources_construction_maintenance", "occupation_production_transportation_material", "occupation_sales_office", "occupation_services", "occupied_housing_units", "one_car", "one_parent_families_with_young_children", "one_year_more_college", "other_race_pop", "owner_occupied_housing_units", "owner_occupied_housing_units_lower_value_quartile", "owner_occupied_housing_units_median_value", "owner_occupied_housing_units_upper_value_quartile", "percent_income_spent_on_rent", "pop_16_over", "pop_25_64", "pop_25_years_over", "pop_5_years_over", "pop_determined_poverty_status", "pop_in_labor_force", "population_1_year_and_over", "population_3_years_over", "poverty", "rent_10_to_15_percent", "rent_15_to_20_percent", "rent_20_to_25_percent", "rent_25_to_30_percent", "rent_30_to_35_percent", "rent_35_to_40_percent", "rent_40_to_50_percent", "rent_burden_not_computed", "rent_over_50_percent", "rent_under_10_percent", "renter_occupied_housing_units_paying_cash_median_gross_rent", "sales_office_employed", "some_college_and_associates_degree", "speak_only_english_at_home", "speak_spanish_at_home", "speak_spanish_at_home_low_english", "three_cars", "total_pop", "two_cars", "two_or_more_races_pop", "two_parent_families_with_young_children", "two_parents_father_in_labor_force_families_with_young_children", "two_parents_in_labor_force_families_with_young_children", "two_parents_mother_in_labor_force_families_with_young_children", "two_parents_not_in_labor_force_families_with_young_children", "unemployed_pop", "vacant_housing_units", "vacant_housing_units_for_rent", "vacant_housing_units_for_sale", "walked_to_work", "white_including_hispanic", "white_male_45_54", "white_male_55_64", "white_pop", "worked_at_home", "workers_16_and_over"] + resources: + request_ephemeral_storage: "8G" + request_cpu: "1" + + - operator: "GKEDeleteClusterOperator" + args: + task_id: "delete_cluster" + project_id: "{{ var.value.gcp_project }}" + location: "us-central1-c" + name: census_bureau_acs + + + graph_paths: + - "create_cluster >> [ cbsa_2019_1yr, cbsa_2020_1yr, cbsa_2021_1yr, cbsa_2019_5yr, cbsa_2020_5yr, cbsa_2021_5yr, censustract_2019_5yr, censustract_2020_5yr, censustract_2021_5yr, congressionaldistrict_2019_1yr, congressionaldistrict_2020_1yr, congressionaldistrict_2021_1yr, congressionaldistrict_2019_5yr, congressionaldistrict_2020_5yr, congressionaldistrict_2021_5yr, county_2019_1yr, county_2020_1yr, county_2021_1yr, county_2019_5yr, county_2020_5yr, county_2021_5yr, place_2019_1yr, place_2020_1yr, place_2021_1yr, place_2019_5yr, place_2020_5yr, place_2021_5yr, puma_2019_1yr, puma_2020_1yr, puma_2021_1yr, puma_2019_5yr, puma_2020_5yr, puma_2021_5yr, schooldistrictelementary_2016_1yr, schooldistrictelementary_2017_1yr, schooldistrictelementary_2018_1yr, schooldistrictelementary_2019_1yr, schooldistrictelementary_2020_1yr, schooldistrictelementary_2021_1yr, schooldistrictelementary_2016_5yr, schooldistrictelementary_2017_5yr, schooldistrictelementary_2018_5yr, schooldistrictelementary_2019_5yr, schooldistrictelementary_2020_5yr, schooldistrictelementary_2021_5yr, schooldistrictsecondary_2016_1yr, schooldistrictsecondary_2017_1yr, schooldistrictsecondary_2018_1yr, schooldistrictsecondary_2019_1yr, schooldistrictsecondary_2020_1yr, schooldistrictsecondary_2021_1yr, schooldistrictsecondary_2016_5yr, schooldistrictsecondary_2017_5yr, schooldistrictsecondary_2018_5yr, schooldistrictsecondary_2019_5yr, schooldistrictsecondary_2020_5yr, schooldistrictsecondary_2021_5yr, schooldistrictunified_2016_1yr, schooldistrictunified_2017_1yr, schooldistrictunified_2018_1yr, schooldistrictunified_2019_1yr, schooldistrictunified_2020_1yr, schooldistrictunified_2021_1yr, schooldistrictunified_2016_5yr, schooldistrictunified_2017_5yr, schooldistrictunified_2018_5yr, schooldistrictunified_2019_5yr, schooldistrictunified_2020_5yr, schooldistrictunified_2021_5yr, state_2019_1yr, state_2020_1yr, state_2021_1yr, state_2019_5yr, state_2020_5yr, state_2021_5yr, zcta_2019_5yr, zcta_2020_5yr, zcta_2021_5yr ] >> delete_cluster"