create-parquet.yml

name: Create parquet

on:
  workflow_dispatch:

jobs:
  create-parquet:
    if: github.repository_owner == 'european-modelling-hubs'
    runs-on: ubuntu-latest
    env:
      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
    steps:
    - uses: actions/checkout@v3

    - uses: r-lib/actions/setup-r@v2
      with:
        install-r: false
        use-public-rspm: true

    - name: Install R system requirements
      run: |
        sudo apt-get update
        sudo apt-get install libcurl4-openssl-dev

    - uses: r-lib/actions/setup-renv@v2

    - name: Generate parquet file
      run: |
        library(readr)
        library(arrow)
        library(dplyr)
        library(lubridate)

        locations <- read_csv(
          here::here("data-locations", "locations_eu.csv")
        ) |>
          dplyr::select(location, location_name, population)

        raw_forecasts <- arrow::open_dataset(
          here::here("data-processed"),
          format = "csv",
          partitioning = schema(model = string()),
          hive_style = FALSE,
          col_types = schema(
            forecast_date = date32(),
            target = string(),
            target_end_date = date32(),
            location = string(),
            type = string(),
            quantile = float32(),
            value = float32()
          )
        ) |>
          # Ideally, we would handle this with separate() but it's not yet implemented
          # in arrow
          dplyr::mutate(
            horizon = as.integer(gsub("^(\\d+) .*", "\\1", target)),
            target_variable = gsub(".* (\\w+ \\w+)$", "\\1", target),
            .keep = "unused"
          ) |>
          dplyr::filter(
            forecast_date >= ymd("2021-03-08")
          ) |>
          dplyr::rename(prediction = value) |>
          dplyr::left_join(locations) |>
          # set forecast date to corresponding submission date
          mutate(
            forecast_date = ceiling_date(forecast_date, "week", week_start = 3)
          )

        arrow::write_parquet(raw_forecasts, "covid19-forecast-hub-europe.parquet")
      shell: Rscript {0}

    - uses: actions/upload-artifact@v3
      with:
        name: parquet-dataset
        path: covid19-forecast-hub-europe.parquet