diff --git a/demo/05_prep.sh b/demo/05_prep.sh index d1e5b0c..8cce2a7 100755 --- a/demo/05_prep.sh +++ b/demo/05_prep.sh @@ -16,9 +16,9 @@ export obsid=${obsid:-1341914000} # RAW # # ### # # check for raw files -export raw_glob=${outdir}/${obsid}/raw/${obsid}_2\*.fits -if ! eval ls -1 $raw_glob >/dev/null; then - echo "raw not present: $raw_glob , try ${SCRIPT_BASE}/02_download.sh" +export raw_pattern=${outdir}/${obsid}/raw/${obsid}_2\*.fits +if ! eval ls -1 $raw_pattern >/dev/null; then + echo "raw not present: $raw_pattern , try ${SCRIPT_BASE}/02_download.sh" exit 1 fi # check for metafits files @@ -41,22 +41,32 @@ export timeres_s=8 # time resolution to average to in seconds export edgewidth_khz=80 # edge width to flag on each coarse channel in kHz mkdir -p "${outdir}/${obsid}/prep" + +# Prep_uvfits is where we tell birli to write our preprocessed visibility files to. +# However, If the channels in the observation are non-contiguous, birli will output +# multiple files add a channel suffix to the output file names. +# So to test if the uvfits files exist, we need to look for a pattern. export prep_uvfits="${outdir}/${obsid}/prep/birli_${obsid}.uvfits" -export prepqa="${prep_uvfits%%.uvfits}_qa.json" +export prep_uvfits_pattern=${outdir}/${obsid}/prep/birli_${obsid}\*.uvfits -set -eu -if [[ ! -f $prep_uvfits ]]; then - # --provided-chan-ranges - birli ${birli_args:-} \ +# since we don't expect the uvfits files to exist the first time around, 2>/dev/null silences the warning +if ! eval ls -1 $prep_uvfits_pattern 2>/dev/null; then + echo "running birli on $raw_pattern" \ + $([[ -n "${edgewidth_khz:-}" ]] && echo " edge width ${edgewidth_khz}kHz") \ + $([[ -n "${freqres_khz:-}" ]] && echo " freq res ${freqres_khz}kHz") \ + $([[ -n "${timeres_s:-}" ]] && echo " time res ${timeres_s}s") \ + ; + /software/projects/mwaeor/dev/setonix/2024.05/software/linux-sles15-zen3/gcc-12.2.0/birli-main-soerdr3lmpn6rottzxzdxb2iehej73eu/bin/birli ${birli_args:-} \ -m "${metafits}" \ $([[ -n "${edgewidth_khz:-}" ]] && echo "--flag-edge-width ${edgewidth_khz}") \ $([[ -n "${freqres_khz:-}" ]] && echo "--avg-freq-res ${freqres_khz}") \ $([[ -n "${timeres_s:-}" ]] && echo "--avg-time-res ${timeres_s}") \ -u "${prep_uvfits}" \ - $raw_glob \ + $raw_pattern \ $@ + # -M "${prep_uvfits%%.uvfits}.ms" \ else - echo "prep_uvfits $prep_uvfits exists, skipping birli" + echo "prep_uvfits $prep_uvfits_pattern exists, skipping birli" fi # ####### # @@ -65,19 +75,25 @@ fi # DEMO: use mwa_qa for quality analysis of preprocessed uvfits # details: https://github.com/d3v-null/mwa_qa (my fork of https://github.com/Chuneeta/mwa_qa/ ) -if [[ ! -f "$prepqa" ]]; then - run_prepvisqa.py $prep_uvfits $metafits --out $prepqa -else - echo "prepqa=$prepqa exists, skipping run_prepvisqa.py" -fi +# loop over all the preprocessed visibility files birli produced +eval ls -1 $prep_uvfits_pattern | while read -r prep_uvfits; do + export prep_uvfits + # obsid plus any channel suffix that birli might add. + export obs_ch=${prep_uvfits##*/birli_} + export obs_ch=${obs_ch%%.uvfits} + + # store prepqa relative to this uvfits + export prepqa="${prep_uvfits%%.uvfits}_qa.json" + if [[ ! -f "$prepqa" ]]; then + run_prepvisqa.py $prep_uvfits $metafits --out $prepqa + fi -# DEMO: extract bad antennas from prepqa json with jq -# - both of the provided observations pass QA, so no bad antennas are reported -prep_bad_ants=$(jq -r $'.BAD_ANTS|join(" ")' $prepqa) + # DEMO: extract bad antennas from prepqa json with jq + prep_bad_ants=$(jq -r $'.BAD_ANTS|join(" ")' $prepqa) -# DEMO: plot the prep qa results -# - RMS plot: RMS of all autocorrelation values for each antenna -# - zscore: -plot_prepvisqa.py $prepqa --save --out ${prep_uvfits%%.uvfits} + # DEMO: plot the prep qa results + # - RMS plot: RMS of all autocorrelation values for each antenna + plot_prepvisqa.py $prepqa --save --out ${prep_uvfits%%.uvfits} -echo $obsid $prep_bad_ants + echo $obs_ch $prep_bad_ants +done \ No newline at end of file diff --git a/demo/06_cal.sh b/demo/06_cal.sh index 650419f..9384462 100755 --- a/demo/06_cal.sh +++ b/demo/06_cal.sh @@ -23,31 +23,23 @@ if [[ ! -f "$metafits" ]]; then fi # check preprocessed visibility and qa files exist from previous steps -export prep_uvfits="${outdir}/${obsid}/prep/birli_${obsid}.uvfits" -if [ ! -f "$prep_uvfits" ]; then - echo "prep_uvfits=$prep_uvfits does not exist. trying 05_prep.sh" +# - birli adds a channel suffix when processing observations with non-contiguous coarse channels. +# - if the files we need are missing, then run 05_prep. +export prep_uvfits_pattern=${outdir}/${obsid}/prep/birli_${obsid}\*.uvfits +if ! eval ls -1 $prep_uvfits_pattern >/dev/null; then + echo "prep_uvfits $prep_uvfits_pattern does not exist. trying 05_prep.sh" $SCRIPT_BASE/05_prep.sh fi -export prepqa="${prep_uvfits%%.uvfits}_qa.json" -if [[ ! -f "$prepqa" ]]; then - echo "warning: prepqa=$prepqa does not exist. trying 05_prep.sh" +export prep_qa_pattern="${outdir}/${obsid}/prep/birli_${obsid}*_qa.json" +if ! eval ls -1 $prep_qa_pattern >/dev/null; then + echo "warning: prepqa $prep_qa_pattern does not exist. trying 05_prep.sh" $SCRIPT_BASE/05_prep.sh fi -if [[ -f "$prepqa" ]]; then - prep_bad_ants=$(jq -r '.BAD_ANTS|join(" ")' "$prepqa") - export prep_bad_ants -else - export prep_bad_ants="" -fi # ### # # CAL # # ### # # direction independent calibration with hyperdrive -mkdir -p "${outdir}/${obsid}/cal" -export hyp_soln="${outdir}/${obsid}/cal/hyp_soln_${obsid}.fits" -export cal_ms="${outdir}/${obsid}/cal/hyp_cal_${obsid}.ms" -export model_ms="${outdir}/${obsid}/cal/hyp_model_${obsid}.ms" # DEMO: generate calibration solutions from preprocessed visibilities # details: https://mwatelescope.github.io/mwa_hyperdrive/user/di_cal/intro.html @@ -59,72 +51,106 @@ if [[ -n "${gpus:-}" ]]; then dical_args="" fi +mkdir -p "${outdir}/${obsid}/cal" set -eu - -if [[ ! -f "$hyp_soln" ]]; then - echo "calibrating with sourcelist $srclist" - hyperdrive di-calibrate ${dical_args:-} \ - --data "$metafits" "$prep_uvfits" \ - --source-list "$srclist" \ - --outputs "$hyp_soln" \ - $([[ -n "${prep_bad_ants:-}" ]] && echo --tile-flags $prep_bad_ants) -else - echo "hyp_soln $hyp_soln exists, skipping hyperdrive di-calibrate" -fi - -# plot solutions file -if [[ ! -f "${hyp_soln%%.fits}_phases.png" ]]; then - hyperdrive solutions-plot \ - -m "$metafits" \ - --no-ref-tile \ - --max-amp 1.5 \ - --output-directory "${outdir}/${obsid}/cal" \ - "$hyp_soln" -else - echo "phases_png ${hyp_soln%%.fits}_phases.png exists, skipping hyperdrive solutions-plot" -fi - -# ###### # -# CAL QA # -# ###### # -# DEMO: use mwa_qa to check calibration solutions - -export calqa="${hyp_soln%%.fits}_qa.json" - -if [[ ! -f "$calqa" ]]; then - echo "running calqa on solutions $hyp_soln" - run_calqa.py --pol X --out "$calqa" "$hyp_soln" "$metafits" -else - echo "calqa $calqa exists, skipping run_calqa.py" -fi - -# plot the cal qa results -plot_calqa.py "$calqa" --save --out "${hyp_soln%%.fits}" - -# extract the percentage of channels that converged -cal_pct_nonconvg=$(jq -r $'.PERCENT_NONCONVERGED_CHS|tonumber|round' "$calqa") -export cal_pct_nonconvg - -if [[ $cal_pct_nonconvg -ge 95 ]]; then - echo "calibration failed, $cal_pct_nonconvg% of channels did not converge. hint: try a different sky model in demo/00_env.sh" - exit 1 -fi - -# extract bad antennas from calqa json with jq -cal_bad_ants=$(jq -r $'.BAD_ANTS|join(" ")' "$calqa") -export cal_bad_ants - -echo "deliberately disabling cal bad ants for the first round :)" -export cal_bad_ants="" - -# apply calibration solutions to preprocessed visibilities -# details: https://mwatelescope.github.io/mwa_hyperdrive/user/solutions_apply/intro.html -if [[ ! -d "$cal_ms" ]]; then - hyperdrive apply ${apply_args:-} \ - --data "$metafits" "$prep_uvfits" \ - --solutions "$hyp_soln" \ - --outputs "$cal_ms" \ - $([[ -n "${cal_bad_ants:-}" ]] && echo --tile-flags $cal_bad_ants) -else - echo "cal_ms $cal_ms exists, skipping hyperdrive apply" -fi +# loop over all the preprocessed files +eval ls -1 $prep_uvfits_pattern | while read -r prep_uvfits; do + export prep_uvfits; + + # find prepqa relative to this uvfits file + export prepqa="${prep_uvfits%%.uvfits}_qa.json" + if [[ -f "$prepqa" ]]; then + prep_bad_ants=$(jq -r '.BAD_ANTS|join(" ")' "$prepqa") + export prep_bad_ants + else + export prep_bad_ants="" + fi + + # store calibration outputs for the prepqa file in the sibling cal/ folder + # e.g. for prep_uvfits=a/b/prep/birli_X_chY.uvfits, parent=a/b, obs=X_chY + export parent=${prep_uvfits%/*} + export parent=${parent%/*} + export obs=${prep_uvfits##*/birli_} + export obs=${obs%.uvfits} + export hyp_soln="${parent}/cal/hyp_soln_${obs}.fits" + export cal_ms="${parent}/cal/hyp_cal_${obs}.ms" + export model_ms="${parent}/cal/hyp_model_${obs}.ms" + + if [[ ! -f "$hyp_soln" ]]; then + echo "calibrating with sourcelist $srclist" + hyperdrive di-calibrate ${dical_args:-} \ + --data "$metafits" "$prep_uvfits" \ + --source-list "$srclist" \ + --outputs "$hyp_soln" \ + $([[ -n "${prep_bad_ants:-}" ]] && echo --tile-flags $prep_bad_ants) + # TODO: --cpu if login node + else + echo "hyp_soln $hyp_soln exists, skipping hyperdrive di-calibrate" + fi + + # plot solutions file + if [[ ! -f "${hyp_soln%%.fits}_phases.png" ]]; then + hyperdrive solutions-plot \ + -m "$metafits" \ + --no-ref-tile \ + --max-amp 1.5 \ + --output-directory "${outdir}/${obsid}/cal" \ + "$hyp_soln" + else + echo "phases_png ${hyp_soln%%.fits}_phases.png exists, skipping hyperdrive solutions-plot" + fi + + # ###### # + # CAL QA # + # ###### # + # DEMO: use mwa_qa to check calibration solutions + + export calqa="${hyp_soln%%.fits}_qa.json" + + if [[ ! -f "$calqa" ]]; then + echo "running calqa on solutions $hyp_soln" + run_calqa.py --pol X --out "$calqa" "$hyp_soln" "$metafits" + else + echo "calqa $calqa exists, skipping run_calqa.py" + fi + + # plot the cal qa results + plot_calqa.py "$calqa" --save --out "${hyp_soln%%.fits}" + + # extract the percentage of channels that converged + cal_pct_nonconvg=$(jq -r $'.PERCENT_NONCONVERGED_CHS|tonumber|round' "$calqa") + export cal_pct_nonconvg + + if [[ $cal_pct_nonconvg -ge 95 ]]; then + echo "calibration failed, $cal_pct_nonconvg% of channels did not converge. hint: try a different sky model in demo/00_env.sh" + continue + fi + + # extract bad antennas from calqa json with jq + cal_bad_ants=$(jq -r $'.BAD_ANTS|join(" ")' "$calqa") + export cal_bad_ants + + # echo "deliberately disabling cal bad ants for the first round :)" + # export cal_bad_ants="" + + # apply calibration solutions to preprocessed visibilities + # details: https://mwatelescope.github.io/mwa_hyperdrive/user/solutions_apply/intro.html + if [[ ! -d "$cal_ms" ]]; then + hyperdrive apply ${apply_args:-} \ + --data "$metafits" "$prep_uvfits" \ + --solutions "$hyp_soln" \ + --outputs "$cal_ms" \ + $([[ -n "${cal_bad_ants:-}" ]] && echo --tile-flags $cal_bad_ants) + else + echo "cal_ms $cal_ms exists, skipping hyperdrive apply" + fi +done + +# TODO phase fits, for now you need to singularity exec -B$PWD -B${outdir:-$PWD} -W$PWD --cleanenv docker://mwatelescope/mwa-demo:latest /bin/bash +cat <