camera_cards

#!/usr/bin/env bash

# a script to turn raw camera cards into SIPs

# set script variables
DEPENDENCIES=(ffmpeg md5sum tree mediaconch exiftool ffprobe)
OP="${USER}"
AIP="YES"
TAR="NO"

# define functions
_cleanup(){
    _report -rt "Process cancelled" # tell operator that script is ending
    exit 1
}

_generate_metadata(){
    _report -g "Generating metadata reports..."
    # create tree of all files in package
    TREE="${METADATA_REPORTS_DIR}/${MEDIAID}_tree.txt"
    tree -DaNs --du --timefmt "%Y-%m-%dT%H:%M:%SZ" "${CAMERA_CARD_DIR[@]}" > "${TREE}"
    # create mediainfo, exiftool, ffprobe reports
    while read -r FILE ; do
        FN="$(basename "${FILE}")"
        MEDIAINFO_OUTPUT="${METADATA_REPORTS_DIR}/${FN}_mediainfo.txt"
        EXIFTOOL_OUTPUT="${METADATA_REPORTS_DIR}/${FN}_exiftool.txt"
        FFPROBE_OUTPUT="${METADATA_REPORTS_DIR}/${FN}_ffprobe.xml"
        mediaconch -mi -ft "${FILE}" >> "${MEDIAINFO_OUTPUT}"
        exiftool "${FILE}" >> "${EXIFTOOL_OUTPUT}"
        ffprobe -v 0 "${FILE}" -show_format -show_streams -show_data -show_error -show_versions -show_chapters -noprivate -of xml="q=1:x=1" > "${FFPROBE_OUTPUT}"
    done < <(_read_inventory -f ",video,\|,audio," "FILE_NAME")
    _report -g "Metadata reports generated for ${MEDIAID}"
}

_maketemp(){
    mktemp -t "$(basename "${0}")"
    if [ "${?}" -ne 0 ]; then
        _report -rt "${0}: Can't create temp file, exiting..."
        exit 1
    fi
}

_report(){
    local RED="$(tput setaf 1)"   # Red      - For Warnings
    local GREEN="$(tput setaf 2)" # Green    - For Declarations
    local BLUE="$(tput setaf 4)"  # Blue     - For Questions
    local NC="$(tput sgr0)"       # No Color
    local COLOR=""
    local STARTMESSAGE=""
    local ECHOOPT=""
    OPTIND=1
    while getopts ":bgrstn" OPT; do
        case "${OPT}" in
            b) COLOR="${BLUE}" ;;                         # question mode, use color blue
            g) COLOR="${GREEN}" ;;                        # declaration mode, use color green
            r) COLOR="${RED}" ;;                          # warning mode, use color red
            s) STARTMESSAGE+=([$(basename "${0}")] ) ;;   # prepend scriptname to the message
            t) STARTMESSAGE+=($(date +%FT%T) '- ' ) ;;    # prepend timestamp to the message
            n) ECHOOPT="-n" ;;                            # to avoid line breaks after echo
        esac
    done
    shift $(( OPTIND - 1 ))
    MESSAGE="${1}"
    echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[@]} ${MESSAGE}${NC}"
}

_usage(){
cat <<EOF
$(basename "${0}")
 1. By default, this script will detect a camera card file/directory structure and transform the original camera card files into a usable AIP. The script will concatenate video files into a single file, move important metadata files into a new directory structure, and create a log of these changes."
  Current camera cards recognized:
      - AVCHD (e.g. Canon C100)
      - Canon XF (e.g. Canon C300)
      - P2 (Panasonic)
      - XAVC (Sony)
      - XDCAM EX (e.g. Canon XF100)
2. If you prefer to package files as-is to preserve your original camera directories, you can choose to compress your directory structure into a tarball. To select this option, use the '-t' flag.
If your camera card structure is not recognized, the script will prompt you to choose one or both of these strategies. If you want to create an AIP with concatenated video files and restructured metadata directories, it is recommended that you review the output to make sure the script was able to process your camera files properly.

Your input package is the top-level directory of your camera card, which should be mounted on your computer or provided as a directory.
Your output package will be named after the MEDIAID you supply and delivered to the AIP destination set in nmaahcmmconfig (AIP_DESTINATION).

The AIP_DESTINATION can be set via nmaahcconfig, or via the -o options, or the script with request it.

Dependencies: ${DEPENDENCIES[@]}

Usage: $(basename "${0}") -m MEDIAID [ -o AIP_DESTINATION ] [-tca] /path/to/input/camera_card_directory [ /path/to/input/camera_card_directory_2... ]
  -m MEDIAID (type media id for final package, e.g. SC0001_20190101_SMITH_VHS_01)
  -N <parts> (List the numbers of the clips that should be packaged.
              Multiple clip numbers may be listed as comma-delimited ranges; for
              instance "-N '1,3-5'" would output the 1st, 3rd, 4th, and 5th clips and not others.)
  -f FORMAT (choose the output format (as a file extension) for the rewrapping process, such as 'mxf' or 'mkv'. The default is set to match the input card set. Experimental.)
  -o AIP_DESTINATION (the output directory for the package); if not supplied, script will look for it in nmaahcconfig
  -t tar camera files and folders into a tarball, compressed with gzip
  -a create packaged AIP; this is the default option, but you can use this flag to specify you want an AIP alongside a tarball (e.g. '-ta')
  -n Do not actually package anything, but just show information about the input and what would happen.
  -h display this help

EOF
    exit
}

_write_package_log(){
    if [[ -z "${LOG}" ]] ; then
        _report -r "Error, can not write to log, ingest log not yet created"
        exit
    fi
    if [[ ! -w "${METADATADIR}" ]] ; then
        _report -r "Error, can not write to log as the metadata directory, ${METADATADIR}, is not writable."
        exit
    fi
    KEY="${1}"                              # key is first string passed to function
    VALUE="${2}"                            # value is variable passed to function
    OPTIND=1
    while getopts ":t" OPT; do
        case "${OPT}" in
            t) KEY="${2}" && VALUE="$(date +%FT%T)" ;;    # value is day and time in format YYYY-MM-DDTHH:MM:
        esac
    done
    # need to add yaml style escaping
    if [[ -n "${VALUE}" ]] ; then
        echo "${KEY}: ${VALUE}" >> "${LOG}"
    fi
}

_read_inventory(){
    OPTIND=1
    while getopts ":f:" OPT; do
        case "${OPT}" in
            f) FILTER="${OPTARG}" ;;
        esac
    done
    shift $(( OPTIND - 1 ))
    COL_NAME="${1}"
    if [[ -f "${FILE_INVENTORY}" ]] ; then
        COL_NUM="$(head -n1 "${FILE_INVENTORY}" | awk -v RS=',' "/${COL_NAME}/{print NR; exit}")"
        if [[ -n "${COL_NAME}" ]] ; then
            if [[ -z "${COL_NUM}" ]] ; then
                _report -w "The _read_inventory function was called to find a column called ${COL_NAME} but it isn't found in the inventory."
            fi
            if [[ -n "${FILTER}" ]] ; then
                tail -n +2 "${FILE_INVENTORY}" | grep "$FILTER" | cut -d "," -f "${COL_NUM}"
            else
                tail -n +2 "${FILE_INVENTORY}" | cut -d "," -f "${COL_NUM}"
            fi
        else
            tail -n +2 "${FILE_INVENTORY}" | grep "$FILTER"
        fi
    else
        _report -w "The _read_inventory function was called but there's no inventory at ${FILE_INVENTORY}."
    fi
}

_make_inventory(){
    _catalog_video(){
        FILE="${1}"
        IFS="," read -r Z DURATION COMPANY_NAME PRODUCT_NAME MOD_TS TC < <(ffprobe -sexagesimal -v 0 "${FILE}" -show_entries format=duration:format_tags=company_name,product_name,modification_date,timecode -of csv)
        IFS="," read -r Z AUDIO_CODEC AUDIO_SAMPLE_FMT < <(ffprobe -sexagesimal -v 0 "${FILE}" -select_streams a:0 -show_entries stream=codec_name,sample_fmt -of csv | grep "^stream,")
        echo "${CARD_NUM},${CAMERA_CARD_TYPE},${VIDEO_NUM},${FILE_NUM},video,${DURATION},${COMPANY_NAME},${PRODUCT_NAME},${MOD_TS},${TC},${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
    }
    echo "CARD_NUM,CARD_TYPE,VIDEO_NUM,FILE_NUM,MEDIA_TYPE,DURATION,COMPANY_NAME,PRODUCT_NAME,MOD_TS,TIMECODE,FILE_NAME,AUDIO_CODEC,AUDIO_SAMPLE_FMT" > "${FILE_INVENTORY}"
    CARD_NUM=1
    VIDEO_NUM=1
    FILE_NUM=1
    for CARD in "${CAMERA_CARD_DIR[@]}" ; do
        _identify_card_type "${CARD}"
        if [[ "${CAMERA_CARD_TYPE}" == "P2" ]] ; then
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" -path "*CONTENTS/VIDEO*" -iname "*.${EXTENSION}" | sort)
            while read FILE ; do
                echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},audio,${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
                ((FILE_NUM++))
            done < <(find "${CARD}" -path "*CONTENTS/AUDIO*" -iname "*.${EXTENSION}" | sort)
        elif [[ "${CAMERA_CARD_TYPE}" == "GENERAL" ]] ; then
            while read FILE ; do
                if [[ -n $(mediainfo --Inform="General;%VideoCount%" "${FILE}") ]] ; then
                    IFS="," read -r Z DURATION COMPANY_NAME PRODUCT_NAME MOD_TS TC < <(ffprobe -sexagesimal -v 0 "${FILE}" -show_entries format=duration:format_tags=company_name,product_name,modification_date,timecode -of csv)
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},${VIDEO_NUM},${FILE_NUM},video,${DURATION},${COMPANY_NAME},${PRODUCT_NAME},${MOD_TS},${TC},${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
                    ((VIDEO_NUM++))
                    ((FILE_NUM++))
                elif [[ -n $(mediainfo --Inform="General;%AudioCount%" "${FILE}") ]] ; then
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},audio,,,,,,${FILE},${AUDIO_CODEC},${AUDIO_SAMPLE_FMT}" >> "${FILE_INVENTORY}"
                    ((FILE_NUM++))
                elif [[ -f "${FILE}" ]] ; then
                    echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},metadata,,,,,,${FILE},," >> "${FILE_INVENTORY}"
                    ((FILE_NUM++))
                fi
                ((FILE_NUM++))
            done < <(find "${CARD}" -type f | sort)
        elif [[ "${CAMERA_CARD_TYPE}" == "XAVC" ]] ; then
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" -iname "*.${EXTENSION}" ! -path "*/PROAV/*" ! -path "*/Sub/*" -type f | sort)
        elif [[ "${CAMERA_CARD_TYPE}" ==  "MXF" ]] ; then
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" -iname "*.${EXTENSION}" -type f | sort)
        else
            while read FILE ; do
                _catalog_video "${FILE}"
                ((VIDEO_NUM++))
                ((FILE_NUM++))
            done < <(find "${CARD}" -iname "*.${EXTENSION}" -type f | sort)
        fi
        while read FILE ; do
            echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},metadata,,,,,,${FILE},," >> "${FILE_INVENTORY}"
            ((FILE_NUM++))
        done < <(find "${CARD}" -type f "${RSYNC_OPTIONS[@]}" | sort)
        # grab everything else
        while read FILE ; do
            if [[ -z "$(grep ",${FILE}," "${FILE_INVENTORY}")" ]] ; then
                echo "${CARD_NUM},${CAMERA_CARD_TYPE},0,${FILE_NUM},other,,,,,,${FILE},," >> "${FILE_INVENTORY}"
                ((FILE_NUM++))
            fi
        done < <(find "${CARD}" ! -name ".DS_Store" -type f | sort)
        ((CARD_NUM++))
    done
}

_convert_hhmmssmmm2ns(){
    TS="${1}"
    echo "${TS}" | awk -F ":" '{ printf "%8i\n", (($1 * 3600) + ($2 * 60) + $3 ) * 100000}'
}

_print_table(){
    # this prints a table of files but also creates a chapter list at the same time
    CHAPTER_LIST="$(_maketemp).ffmetadata"
    echo ";FFMETADATA1" > "${CHAPTER_LIST}"
    START_VALUE=0
    END_VALUE=0
    echo
    echo "Card#   Index#  File Name                    Modification Date     Duration            Timecode"
    while IFS=, read -r CARD_NUM CARD_TYPE VIDEO_NUM FILE_NUM MEDIA_TYPE DURATION COMPANY_NAME PRODUCT_NAME MOD_TS TIMECODE FILE_NAME AUDIO_CODEC AUDIO_SAMPLE_FMT ; do
        FILEBASENAME="$(basename "${FILE_NAME}")"
        if [[ -z "${CLIP_SELECTION}" || "$(_check_if_in_list "${VIDEO_NUM}" "${CLIP_SELECTION}")" == "y" ]] ; then
            # this clip is selected for the output
            VIDEO_NUM_EXT="${VIDEO_NUM}*"
            echo "[CHAPTER]" >> "${CHAPTER_LIST}"
            echo "TIMEBASE=1/100000" >> "${CHAPTER_LIST}"
            DURATION_S="$(_convert_hhmmssmmm2ns "${DURATION}")"
            END_VALUE=$((END_VALUE + DURATION_S))
            echo "START=${START_VALUE}" >> "${CHAPTER_LIST}"
            echo "END=${END_VALUE}" >> "${CHAPTER_LIST}"
            START_VALUE=$((START_VALUE + DURATION_S))
            if [[ -n "${MOD_TS}" && -n "${TIMECODE}" ]] ; then
                echo "title=${FILEBASENAME}: ${TIMECODE} - ${MOD_TS}" >> "${CHAPTER_LIST}"
            elif [[ -n "${MOD_TS}" ]] ; then
                echo "title=${FILEBASENAME}: ${MOD_TS}" >> "${CHAPTER_LIST}"
            elif [[ -n "${TIMECODE}" ]] ; then
                echo "title=${FILEBASENAME}: ${TIMECODE}" >> "${CHAPTER_LIST}"
            else
                echo "title=${FILEBASENAME}" >> "${CHAPTER_LIST}"
            fi
        else
            VIDEO_NUM_EXT="${VIDEO_NUM}"
        fi
        printf '%-7s %-7s %-27s %20s %16s %16s\n'  "${CARD_NUM}" "${VIDEO_NUM_EXT}" "${FILEBASENAME}" "${MOD_TS%.*}" "${DURATION}" "${TIMECODE}"
    done < <(tail -n +2 "${FILE_INVENTORY}" | grep ",video,")
    echo
}

_identify_card_type(){
    CARD="${1}"
    CARD_NAME="$(basename "${1}")"
    if [[ -n "$(find "${CARD}" -type d -iname "BDMV" | head -n 1)" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is AVCHD (e.g. Canon C100)"
        CAMERA_CARD_TYPE="AVCHD"
        CAMERA_CARD_DIR="$(find "${CAMERA_CARD_DIR[0]}" -type d -iname "BDMV" | head -n 1)" # set CAMERA_CARD_DIR path to the standard AVCHD directory structure
        EXTENSION="mts"
        RSYNC_OPTIONS=(-iname "*.cpi")
    elif [[ -n "$(find "${CARD}" -type d -iname "CLIPS001")" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is Canon XF (e.g. Canon C300)"
        CAMERA_CARD_TYPE="MXF"
        EXTENSION="mxf"
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.xmp" -o -iname "INDEX.MIF" -o -iname "*.cpf" \))
    elif [[ -n "$(find "${CARD}" -type f -iname "LASTCLIP.TXT")" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is P2 (Panasonic brand)"
        CAMERA_CARD_TYPE="P2"
        EXTENSION="mxf"
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.txt" -o -ipath "VOICE" \))
    elif [[ -n "$(find "${CARD}" \( -type d -iname "XDROOT" \) -o \( -type f -name "DISCMETA.XML" \) )" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is XAVC (Sony brand)"
        CAMERA_CARD_TYPE="XAVC"
        EXTENSION="mxf"
        # example (from an XDCam disc loaded with proav disabled):
        #├── ALIAS.XML
        #├── Clip
        #│   ├── C0001.MXF
        #│   ├── C0001M01.XML
        #│   ├── C0002.MXF
        #│   ├── C0002M01.XML
        #│   ├── C0003.MXF
        #│   └── C0003M01.XML
        #├── Component
        #├── DISCMETA.XML
        #├── Edit
        #├── General
        #│   └── Sony
        #│       └── Planning
        #├── INDEX.XML
        #├── MEDIAPRO.XML
        #├── SYSPRO.XML
        #├── Sub
        #│   ├── C0001S01.MXF
        #│   ├── C0002S01.MXF
        #│   └── C0003S0
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.bim" \))
    elif [[ -n "$(find "${CARD}" -type d -iname "BPAV")" ]] ; then
        _report -g "Camera card type identified: ${CARD_NAME} is XDCAM EX (e.g. Canon XF100)"
        CAMERA_CARD_TYPE="XDCAMEX"
        EXTENSION="mp4"
        RSYNC_OPTIONS=(\( -iname "*.xml" -o -iname "*.xmp" -o -iname "INDEX.MIF" -o -iname "*.cpf" \))
    fi
}

_check_if_in_list(){
    VALUE="${1}"
    LIST="${2}" # list is expected as comma-delimited
    unset LIST_EXPANDED
    # inspired by https://unix.stackexchange.com/posts/463617/revisions
    IFS=',' read -a RANGES  <<< "${LIST}"
    for RANGE in "${RANGES[@]}"; do
        if [[ ${RANGE} == *"-"* ]] ; then
            IFS=- read START END <<< "${RANGE}"
            [ -z "${START}" ] && continue
            [ -z "${END}" ] && end=$start
            for (( i=START ; i <= END ; i++ )); do
                LIST_EXPANDED+=",${i}"
            done
        else
            LIST_EXPANDED+=",${RANGE}"
        fi
    done
    if [[ ",${LIST_EXPANDED}," = *",${VALUE},"* ]] ; then
        echo "y"
    else
        echo "n"
    fi
}

# getopts loop
OPTIND=1
while getopts ":m:N:o:f:F:ctanh" OPT; do
    case "${OPT}" in
        m) MEDIAID="${OPTARG}" ;;
        N) CLIP_SELECTION="${OPTARG}" ;;
        o) AIP_DESTINATION="${OPTARG}" ;;
        f) OUTPUT_EXTENSION="${OPTARG}" ;;
        F) FFMPEG_PATH="${OPTARG}" ;;
        c) _report "The -c option (which combined 2 cards) is now removed. Please simply list as many camera cards as inputs as you need." ; exit ;;
        t) TAR="YES" ; AIP="NO" ;;
        a) TAR="YES" ; AIP="YES" ;;
        n) REPORT_ONLY="Y" ;;
        h) _usage ;;  # if the operator runs "[scriptname] -h" then the _usage text above will display in the terminal
        *) echo "Invalid option -${OPTARG}" ; _usage ;; # if the operator tries to use an option other than the ones listed above, the _usage text will display in the terminal
    esac
done
shift $(( OPTIND - 1 ))

trap _cleanup SIGHUP SIGINT SIGTERM # "termination signals" that ask the system to clean up and safely kill the ongoing process, with increasing urgency

# check for unassigned variables
CAMERA_CARD_DIR=("${@}")
# ask for camera card if it wasn't supplied
if [[ -z "${CAMERA_CARD_DIR[0]}" ]] ; then
    _report -b "Drag in the input directory or type 'q' to quit: "
    read -e CAMERA_CARD_DIR
    [[ "${CAMERA_CARD_DIR}" = "q" ]] && exit 0
fi
# check that each camera card is a directory
for CARD_DIR in "${CAMERA_CARD_DIR[@]}" ; do
    if [[ ! -d "${CARD_DIR}" ]] ; then
        _report -rt "ERROR: Input directory ${CAMERA_CARD_DIR} is not a directory. Exiting..."
        exit 1
    fi
done

# create temp filelists for use during script
FILE_INVENTORY="$(_maketemp).csv"
TEMP_CONCATLIST="$(_maketemp).ffconcat"

_make_inventory
_print_table
if [[ "${REPORT_ONLY}" = "Y" ]] ; then
    exit 0
fi

# determine FFMPEG_PATH
if [[ -z "${FFMPEG_PATH}" ]] ; then
    FFMPEG_PATH="$(which ffmpeg)"
fi
# ask for MEDIAID if it wasn't supplied
if [[ -z "${MEDIAID}" ]] ; then
    _report -b -n "Enter a unique package name or 'q' to quit: "
    read -e MEDIAID
    [[ "${MEDIAID}" = "q" ]] && exit 0
fi
# ask for AIP_DESTINATION if it wasn't supplied
if [[ -z "${AIP_DESTINATION}" ]] ; then
    _report -b -n "Provide the destination for the resulting package or 'q' to quit: "
    read -e AIP_DESTINATION
    echo
    [[ "${AIP_DESTINATION}" = "q" ]] && exit 0
    # check that camera card is a directory
    if [[ ! -d "${AIP_DESTINATION}" ]] ; then
        _report -rt "ERROR: Output directory ${AIP_DESTINATION} is not a directory. Exiting..."
        exit 1
    fi
fi

if [[ "${TAR}" == "NO" ]] && [[ -z "${CAMERA_CARD_TYPE}" ]] ; then
    CAMERA_CARD_TYPE="GENERAL"
    _report -b "Camera card type not identified!"
    echo
    printf "Select a strategy:"
    echo
    PS3="Selection: "
    select STRATEGY in "1. Create AIP: concatenate video files and restructure metadata directories" "2. Tar: Compress camera files and folders into a tarball" "3. Both: Create AIP and also tar original package (produces two packages)" "quit"
    do
        if [[ "${STRATEGY}" == "1. Create AIP: concatenate video files and restructure metadata directories" ]] ; then
            TAR="NO"
            AIP="YES"
        elif [[ "${STRATEGY}" == "2. Tar: Compress camera files and folders into a tarball" ]] ; then
            TAR="YES"
            AIP="NO"
        elif [[ "${STRATEGY}" == "3. Both: Create AIP and also tar original package (produces two packages)" ]] ; then
            TAR="YES"
            AIP="YES"
        fi
        if [[ "${STRATEGY}" == "quit" ]] ; then echo "Bye" && exit 0 ; fi
        break
    done
fi

# note the time we start working on the AIP
START_TIME="$(date +%FT%T)"

# define output directories
AIPDIR="${AIP_DESTINATION}/${MEDIAID}"
METADATADIR="${AIPDIR}/metadata"
LOG="${METADATADIR}/$(basename "${0}")_$(date +%F-%H%M%S).log"
METADATA_REPORTS_DIR="${METADATADIR}/reports"
METADATA_ORIGINAL_CAMERA_FILES_DIR="${METADATADIR}/original_camera_files"
if [[ -d "${AIPDIR}" ]] ; then
    _report -rt "ERROR: Output directory ${AIPDIR} already exists! Exiting to avoid overwriting..."
    exit 1
fi
if [[ "${AIP}" == "YES" && "${TAR}" == "YES" ]] ; then
    AIPDIR_AIP="${AIPDIR}/objects/AIP"
    AIPDIR_TAR="${AIPDIR}/objects/TAR"
    mkdir -p "${METADATA_REPORTS_DIR}" "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" "${AIPDIR_AIP}" "${AIPDIR_TAR}"
elif [[ "${AIP}" == "YES" ]] ; then
    AIPDIR_AIP="${AIPDIR}/objects"
    mkdir -p "${METADATA_REPORTS_DIR}" "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" "${AIPDIR_AIP}"
elif [[ "${TAR}" == "YES" ]] ; then
    AIPDIR_TAR="${AIPDIR}/objects/TAR"
    mkdir -p "${METADATA_REPORTS_DIR}" "${AIPDIR_TAR}"
else
    _report -r "ERROR: camera_cards needs to produce either an AIP or TAR output."
    exit 1
fi
### procedure for creating structured AIPs (concatenating video files and restructuring metadata directories)
if [[ "${AIP}" == "YES" ]] ; then
    if [[ "${TAR}" == "YES" ]] ; then
        AIPDIR_AIP="${AIPDIR}/objects/AIP"
    else
        AIPDIR_AIP="${AIPDIR}/objects"
    fi
    mkdir -p "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" "${AIPDIR_AIP}"
    # copy selected original metadata files to AIP
    _report -g "Moving significant camera-generated metadata files to AIP..."
    CAMERA_CARD_METADATA2KEEP=()
    while read METADATA_FILE ; do
        CAMERA_CARD_METADATA2KEEP+=("${METADATA_FILE}")
    done < <(_read_inventory -f ",metadata," "FILE_NAME")
    if [[ -w "${METADATA_ORIGINAL_CAMERA_FILES_DIR}" ]] ; then
        if (( ${#CAMERA_CARD_METADATA2KEEP[@]} != 0 )) ; then
            _report -g "Copying some camera metadata files into the package."
            rsync -avh "${CAMERA_CARD_METADATA2KEEP[@]}" "${METADATA_ORIGINAL_CAMERA_FILES_DIR}"
        fi
    else
        _report -r "Error: the folder for camera metadata is not writable."
        exit 1
    fi

    # generate metadata reports
    _generate_metadata

    ## concatenate video files into a single file
    echo
    FIRST_FILE="$(_read_inventory -f ",video," "FILE_NAME" | head -n 1)"
    if [[ -z "${OUTPUT_EXTENSION}" ]] ; then
        if [[ "${CAMERA_CARD_TYPE}" == "GENERAL" ]] ; then
            OUTPUT_EXTENSION="${FIRST_FILE#*.}"
        else
            OUTPUT_EXTENSION="${EXTENSION}"
        fi
    fi
    _report -g "Concatenating video files into a ${OUTPUT_EXTENSION} file."
    CONCATENATED_VIDEO_FILE="${AIPDIR_AIP}/${MEDIAID}_concatenated.${OUTPUT_EXTENSION}"

    # create list of files to concatenate, formatted for ffmpeg
    unset AUDIO_STRATEGY
    while read INVENTORY_ROW ; do
        SEG_NUMBER="$(echo "${INVENTORY_ROW}" | cut -d "," -f 3)"
        FILE_NAME="$(echo "${INVENTORY_ROW}" | cut -d "," -f 11)"
        AUDIO_CODEC="$(echo "${INVENTORY_ROW}" | cut -d "," -f 12)"
        AUDIO_SAMPLE_FMT="$(echo "${INVENTORY_ROW}" | cut -d "," -f 13)"
        echo "HEYY ${AUDIO_CODEC} ${AUDIO_SAMPLE_FMT} ${OUTPUT_EXTENSION} "
        if [[ "${AUDIO_CODEC}" == "pcm_bluray" && "${OUTPUT_EXTENSION}" != "m2t" ]] ; then
            if [[ "${AUDIO_SAMPLE_FMT}" == "s16" ]] ; then
                _report -b "Note: A file contains ${AUDIO_CODEC}(${AUDIO_SAMPLE_FMT}) audio but the output is output is not m2t, we'll transcode to pcm_s16le."
                AUDIO_STRATEGY=(-c:a pcm_s16le)
            else
                _report -b "Note: A file contains ${AUDIO_CODEC}(${AUDIO_SAMPLE_FMT}) audio but the output is output is not m2t, we'll transcode to pcm_s24le."
                AUDIO_STRATEGY=(-c:a pcm_s24le)
            fi
        fi
        if [[ -z "${CLIP_SELECTION}" || "$(_check_if_in_list "${SEG_NUMBER}" "${CLIP_SELECTION}")" == "y" ]] ; then
            echo "file '${FILE_NAME}'" >> "${TEMP_CONCATLIST}"
        fi
    done < <(_read_inventory -f ",video,\|,audio,")

    # concatenate video files in the order they are printed in $TEMP_CONCATLIST; map metadata from the first video file (in sequence) onto the concatenated file
    "${FFMPEG_PATH}" -f concat -safe 0 -i "${TEMP_CONCATLIST}" -i "${FIRST_FILE}" -map 0 -map_metadata 1 -c copy "${AUDIO_STRATEGY[@]}" "${CONCATENATED_VIDEO_FILE}"
    # calculate md5 for each stream in the file
    STREAMHASH="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}")_streamhash.md5"
    "${FFMPEG_PATH}" -i "${CONCATENATED_VIDEO_FILE}" -map 0 -c copy -f streamhash -hash md5 "${STREAMHASH}"
    # tests for video file existing
    FFMPEG_EXIT_CODE="${?}"
    if [[ ! -s "${CONCATENATED_VIDEO_FILE}" ]] ; then ((FFMPEG_ERRORS++)) ; fi
    _write_package_log "CONCATENATED VIDEO FILE" "${CONCATENATED_VIDEO_FILE}"
    _report -g "Video concatenation process finished"

    # if there are audio-only files, concatenate them and join with concatenated video file
    if [[ -n "$(_read_inventory -f ",audio," "FILE_NAME")" ]] ; then
        _report -g "Concatenating audio files..."
        FIRST_FILE="$(_read_inventory -f ",audio," "FILE_NAME" | head -n 1)"
        CONCATENATED_AUDIO_FILE="${AIPDIR_AIP}/${MEDIAID}_concatenated_audio.${FIRST_FILE#*.}"
        > "${TEMP_CONCATLIST}" # clear concatlist from video concatenation
        while read FILE ; do
            echo "file '${FILE}'" >> "${TEMP_CONCATLIST}"
        done < <(_read_inventory -f ",audio," "FILE_NAME")
        "${FFMPEG_PATH}" -f concat -safe 0 -i "${TEMP_CONCATLIST}" -i "${FIRST_FILE}" -map 0 -map_metadata 1 -c copy "${CONCATENATED_AUDIO_FILE}"
        _write_package_log "CONCATENATED AUDIO FILE" "${CONCATENATED_AUDIO_FILE}"

        # merge video and audio
        _report -g "Merging concatenated audio and video files..."
        # check that video and audio file durations match
        VIDEO_DURATION=$(mediainfo --Inform="Video;%Duration%" "${CONCATENATED_VIDEO_FILE}")
        AUDIO_DURATION=$(mediainfo --Inform="Audio;%Duration%" "${CONCATENATED_AUDIO_FILE}")
        if [[ "${VIDEO_DURATION}" -ne "${AUDIO_DURATION}" ]] ; then
            _report -r "Audio and video files are not the same length! Review output for sync. (Video duration = ${VIDEO_DURATION}, audio duration = ${AUDIO_DURATION})"
        fi
        # rename concatenated video file to avoid collision with audio file
        RENAMED_CONCATENATED_VIDEO_FILE="${CONCATENATED_VIDEO_FILE//concatenated/concatenated_video}"
        mv "${CONCATENATED_VIDEO_FILE}" "${RENAMED_CONCATENATED_VIDEO_FILE}"
        CONCATENATED_FILE_MERGED="${AIPDIR_AIP}/${MEDIAID}_concatenated.${FIRST_FILE#*.}"
        "${FFMPEG_PATH}" -i "${RENAMED_CONCATENATED_VIDEO_FILE}" -i "${CONCATENATED_AUDIO_FILE}" -c:v copy -c:a copy "${CONCATENATED_FILE_MERGED}"
        _write_package_log "JOINED FILE (AUDIO AND VIDEO)" "${CONCATENATED_FILE_MERGED}"
        _report -g "Audio + video joining process ended"
    fi

    # generate md5 of concatenated file
    if [[ -f "${CONCATENATED_FILE_MERGED}" ]] ; then
        CONCATENATED_MD5_OUTPUT="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_FILE_MERGED%.*}").md5"
        md5sum "${CONCATENATED_FILE_MERGED}" > "${CONCATENATED_MD5_OUTPUT}"
    else
        CONCATENATED_MD5_OUTPUT="${METADATA_REPORTS_DIR}/$(basename "${CONCATENATED_VIDEO_FILE%.*}").md5"
        md5sum "${CONCATENATED_VIDEO_FILE}" > "${CONCATENATED_MD5_OUTPUT}"
    fi

    if [[ "${CAMERA_CARD_TYPE}" == "GENERAL" ]] ; then
        # general profile only: move all video files to AIP as separate files, in case concatenation does not work for this camera card structure
        echo
        _report -g "Moving original audiovisual files to AIP..."
        if [[ -n "$(for FILE in $(_read_inventory "FILE_NAME") ; do basename "${FILE}" ; done | sort | uniq -d)" ]] ; then # check for duplicate filenames on camera card before beginning transfer process
            _report -r "Duplicate filenames found on original camera card! Not completing automated packaging process in order to avoid overwriting files with the same name. Please take a closer look at your original camera card's file structure to create this AIP."
            _report -r "Also note that metadata files (in the nmaahc_metadata folder of your AIP) may not have been properly generated for all files with the same filename."
            exit 1
        else
            while read FILE ; do
                rsync -avh --progress "${FILE}" "${AIPDIR_AIP}/"
            done < <(_read_inventory -f ",video,\|,audio," "FILE_NAME")
        fi
    fi
fi

### procedure for packaging files into compressed tars
if [[ "${TAR}" == "YES" ]] ; then
    ## create metadata reports for original package and audiovisual files
    # generate metadata reports
    _generate_metadata

    ## tar supplied input directory
    echo
    _report -g "Creating tar archive..."
    tar -czvf "${AIPDIR_TAR}/${MEDIAID}.tar.gz" -C "${CAMERA_CARD_DIR}" .
    _write_package_log "TAR ARCHIVE" "${MEDIAID}.tar.gz"

    ## generate checksum for tar archive
    TAR_MD5_OUTPUT="${METADATA_REPORTS_DIR}/${MEDIAID}.md5"
    md5sum "${AIPDIR_TAR}/${MEDIAID}.tar.gz" > "${TAR_MD5_OUTPUT}"
fi

# record variables in ingest log
_write_package_log "OPERATOR" "${OP}"
_write_package_log "CAMERA CARD DIRECTORY" "${CAMERA_CARD_DIR[@]}"
_write_package_log "MEDIAID" "${MEDIAID}"
_write_package_log "OUTPUT DIRECTORY" "${AIPDIR}"
_write_package_log "CAMERA CARD TYPE" "${CAMERA_CARD_TYPE}"
_write_package_log "STRATEGY" "${STRATEGY}"
_write_package_log "START TIME" "${START_TIME}"

# error reporting
echo
_report -b "Checking for errors during the camera_cards process..."
if [[ ! -s "$TREE" ]] || [[ -z $(cat "${MEDIAINFO_OUTPUT}") ]] || [[ -z $(cat "${EXIFTOOL_OUTPUT}") ]] || [[ -z $(cat "${FFPROBE_OUTPUT}") ]] ; then
    _report -r "Metadata generation process created one or more blank files. Input files may be empty. Review metadata output at ${METADATA_REPORTS_DIR}."
    _write_package_log "POSSIBLE_ERROR_REVIEW" "Metadata generation process created one or more blank files"
else
    _report -g "Metadata generation process looks ok"
fi
if [[ "${FFMPEG_EXIT_CODE}" -gt 0 ]] || [[ "${FFMPEG_ERRORS}" -gt 0 ]] ; then
    _report -r "FFmpeg may have encountered errors - review terminal output for more detail."
else
    _report -g "FFmpeg exit codes look ok"
    if [[ -s "${CONCATENATED_AUDIO_FILE}" ]] ; then
        if [[ "${VIDEO_DURATION}" -ne "${AUDIO_DURATION}" ]] ; then
        _report -r "Concatenated audio and video files were not the same length - review final merged file for sync at ${CONCATENATED_FILE_MERGED} (video duration = ${VIDEO_DURATION}, audio duration = ${AUDIO_DURATION})"
        _write_package_log "POSSIBLE_ERROR_REVIEW" "Concatenated audio and video files were not the same length"
        else
            _report -g "Concatenated audio and video process looks ok"
        fi
    fi
    if [[ "${AIP}" == "YES" ]] ; then # check for losslessness of concatenated file by comparing bitstream md5s
        OUTPUT_STREAMHASH="$(_maketemp)"
        _report -b "Checking streamhash between input and output..."
        "${FFMPEG_PATH}" -y -loglevel quiet -report -i "${CONCATENATED_VIDEO_FILE}" -map 0 -c copy -f streamhash -hash md5 "${OUTPUT_STREAMHASH}"
        if [[ $(cat "${STREAMHASH}") != $(cat "${OUTPUT_STREAMHASH}") ]] ; then
            _report -r "Concatenation process was not lossless! Review final merged file at ${CONCATENATED_VIDEO_FILE}"
            _write_package_log "POSSIBLE_ERROR_REVIEW" "Concatenation process was not lossless based on bitstream md5s"
        else
            _report -g "Concatenated process looks lossless"
        fi
    fi
fi

# log script ending
_report -g "camera_cards process complete. Your package can be found at ${AIPDIR}. A log file can be found at ${LOG}"