Merge pull request #22 from theHenks/processing-updates

Processing updates

src/LegendDataManagement.jl

@@ -21,7 +21,7 @@ using Printf: @printf
 using IntervalSets: AbstractInterval, ClosedInterval, leftendpoint, rightendpoint
 using LRUCache: LRU
 using ProgressMeter: @showprogress
-using PropertyFunctions: PropertyFunction, @pf
+using PropertyFunctions: PropertyFunction, @pf, filterby
 using StaticStrings: StaticString
 using Tables: columns
 

src/dataprod_config.jl

@@ -144,6 +144,92 @@ function _resolve_partition_runs(data::LegendData, period::DataPeriod, runs::Abs
     end
 end
 
+const _cached_analysis_runs = LRU{UInt, StructVector{@NamedTuple{period::DataPeriod, run::DataRun}}}(maxsize = 10)
+
+"""
+    analysis_runs(data::LegendData)
+
+Return cross-period analysis runs.
+"""
+function analysis_runs(data::LegendData)
+    get!(_cached_analysis_runs, objectid(data)) do
+        aruns = pydataprod_config(data).analysis_runs
+        periods_and_runs = [
+            let period = DataPeriod(string(p))
+                map(run -> (period = period, run = run), _resolve_partition_runs(data, period, rs))
+            end
+            for (p,rs) in aruns
+        ]
+        flat_pr = vcat(periods_and_runs...)::Vector{@NamedTuple{period::DataPeriod, run::DataRun}}
+        StructArray(flat_pr)
+    end
+end
+export analysis_runs
+
+"""
+    is_analysis_run(data::LegendData, period::DataPeriod, run::DataRun)
+
+Return `true` if `run` is an analysis run for `data` in `period`.
+"""
+function is_analysis_run(data::LegendData, period::DataPeriodLike, run::DataRunLike)
+    (period = DataPeriod(period), run = DataRun(run)) in analysis_runs(data)
+end
+export is_analysis_run
+
+
+const _cached_runinfo = LRU{Tuple{UInt, DataPeriod, DataRun, DataCategory}, typeof((startkey = FileKey("l200-p02-r006-cal-20221226T200846Z"), livetime = 0.0u"s"))}(maxsize = 30)
+
+"""
+    runinfo(data::LegendData, runsel::RunSelLike)::NamedTuple
+
+Get the run information for `data` in `runsel`.
+"""
+function runinfo(data::LegendData, runsel::RunCategorySelLike)::NamedTuple
+    # unpack runsel
+    period_in, run_in, category_in = runsel
+    period, run, category = DataPeriod(period_in), DataRun(run_in), DataCategory(category_in)
+    get!(_cached_runinfo, (objectid(data), period, run, category)) do
+        # check if run, period and category is available
+        if !haskey(data.metadata.dataprod.runinfo, Symbol(period))
+            throw(ArgumentError("Invalid period $period"))
+        elseif !haskey(data.metadata.dataprod.runinfo[Symbol(period)], Symbol(run))
+            throw(ArgumentError("Invalid run $run in period $period"))
+        elseif !haskey(data.metadata.dataprod.runinfo[Symbol(period)][Symbol(run)], Symbol(category))
+            throw(ArgumentError("Invalid category $category in period $period and run $run"))
+        end
+        (
+            startkey = FileKey(data.name, period, run, category, Timestamp(data.metadata.dataprod.runinfo[Symbol(period)][Symbol(run)][Symbol(category)][:start_key])), 
+            livetime =  if haskey(data.metadata.dataprod.runinfo[Symbol(period)][Symbol(run)][Symbol(category)], :livetime_in_s)
+                            Unitful.Quantity(data.metadata.dataprod.runinfo[Symbol(period)][Symbol(run)][Symbol(category)][:livetime_in_s], u"s") 
+                        else 
+                            Unitful.Quantity(NaN, u"s")
+                        end
+        )
+    end
+end
+export runinfo
+
+
+"""
+    start_filekey(data::LegendData, runsel::RunCategorySelLike)
+
+Get the starting filekey for `data` in `period`, `run`, `category`.
+"""
+function start_filekey(data::LegendData, runsel::RunCategorySelLike)
+    runinfo(data, runsel).startkey
+end
+export start_filekey
+
+
+"""
+    phy_livetime(data::LegendData, runsel::RunSelLike)
+
+Get the livetime for `data` in physics data taking of `run` in `period`.
+"""
+function livetime(data::LegendData, runsel::RunCategorySelLike)
+    runinfo(data, runsel).livetime
+end
+export phy_livetime
 
 const _cached_bad_filekeys = LRU{UInt, Set{FileKey}}(maxsize = 10)
 

src/filekey.jl

@@ -157,15 +157,20 @@ function DataPeriod(s::AbstractString)
     end
 end
 
+function DataPeriod(s::Symbol) 
+    DataPeriod(string(s)) 
+end
+
 Base.convert(::Type{DataPeriod}, s::AbstractString) = DataPeriod(s)
+Base.convert(::Type{DataPeriod}, s::Symbol) = DataPeriod(string(s))
 
 
 """
     DataPeriodLike = Union{DataPeriod, Integer, AbstractString}
 
 Anything that can represent a data period, like `DataPeriod(2)` or "p02".
 """
-const DataPeriodLike = Union{DataPeriod, AbstractString}
+const DataPeriodLike = Union{DataPeriod, Symbol, AbstractString}
 export DataPeriodLike
 
 
@@ -210,17 +215,21 @@ function DataRun(s::AbstractString)
     end
 end
 
+function DataRun(s::Symbol) 
+    DataRun(string(s)) 
+end
+
 Base.convert(::Type{DataRun}, s::AbstractString) = DataRun(s)
+Base.convert(::Type{DataRun}, s::Symbol) = DataRun(string(s))
 
 """
     DataRunLike = Union{DataRun, Integer, AbstractString}
 
 Anything that can represent a data run, like `DataRun(6)` or "r006".
 """
-DataRunLike = Union{DataRun, AbstractString}
+DataRunLike = Union{DataRun, Symbol, AbstractString}
 export DataRunLike
 
-
 """
     struct DataCategory <: DataSelector
 
@@ -274,6 +283,19 @@ const DataCategoryLike = Union{DataCategory, Symbol, AbstractString}
 export DataCategoryLike
 
 
+"""
+    struct RunSelLike = Tuple{<:DataPeriodLike, <:DataRunLike}  
+
+Represents a LEGEND run selection.
+"""
+const RunSelLike = Tuple{<:DataPeriodLike, <:DataRunLike}
+
+"""
+    struct RunCategorySelLike = Tuple{<:DataPeriodLike, <:DataRunLike}  
+
+Represents a LEGEND run selection for a specific `category`.
+"""
+const RunCategorySelLike = Tuple{<:DataPeriodLike, <:DataRunLike, <:DataCategoryLike}
 
 """
     struct Timestamp <: DataSelector

src/legend_data.jl

@@ -54,15 +54,19 @@ SolidStateDetector(l200, :V99000A)
 struct LegendData <: AbstractSetupData
     # ToDo: Add setup name
     _config::SetupConfig
+    _name::Symbol
 end
 export LegendData
 
 get_setup_config(data::LegendData) = getfield(data, :_config)
+get_setup_name(data::LegendData) = getfield(data, :_name)
 
 @inline function Base.getproperty(d::LegendData, s::Symbol)
     # Include internal fields:
     if s == :_config
         getfield(d, :_config)
+    elseif s == :name
+        getfield(d, :_name)
     elseif s == :metadata
         AnyProps(data_path(d, "metadata"))
     elseif s == :tier
@@ -87,14 +91,13 @@ end
 
 
 function LegendData(setup::Symbol)
-    LegendData(getproperty(LegendDataConfig().setups, setup))
+    LegendData(getproperty(LegendDataConfig().setups, setup), setup)
 end
 
 Base.@deprecate data_filename(data::LegendData, filekey::FileKey, tier::DataTierLike) data.tier[tier, filekey]
 export data_filename
 
 
-
 """
     struct LegendDataManagement.LegendTierData
 
@@ -215,12 +218,13 @@ end
 
 
 """
-    channelinfo(data::LegendData, sel::AnyValiditySelection)
+    channelinfo(data::LegendData, sel::AnyValiditySelection; system::Symbol = :all, only_processable::Bool = false)
+    channelinfo(data::LegendData, sel::RunCategorySelLike; system::Symbol = :all, only_processable::Bool = false)
 
 Get all channel information for the given [`LegendData`](@ref) and
 [`ValiditySelection`](@ref).
 """
-function channelinfo(data::LegendData, sel::AnyValiditySelection)
+function channelinfo(data::LegendData, sel::AnyValiditySelection; system::Symbol = :all, only_processable::Bool = false)
     chmap = data.metadata(sel).hardware.configuration.channelmaps
     dpcfg = data.metadata(sel).dataprod.config.analysis
 
@@ -249,9 +253,11 @@ function channelinfo(data::LegendData, sel::AnyValiditySelection)
         channel::ChannelId = ChannelId(fcid >= 0 ? fcid : rawid)
 
         detector::DetectorId = DetectorId(k)
-        system::Symbol = Symbol(chmap[k].system)
+        local system::Symbol = Symbol(chmap[k].system)
         processable::Bool = get(dpcfg[k], :processable, false)
         usability::Symbol = Symbol(get(dpcfg[k], :usability, :unkown))
+        is_blinded::Bool = get(dpcfg[k], :is_blinded, false)
+        aoe_status::Symbol = Symbol(get(get(dpcfg[k], :psd_status, PropDict()), Symbol("A/E"), :unkown))
 
         location::Symbol, detstring::Int, position::Union{Int,Symbol}, fiber::StaticString{8} = _convert_location(chmap[k].location)
 
@@ -263,26 +269,37 @@ function channelinfo(data::LegendData, sel::AnyValiditySelection)
         hvch::Int = get(chmap[k].voltage, :channel, -1)
 
         return (;
-            detector, channel, fcid, rawid, system, processable, usability,
+            detector, channel, fcid, rawid, system, processable, usability, is_blinded, aoe_status,
             location, detstring, fiber, position, cc4, cc4ch, daqcrate, daqcard, hvcard, hvch
         )
     end
 
-    StructArray(make_row.(channel_keys))
+    chinfo = StructArray(make_row.(channel_keys)) 
+    if !(system == :all)
+        chinfo = chinfo |> filterby(@pf $system .== system)
+    end
+    if only_processable
+        chinfo = chinfo |> filterby(@pf $processable .== true)
+    end
+    return chinfo
 end
 export channelinfo
 
+function channelinfo(data::LegendData, sel::RunCategorySelLike; kwargs...)
+    channelinfo(data, start_filekey(data, sel); kwargs...)
+end
+
 
 """
-    channelinfo(data::LegendData, sel::AnyValiditySelection, channel::ChannelId)
-    channelinfo(data::LegendData, sel::AnyValiditySelection, detector::DetectorId)
+    channelinfo(data::LegendData, sel::AnyValiditySelection, channel::ChannelIdLike)
+    channelinfo(data::LegendData, sel::AnyValiditySelection, detector::DetectorIdLike)
 
 Get channel information validitiy selection and [`DetectorId`](@ref) resp.
 [`ChannelId`](@ref).
 """
-function channelinfo(data::LegendData, sel::AnyValiditySelection, channel::ChannelId)
-    chinfo = channelinfo(data, sel)
-    idxs = findall(x -> ChannelId(x) == channel, chinfo.channel)
+function channelinfo(data::LegendData, sel::AnyValiditySelection, channel::ChannelIdLike; kwargs...)
+    chinfo = channelinfo(data, sel; kwargs...)
+    idxs = findall(x -> ChannelId(x) == ChannelId(channel), chinfo.channel)
     if isempty(idxs)
         throw(ArgumentError("No channel information found for channel $channel"))
     elseif length(idxs) > 1
@@ -292,9 +309,9 @@ function channelinfo(data::LegendData, sel::AnyValiditySelection, channel::Chann
     end
 end
 
-function channelinfo(data::LegendData, sel::AnyValiditySelection, detector::DetectorId)
-    chinfo = channelinfo(data, sel)
-    idxs = findall(x -> DetectorId(x) == detector, chinfo.detector)
+function channelinfo(data::LegendData, sel::AnyValiditySelection, detector::DetectorIdLike; kwargs...)
+    chinfo = channelinfo(data, sel; kwargs...)
+    idxs = findall(x -> DetectorId(x) == DetectorId(detector), chinfo.detector)
     if isempty(idxs)
         throw(ArgumentError("No channel information found for detector $detector"))
     elseif length(idxs) > 1
@@ -305,7 +322,6 @@ function channelinfo(data::LegendData, sel::AnyValiditySelection, detector::Dete
 end
 
 
-
 function channel_info(data::LegendData, sel::AnyValiditySelection)
     Base.depwarn(
         "`channel_info(data::LegendData, sel::AnyValiditySelection)` is deprecated, use `channelinfo(data, sel)` instead (note: output format differs).",