[CPU][DEBUG_CAPS] Add env option to collect average counters to a file (

#27577)

called OV_CPU_AVERAGE_COUNTERS

### TODO
- [x] Update documentation

src/plugins/intel_cpu/docs/debug_capabilities/README.md

@@ -24,3 +24,5 @@ Use the following cmake option to enable debug capabilities:
   `OV_CPU_SUMMARY_PERF=1`  
   Set the environment variable to display performance summary at the time when model is being destructed.  
   Internal performance counter will be enabled automatically. 
+* [Average counters](average_counters.md)  
+  `OV_CPU_AVERAGE_COUNTERS=filename`

src/plugins/intel_cpu/docs/debug_capabilities/average_counters.md

@@ -0,0 +1,17 @@
+# Average counters
+
+To enable collection of per-node average counters the following environment variable should be used:
+
+```sh
+    OV_CPU_AVERAGE_COUNTERS=<filename> binary ...
+```
+
+The output table has the same format as:
+
+```sh
+    benchmark_app --report_type average_counters
+```
+
+The avarage counters can be aggregated using:
+
+* [aggregate-average-counters.py](../../tools/aggregate-average-counters/aggregate-average-counters.py)

src/plugins/intel_cpu/src/config.cpp

@@ -60,9 +60,12 @@ Config::Config() {
  * configuration properties
  */
 void Config::applyDebugCapsProperties() {
-    // always enable perf counters for verbose mode and performance summary
-    if (!debugCaps.verbose.empty() || !debugCaps.summaryPerf.empty())
+    // always enable perf counters for verbose, performance summary and average counters
+    if (!debugCaps.verbose.empty() ||
+        !debugCaps.summaryPerf.empty() ||
+        !debugCaps.averageCountersPath.empty()) {
         collectPerfCounters = true;
+    }
 }
 #endif
 

src/plugins/intel_cpu/src/graph.cpp

@@ -58,6 +58,7 @@ namespace intel_cpu {
 
 Graph::~Graph() {
     CPU_DEBUG_CAP_ENABLE(summary_perf(*this));
+    CPU_DEBUG_CAP_ENABLE(average_counters(*this));
 }
 
 template<typename NET>

src/plugins/intel_cpu/src/graph_dumper.cpp

@@ -10,10 +10,12 @@
 #include "openvino/runtime/exec_model_info.hpp"
 #include "utils/debug_capabilities.h"
 
+#include <chrono>
 #include <map>
 #include <memory>
 #include <string>
 #include <vector>
+#include <fstream>
 
 namespace ov {
 namespace intel_cpu {
@@ -346,6 +348,62 @@ void summary_perf(const Graph &graph) {
     }
 }
 
+void average_counters(const Graph &graph) {
+    /**
+     * @todo improve logic for a graph with inner graphs:
+     * - collect counters only for the outer graph if full path is specified
+     * - collect counters for all the graphs if some keyword (i.e. 'all') is specified, using the following form:
+     * - <nesting-level>_<graph-name>.csv
+     * For example: 0_MyModel.csv
+     */
+    static int graphIndex = 0;
+
+    std::ofstream file;
+    std::string fileName = graph.getConfig().debugCaps.averageCountersPath + "_" + std::to_string(graphIndex++) + ".csv";
+
+    file.open(fileName);
+
+    // table structure is identical to the benchmark_app average_counters report
+    const std::string header = "layerName;execStatus;layerType;execType;realTime (ms);cpuTime (ms);";
+    file << header << "\n";
+
+    uint64_t total = 0;
+
+    auto toMs = [](uint64_t value) {
+        return std::chrono::microseconds(value).count() / 1000.0;
+    };
+
+    auto printAverageCounter = [&toMs, &file](NodePtr node) {
+        const uint64_t avg = node->PerfCounter().avg();
+        const std::string status = avg > 0 ? "EXECUTED" : "NOT_RUN";
+        const auto cpuTime = toMs(avg);
+        const auto realTime = cpuTime;
+
+        file << node->getName() << ";"
+               << status << ";"
+               << node->getTypeStr() << ";"
+               << node->getPrimitiveDescriptorType() << ";"
+               << realTime << ";"
+               << cpuTime << ";"
+               << "\n";
+
+        return avg;
+    };
+
+    for (auto &node : graph.GetNodes()) {
+        if (node->isConstant())
+            continue;
+
+        total += printAverageCounter(node);
+    }
+
+    const auto totalMs = toMs(total);
+
+    file << "Total;;;;" << totalMs << ";" << totalMs << ";" << "\n";
+
+    file.close();
+}
+
 #endif
 }   // namespace intel_cpu
 }   // namespace ov

src/plugins/intel_cpu/src/graph_dumper.h

@@ -15,6 +15,7 @@ std::shared_ptr<ov::Model> dump_graph_as_ie_ngraph_net(const Graph &graph);
 #ifdef CPU_DEBUG_CAPS
 void serialize(const Graph &graph);
 void summary_perf(const Graph &graph);
+void average_counters(const Graph &graph);
 #endif // CPU_DEBUG_CAPS
 
 }   // namespace intel_cpu

src/plugins/intel_cpu/src/utils/debug_caps_config.cpp

@@ -54,15 +54,18 @@ void DebugCapsConfig::readProperties() {
     if ((envVarValue = readEnv("OV_CPU_BLOB_DUMP_NODE_NAME")))
         blobDumpFilters[FILTER::BY_NAME] = envVarValue;
 
-    if ((envVarValue = readEnv("OV_CPU_SUMMARY_PERF"))) {
-        summaryPerf = envVarValue;
-    }
-
     if ((envVarValue = readEnv("OV_CPU_DISABLE")))
         disable.parseAndSet(envVarValue);
 
     if ((envVarValue = readEnv("OV_CPU_DUMP_IR")))
         dumpIR.parseAndSet(envVarValue);
+
+    if ((envVarValue = readEnv("OV_CPU_SUMMARY_PERF"))) {
+        summaryPerf = envVarValue;
+    }
+
+    if ((envVarValue = readEnv("OV_CPU_AVERAGE_COUNTERS")))
+        averageCountersPath = envVarValue;
 }
 
 }   // namespace intel_cpu

src/plugins/intel_cpu/src/utils/debug_caps_config.h

@@ -40,6 +40,7 @@ class DebugCapsConfig {
     };
 
     std::string execGraphPath;
+    std::string averageCountersPath;
     std::string verbose;
     std::string blobDumpDir = "cpu_dump";
     FORMAT blobDumpFormat = FORMAT::TEXT;