From 096cde3d0c384eb46f1ef05014b78f86ed1a171a Mon Sep 17 00:00:00 2001 From: Yury Gaydaychuk Date: Fri, 29 Nov 2024 14:15:16 +0100 Subject: [PATCH] [Commit Slider] AC mode extended (#26497) ### Details: - *item1* - *...* ### Tickets: - *ticket-id* --- .../commit_slider/tests/commit_slider_test.py | 7 ++-- .../tools/commit_slider/tests/test_data.py | 16 ++++++- .../tests/tests_res/fbv_ac_mode_bitwise.cpp | 42 +++++++++++++++++++ .../tests/tests_res/tests_res.json | 23 +++++++++- .../tools/commit_slider/utils/modes.py | 36 +++++++--------- 5 files changed, 96 insertions(+), 28 deletions(-) create mode 100644 src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py index 2d342da339e810..c163a314b6d9e7 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/commit_slider_test.py @@ -14,7 +14,8 @@ BenchmarkAppDataUnstable, BenchmarkAppDataStable, BenchmarkAppNoDegradationData,\ BenchmarkAppUnstableDevData, BenchmarkAppWrongPathData, BenchmarkAppPathFoundData,\ BenchmarkFirstFixedAppData, AcModeData, BenchmarkMetricData, CustomizedLogData, \ - MultiConfigData, ConfigMultiplicatorData, ConfigMultiplicatorWithKeyData + MultiConfigData, ConfigMultiplicatorData, ConfigMultiplicatorWithKeyData, \ + AcModeDataBitwise class CommitSliderTest(TestCase): @skip_commit_slider_devtest @@ -81,9 +82,9 @@ def testBmStable(self): self.assertEqual(breakCommit, actualCommit) @skip_commit_slider_devtest - def testACMode(self): + def testACModeBitwise(self): breakCommit, updatedData = getExpectedCommit( - AcModeData()) + AcModeDataBitwise()) actualCommit, _ = getActualCommit(updatedData) self.assertEqual(breakCommit, actualCommit) diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py b/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py index 50249b1f24ab0f..53bd9d2c8a3dde 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/test_data.py @@ -71,7 +71,8 @@ class TestCase(Enum): CustomizedLog = 15, MultiConfig = 16, ConfigMultiplicator = 17, - MultiConfigWithKey = 18 + MultiConfigWithKey = 18, + AcModeDataBitwise = 19 def requireTestData(self, reqLambda): # mapping json to test data holder @@ -190,6 +191,19 @@ def __init__(self): requireBinarySearchData ) +class AcModeDataBitwise(TestData): + def getTestCase(): + return TestData.TestCase.AcModeDataBitwise + + def getTestName(self): + return "ACMode" + + def __init__(self): + from test_util import requireBinarySearchData + self.requireTestData( + requireBinarySearchData + ) + class BenchmarkFirstFixedAppData(TestData): def getTestCase(): return TestData.TestCase.BmFirstFixed diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp new file mode 100644 index 00000000000000..856438809f6acb --- /dev/null +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/fbv_ac_mode_bitwise.cpp @@ -0,0 +1,42 @@ +#include + +int main () { + const char *patchGenerator = R"V0G0N( +[ + { + "str": "std::cout << \"prefix\\Perplexity: 100.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "success_1" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 100.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "success_2" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_1", + "state": "BREAK" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_2" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_1" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_2" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_1" + }, + { + "str": "std::cout << \"prefix\\Perplexity: 50.0% [FAILED: abs error = 9.118 | relative error = 0.3144]\\n\";", + "comment": "error_2" + } +] +)V0G0N"; + return 0; +} \ No newline at end of file diff --git a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json index 7f357753de17e5..8bdd8bec6a29c9 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json +++ b/src/plugins/intel_cpu/tools/commit_slider/tests/tests_res/tests_res.json @@ -345,8 +345,27 @@ } } }, + "ACModeBitwise": { + "repoName": "ACModeBitwise", + "patchedFile": "tests_res/fbv_ac_mode_bitwise.cpp", + "testCfg": { + "appCmd" : "{appCmd}", + "appPath": "{appPath}", + "gitPath" : "{gitPath}", + "buildPath" : "{buildPath}", + "verboseOutput": false, + "runConfig" : { + "commitList" : { + "getCommitListCmd" : "git log {start}..{end} --boundary --pretty=\"%h\"" + }, + "mode" : "ac", + "traversal" : "firstFailedVersion", + "threshold": "80%" + } + } + }, "ACMode": { - "repoName": "BmBinarySearchUnstable", + "repoName": "ACMode", "patchedFile": "tests_res/fbv_ac_mode.cpp", "testCfg": { "appCmd" : "{appCmd}", @@ -359,7 +378,7 @@ "getCommitListCmd" : "git log {start}..{end} --boundary --pretty=\"%h\"" }, "mode" : "ac", - "traversal" : "firstFailedVersion", + "traversal" : "allBreaks", "threshold": "80%" } } diff --git a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py index 4800d0854ae093..3340f1f597efd8 100644 --- a/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py +++ b/src/plugins/intel_cpu/tools/commit_slider/utils/modes.py @@ -288,7 +288,7 @@ class AccuracyCheckerMode(Mode): def __init__(self, cfg): super().__init__(cfg) self.thresholdPattern = ":\s([0-9]*[.][0-9]*)%.*abs error" - self.breakThroughput = 0 + self.curMetric = None self.createCash() def prepareRun(self, list, cfg): @@ -317,29 +317,21 @@ def prepareRun(self, list, cfg): self.sampleThroughput = float(foundThroughput) return list - def checkCfg(self, cfg): - super().checkCfg(cfg) - if not ("threshold" in cfg["runConfig"]): - raise CfgError("Threshold is not configured") - else: - self.threshold = cfg["runConfig"]["threshold"] - self.threshold = float(self.threshold.strip('%')) - - def compareCommits(self, lCommit: str, rCommit: str, cfg: map): - leftThroughput = self.getPseudoMetric(lCommit, cfg) - rightThroughput = self.getPseudoMetric(rCommit, cfg) - isLeftGood = leftThroughput >= float(self.threshold) - isRightGood = rightThroughput >= float(self.threshold) - if not isRightGood: - self.breakThroughput = rightThroughput + leftMetric = self.getPseudoMetric(lCommit, cfg) + rightMetric = self.getPseudoMetric(rCommit, cfg) + isDiff = leftMetric != rightMetric + if isDiff: + self.curMetric = rightMetric curCommit = rCommit.replace('"', "") commitLogger = getCommitLogger(cfg, curCommit) - commitLogger.info("Current accuracy is {}%".format(rightThroughput)) + commitLogger.info("Current accuracy is {}%".format(rightMetric)) commitLogger.info( - "Commit is {status}".format(status=("bad" if isRightGood else "good")) + "Commit {status} from {c}".format( + status=("differs" if isDiff else "doesn't differ"), + c=lCommit) ) - return isLeftGood != isRightGood + return isDiff def getPseudoMetric(self, commit, cfg): commit = commit.replace('"', "") @@ -372,12 +364,12 @@ def getPseudoMetric(self, commit, cfg): return curThroughput def setOutputInfo(self, pathCommit): - pathCommit.breakThroughput = self.breakThroughput + pathCommit.metric = self.curMetric def getCommitInfo(self, commit): - return "{ci}, throughput = {d}".format( + return "{ci}, metric = {d}".format( ci=super().getCommitInfo(commit), - d=commit.breakThroughput) + d=commit.metric) class CompareBlobsMode(Mode):