Yaml path fix

examples/pcb-aoi/incremental_learning_bench/fault_detection/benchmarkingjob.yaml

@@ -0,0 +1,72 @@
+benchmarkingjob:
+  # job name of bechmarking; string type;
+  name: "benchmarkingjob"
+  # the url address of job workspace that will reserve the output of tests; string type;
+  workspace: "/ianvs/incremental_learning_bench/workspace"
+
+  # the url address of test environment configuration file; string type;
+  # the file format supports yaml/yml;
+  testenv: "./examples/pcb-aoi/incremental_learning_bench/fault_detection/testenv/testenv.yaml"
+
+  # the configuration of test object
+  test_object:
+    # test type; string type;
+    # currently the option of value is "algorithms",the others will be added in succession.
+    type: "algorithms"
+    # test algorithm configuration files; list type;
+    algorithms:
+      # algorithm name; string type;
+      - name: "fpn_incremental_learning"
+        # the url address of test algorithm configuration file; string type;
+        # the file format supports yaml/yml
+        url: "./examples/pcb-aoi/incremental_learning_bench/fault_detection/testalgorithms/fpn/fpn_algorithm.yaml"
+
+  # the configuration of ranking leaderboard
+  rank:
+    # rank leaderboard with metric of test case's evaluation and order ; list type;
+    # the sorting priority is based on the sequence of metrics in the list from front to back;
+    sort_by: [ { "f1_score": "descend" }, { "samples_transfer_ratio": "ascend" } ]
+
+    # visualization configuration
+    visualization:
+      # mode of visualization in the leaderboard; string type;
+      # There are quite a few possible dataitems in the leaderboard. Not all of them can be shown simultaneously on the screen.
+      # In the leaderboard, we provide the "selected_only" mode for the user to configure what is shown or is not shown.
+      mode: "selected_only"
+      # method of visualization for selected dataitems; string type;
+      # currently the options of value are as follows:
+      #  1> "print_table": print selected dataitems;
+      method: "print_table"
+
+    # selected dataitem configuration
+    # The user can add his/her interested dataitems in terms of "paradigms", "modules", "hyperparameters" and "metrics",
+    # so that the selected columns will be shown.
+    selected_dataitem:
+      # currently the options of value are as follows:
+      #   1> "all": select all paradigms in the leaderboard;
+      #   2> paradigms in the leaderboard, e.g., "singletasklearning"
+      paradigms: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all modules in the leaderboard;
+      #   2> modules in the leaderboard, e.g., "basemodel"
+      modules: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all hyperparameters in the leaderboard;
+      #   2> hyperparameters in the leaderboard, e.g., "momentum"
+      hyperparameters: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all metrics in the leaderboard;
+      #   2> metrics in the leaderboard, e.g., "F1_SCORE"
+      metrics: [ "f1_score", "samples_transfer_ratio" ]
+
+    # model of save selected and all dataitems in workspace `./rank` ; string type;
+    # currently the options of value are as follows:
+    #  1> "selected_and_all": save selected and all dataitems;
+    #  2> "selected_only": save selected dataitems;
+    save_mode: "selected_and_all"
+
+
+
+
+
+

examples/pcb-aoi/incremental_learning_bench/fault_detection/testalgorithms/fpn/fpn_algorithm.yaml

@@ -0,0 +1,58 @@
+algorithm:
+  # paradigm type; string type;
+  # currently the options of value are as follows:
+  #   1> "singletasklearning"
+  #   2> "incrementallearning"
+  paradigm_type: "incrementallearning"
+  incremental_learning_data_setting:
+    # ratio of training dataset; float type;
+    # the default value is 0.8.
+    train_ratio: 0.8
+    # the method of splitting dataset; string type; optional;
+    # currently the options of value are as follows:
+    #   1> "default": the dataset is evenly divided based train_ratio;
+    splitting_method: "default"
+  # the url address of initial model for model pre-training; string url;
+  initial_model_url: "/ianvs/initial_model/model.zip"
+
+  # algorithm module configuration in the paradigm; list type;
+  modules:
+    # type of algorithm module; string type;
+    # currently the options of value are as follows:
+    #   1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module;
+    - type: "basemodel"
+      # name of python module; string type;
+      # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking;
+      name: "FPN"
+      # the url address of python module; string type;
+      url: "./examples/pcb-aoi/incremental_learning_bench/fault_detection/testalgorithms/fpn/basemodel.py"
+
+      # hyperparameters configuration for the python module; list type;
+      hyperparameters:
+        # name of the hyperparameter; string type;
+        - momentum:
+            # values of the hyperparameter; list type;
+            # types of the value are string/int/float/boolean/list/dictionary
+            values:
+              - 0.95
+              - 0.5
+        - learning_rate:
+            values:
+              - 0.1
+      #  2> "hard_example_mining": check hard example when predict ; optional module;
+    - type: "hard_example_mining"
+      # name of python module; string type;
+      name: "IBT"
+      # the url address of python module; string type;
+      url: "./examples/pcb-aoi/incremental_learning_bench/fault_detection/testalgorithms/fpn/hard_example_mining.py"
+      # hyperparameters configuration for the python module; list type;
+      hyperparameters:
+        # name of the hyperparameter; string type;
+        # threshold of image; value is [0, 1]
+        - threshold_img:
+            values:
+              - 0.9
+        # predict box of image; value is [0, 1]
+        - threshold_box:
+            values:
+              - 0.9

examples/pcb-aoi/incremental_learning_bench/fault_detection/testalgorithms/fpn/hard_example_mining.py

@@ -0,0 +1,110 @@
+# Copyright 2022 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Hard Example Mining Algorithms"""
+
+import abc
+
+from sedna.common.class_factory import ClassFactory, ClassType
+
+__all__ = ('IBTFilter')
+
+
+class BaseFilter(metaclass=abc.ABCMeta):
+    """The base class to define unified interface."""
+
+    def __call__(self, infer_result=None):
+        """
+        predict function, judge the sample is hard or not.
+
+        Parameters
+        ----------
+        infer_result : array_like
+            prediction result
+
+        Returns
+        -------
+        is_hard_sample : bool
+            `True` means hard sample, `False` means not.
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def data_check(cls, data):
+        """Check the data in [0,1]."""
+        return 0 <= float(data) <= 1
+
+
+@ClassFactory.register(ClassType.HEM, alias="IBT")
+class IBTFilter(BaseFilter, abc.ABC):
+    """
+    **Object detection** Hard samples discovery methods named `IBT`
+
+    Parameters
+    ----------
+    threshold_img: float
+        hard coefficient threshold score to filter img, default to 0.5.
+    threshold_box: float
+        threshold_box to calculate hard coefficient, formula  is hard
+        coefficient = number(prediction_boxes less than threshold_box) /
+        number(prediction_boxes)
+    """
+
+    def __init__(self, threshold_img=0.5, threshold_box=0.5, **kwargs):
+        self.threshold_box = float(threshold_box)
+        self.threshold_img = float(threshold_img)
+
+    def __call__(self, infer_result=None) -> bool:
+        """Judge the img is hard sample or not.
+
+        Parameters
+        ----------
+        infer_result: array_like
+            prediction boxes list, such as [bbox1, bbox2, bbox3,....],
+            where bbox = [xmin, ymin, xmax, ymax, score, label]
+            score should be in [0,1], who will be ignored if its value not
+            in [0,1].
+
+        Returns
+        -------
+        is hard sample: bool
+            `True` means hard sample, `False` means not.
+        """
+
+        def _convert_to_bboxes():
+            bboxes = []
+            for vs in infer_result.values():
+                for v in vs:
+                    bbox = v.get("bbox").tolist()
+                    if bbox:
+                        bboxes.extend(bbox)
+            return bboxes
+
+        infer_result = _convert_to_bboxes()
+
+        if not (infer_result
+                and all(map(lambda x: len(x) > 4, infer_result))):
+            # if invalid input, return False
+            return False
+
+        data_check_list = [bbox[4] for bbox in infer_result
+                           if self.data_check(bbox[4])]
+        if len(data_check_list) != len(infer_result):
+            return False
+
+        confidence_score_list = [
+            float(box_score) for box_score in data_check_list
+            if float(box_score) <= self.threshold_box]
+        return (len(confidence_score_list) / len(infer_result)
+                >= (1 - self.threshold_img))

examples/pcb-aoi/incremental_learning_bench/fault_detection/testenv/f1_score.py

@@ -0,0 +1,54 @@
+# Copyright 2022 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from FPN_TensorFlow.libs.label_name_dict.label_dict import NAME_LABEL_MAP
+from FPN_TensorFlow.data.io.read_tfrecord import convert_labels
+from FPN_TensorFlow.help_utils.tools import get_single_label_dict, single_label_eval
+from sedna.common.class_factory import ClassType, ClassFactory
+
+__all__ = ["f1_score"]
+
+
+@ClassFactory.register(ClassType.GENERAL, alias="f1_score")
+def f1_score(y_true, y_pred):
+    predict_dict = {}
+
+    for k, v in y_pred.items():
+        k = f"b'{k}'"
+        if not predict_dict.get(k):
+            predict_dict[k] = v
+
+    gtboxes_dict = convert_labels(y_true)
+
+    f1_score_list = []
+
+    for label in NAME_LABEL_MAP.keys():
+        if label == 'back_ground':
+            continue
+
+        rboxes, gboxes = get_single_label_dict(predict_dict, gtboxes_dict, label)
+        rec, prec, ap, box_num = single_label_eval(rboxes, gboxes, 0.3, False)
+        recall = 0 if rec.shape[0] == 0 else rec[-1]
+        precision = 0 if prec.shape[0] == 0 else prec[-1]
+        f1_score = 0 if not (recall + precision) else (2 * precision * recall / (recall + precision))
+
+        f1_score_list.append(f1_score)
+
+    f1_score_avg = 0
+    if f1_score_list:
+        f1_score_avg = round(float(sum(f1_score_list)) / len(f1_score_list), 4)
+
+    print(f"f1_score_avg: {f1_score_avg}")
+
+    return f1_score_avg

examples/pcb-aoi/incremental_learning_bench/fault_detection/testenv/testenv.yaml

@@ -0,0 +1,34 @@
+testenv:
+  # dataset configuration
+  dataset:
+    # the url address of train dataset index; string type;
+    train_url: "/ianvs/dataset/train_data/index.txt"
+    # the url address of test dataset index; string type;
+    test_url: "/ianvs/dataset/test_data/index.txt"
+
+  # model eval configuration of incremental learning;
+  model_eval:
+    # metric used for model evaluation
+    model_metric:
+      # metric name; string type;
+      name: "f1_score"
+      # the url address of python file
+      url: "./examples/pcb-aoi/incremental_learning_bench/fault_detection/testenv/f1_score.py"
+
+    # condition of triggering inference model to update
+    # threshold of the condition; types are float/int
+    threshold: 0.01
+    # operator of the condition; string type;
+    # values are ">=", ">", "<=", "<" and "=";
+    operator: ">="
+
+  # metrics configuration for test case's evaluation; list type;
+  metrics:
+      # metric name; string type;
+    - name: "f1_score"
+      # the url address of python file
+      url: "./examples/pcb-aoi/incremental_learning_bench/fault_detection/testenv/f1_score.py"
+    - name: "samples_transfer_ratio"
+
+  # incremental rounds setting of incremental learning; int type; default value is 2;
+  incremental_rounds: 2