diff --git a/core/testcasecontroller/metrics/metrics.py b/core/testcasecontroller/metrics/metrics.py
index 112dc691..b03ce7ce 100644
--- a/core/testcasecontroller/metrics/metrics.py
+++ b/core/testcasecontroller/metrics/metrics.py
@@ -49,41 +49,65 @@ def samples_transfer_ratio_func(system_metric_info: dict):
def compute(key, matrix):
"""
- compute BWT and FWT
+ Compute BWT and FWT scores for a given matrix.
"""
# pylint: disable=C0103
+ print(f"compute function: key={key}, matrix={matrix}, type(matrix)={type(matrix)}")
+
length = len(matrix)
accuracy = 0.0
BWT_score = 0.0
FWT_score = 0.0
flag = True
- for i in range(length):
- if len(matrix[i]) != length-1:
+
+ if key == 'all':
+ for i in range(length-1, 0, -1):
+ sum_before_i = sum(item['accuracy'] for item in matrix[i][:i])
+ sum_after_i = sum(item['accuracy'] for item in matrix[i][-(length - i - 1):])
+ if i == 0:
+ seen_class_accuracy = 0.0
+ else:
+ seen_class_accuracy = sum_before_i / i
+ if length - 1 - i == 0:
+ unseen_class_accuracy = 0.0
+ else:
+ unseen_class_accuracy = sum_after_i / (length - 1 - i)
+ print(f"round {i} : unseen class accuracy is {unseen_class_accuracy}, seen class accuracy is {seen_class_accuracy}")
+
+ for row in matrix:
+ if not isinstance(row, list) or len(row) != length-1:
flag = False
break
- if flag is False:
+
+ if not flag:
BWT_score = np.nan
FWT_score = np.nan
return BWT_score, FWT_score
for i in range(length-1):
- accuracy += matrix[length-1][i]['accuracy']
- BWT_score += matrix[length-1][i]['accuracy'] - matrix[i+1][i]['accuracy']
- for i in range(0,length-1):
- FWT_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy']
- accuracy = accuracy/(length)
- BWT_score = BWT_score/(length-1)
- FWT_score = FWT_score/(length-1)
- #print(f"{key} accuracy: ", accuracy)
- print(f"{key} BWT_score: ", BWT_score)
- print(f"{key} FWT_score: ", FWT_score)
+ for j in range(length-1):
+ if 'accuracy' in matrix[i+1][j] and 'accuracy' in matrix[i][j]:
+ accuracy += matrix[i+1][j]['accuracy']
+ BWT_score += matrix[i+1][j]['accuracy'] - matrix[i][j]['accuracy']
+
+ for i in range(0, length-1):
+ if 'accuracy' in matrix[i][i] and 'accuracy' in matrix[0][i]:
+ FWT_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy']
+
+ accuracy = accuracy / ((length-1) * (length-1))
+ BWT_score = BWT_score / ((length-1) * (length-1))
+ FWT_score = FWT_score / (length-1)
+
+ print(f"{key} BWT_score: {BWT_score}")
+ print(f"{key} FWT_score: {FWT_score}")
+
my_matrix = []
for i in range(length-1):
my_matrix.append([])
- for i in range(length-1):
for j in range(length-1):
- my_matrix[i].append(matrix[i+1][j]['accuracy'])
- #self.draw_picture(key,my_matrix)
+ if 'accuracy' in matrix[i+1][j]:
+ my_matrix[i].append(matrix[i+1][j]['accuracy'])
+
return my_matrix, BWT_score, FWT_score
def bwt_func(system_metric_info: dict):
diff --git a/docs/proposals/algorithms/lifelong-learning/Implementation of a Class Incremental Learning Algorithm Evaluation System based on Ianvs.md b/docs/proposals/algorithms/lifelong-learning/Implementation of a Class Incremental Learning Algorithm Evaluation System based on Ianvs.md
new file mode 100644
index 00000000..b516515a
--- /dev/null
+++ b/docs/proposals/algorithms/lifelong-learning/Implementation of a Class Incremental Learning Algorithm Evaluation System based on Ianvs.md
@@ -0,0 +1,270 @@
+
+
+- [Implementation of a Class Incremental Learning Algorithm Evaluation System based on Ianvs](#implementation-of-a-class-incremental-learning-algorithm-evaluation-system-based-on-ianvs)
+ - [Motivation](#motivation)
+ - [Background](#background)
+ - [Goals](#goals)
+ - [Proposal](#proposal)
+ - [Design Details](#design-details)
+ - [Overall Design](#overall-design)
+ - [Datasets](#datasets)
+ - [File-level Design](#file-level-design)
+ - [Test Environment](#test-environment)
+ - [Test Algorithm](#test-algorithm)
+ - [Test Report](#test-report)
+ - [Roadmap](#roadmap)
+ - [Phase 1 July 1st - August 15th](#phase-1-july-1st---august-15th)
+ - [Phase 2 August 16th - September 30th](#phase-2-august-16th---september-30th)
+
+
+
+# Implementation of a Class Incremental Learning Algorithm Evaluation System based on Ianvs
+
+## 1 Motivation
+
+### 1.1 Background
+Currently, lifelong learning is facing a challenge: new classes may appear when models are trained on a new data domain ( for example, in the figure below, three classes in red are new classes in `Domain 2` ), which makes it difficult for models to maintain generalization ability and results in a severe performance drop.
+
+
+
+
+
+
+
+
Welcome to the RestNet API!
+ To use this service, send a POST HTTP request to {this-url}/predict
+ The JSON payload has the following format: {"image": "BASE64_STRING_OF_IMAGE",
+ "depth": "BASE64_STRING_OF_DEPTH"}
+ """)
+
+ async def predict(self, image: UploadFile = File(...), depth: Optional[UploadFile] = None) -> ResultResponse:
+ contents = await image.read()
+ recieve_img_time = time.time()
+ print("Recieve image from the robo:", recieve_img_time)
+
+ image = Image.open(BytesIO(contents)).convert('RGB')
+
+ img_dep = None
+ self.index_frame = self.index_frame + 1
+
+ if depth:
+ depth_contents = await depth.read()
+ depth = Image.open(BytesIO(depth_contents)).convert('RGB')
+ img_dep = cv2.resize(np.array(depth), (2048, 1024), interpolation=cv2.INTER_CUBIC)
+ img_dep = Image.fromarray(img_dep)
+
+ img_rgb = cv2.resize(np.array(image), (2048, 1024), interpolation=cv2.INTER_CUBIC)
+ img_rgb = Image.fromarray(img_rgb)
+
+ sample = {'image': img_rgb, "depth": img_dep, "label": img_rgb}
+ results = sedna_predict.predict(self.job, data=sample, validator=self.detection_validator)
+
+ predict_finish_time = time.time()
+ print(f"Prediction costs {predict_finish_time - recieve_img_time} seconds")
+
+ post_process = True
+ if results["result"]["box"] is None:
+ results["result"]["curr"] = None
+ results["result"]["future"] = None
+ elif post_process:
+ curr, future = get_curb(results["result"]["box"])
+ results["result"]["curr"] = curr
+ results["result"]["future"] = future
+ results["result"]["box"] = None
+ print("Post process cost at worker:", (time.time()-predict_finish_time))
+ else:
+ results["result"]["curr"] = None
+ results["result"]["future"] = None
+
+ print("Result transmit to robo time:", time.time())
+ return results
+
+def parse_result(label, count):
+ label_map = ['road', 'sidewalk', ]
+ count_d = dict(zip(label, count))
+ curb_count = count_d.get(19, 0)
+ if curb_count / np.sum(count) > 0.3:
+ return "curb"
+ r = sorted(label, key=count_d.get, reverse=True)[0]
+ try:
+ c = label_map[r]
+ except:
+ c = "other"
+
+ return c
+
+def get_curb(results):
+ results = np.array(results[0])
+ input_height, input_width = results.shape
+
+ closest = np.array([
+ [0, int(input_height)],
+ [int(input_width),
+ int(input_height)],
+ [int(0.118 * input_width + .5),
+ int(.8 * input_height + .5)],
+ [int(0.882 * input_width + .5),
+ int(.8 * input_height + .5)],
+ ])
+
+ future = np.array([
+ [int(0.118 * input_width + .5),
+ int(.8 * input_height + .5)],
+ [int(0.882 * input_width + .5),
+ int(.8 * input_height + .5)],
+ [int(.765 * input_width + .5),
+ int(.66 * input_height + .5)],
+ [int(.235 * input_width + .5),
+ int(.66 * input_height + .5)]
+ ])
+
+ mask = np.zeros((input_height, input_width), dtype=np.uint8)
+ mask = cv2.fillPoly(mask, [closest], 1)
+ mask = cv2.fillPoly(mask, [future], 2)
+ d1, c1 = np.unique(results[mask == 1], return_counts=True)
+ d2, c2 = np.unique(results[mask == 2], return_counts=True)
+ c = parse_result(d1, c1)
+ f = parse_result(d2, c2)
+
+ return c, f
+
+if __name__ == '__main__':
+ web_app = InferenceServer("lifelong-learning-robo", host=get_host_ip())
+ web_app.start()
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_evaluate.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_evaluate.py
new file mode 100644
index 00000000..56633347
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_evaluate.py
@@ -0,0 +1,50 @@
+import os
+os.environ['BACKEND_TYPE'] = 'PYTORCH'
+# os.environ["KB_SERVER"] = "http://0.0.0.0:9020"
+# os.environ["test_dataset_url"] = "./data_txt/sedna_data.txt"
+# os.environ["MODEL_URLS"] = "./cloud_next_kb/index.pkl"
+# os.environ["operator"] = "<"
+# os.environ["model_threshold"] = "0"
+
+from sedna.core.lifelong_learning import LifelongLearning
+from sedna.datasources import IndexDataParse
+from sedna.common.config import Context
+
+from accuracy import accuracy
+from basemodel import Model
+
+def _load_txt_dataset(dataset_url):
+ # use original dataset url
+ original_dataset_url = Context.get_parameters('original_dataset_url')
+ return os.path.join(os.path.dirname(original_dataset_url), dataset_url)
+
+def eval():
+ estimator = Model()
+ eval_dataset_url = Context.get_parameters("test_dataset_url")
+ eval_data = IndexDataParse(data_type="eval", func=_load_txt_dataset)
+ eval_data.parse(eval_dataset_url, use_raw=False)
+
+ task_allocation = {
+ "method": "TaskAllocationByOrigin",
+ "param": {
+ "origins": ["real", "sim"]
+ }
+ }
+
+ ll_job = LifelongLearning(estimator,
+ task_definition=None,
+ task_relationship_discovery=None,
+ task_allocation=task_allocation,
+ task_remodeling=None,
+ inference_integrate=None,
+ task_update_decision=None,
+ unseen_task_allocation=None,
+ unseen_sample_recognition=None,
+ unseen_sample_re_recognition=None
+ )
+
+ ll_job.evaluate(eval_data, metrics=accuracy)
+
+
+if __name__ == '__main__':
+ print(eval())
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_predict.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_predict.py
new file mode 100644
index 00000000..b32c01d2
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_predict.py
@@ -0,0 +1,132 @@
+import os
+
+os.environ['BACKEND_TYPE'] = 'PYTORCH'
+# os.environ["UNSEEN_SAVE_URL"] = "s3://kubeedge/sedna-robo/unseen_samples/"
+# set at yaml
+# os.environ["PREDICT_RESULT_DIR"] = "./inference_results"
+os.environ["TEST_DATASET_URL"] = "./data_txt/door_test.txt"
+os.environ["EDGE_OUTPUT_URL"] = "./edge_kb"
+os.environ["ORIGINAL_DATASET_URL"] = "/tmp"
+
+import torch
+import numpy as np
+from PIL import Image
+import base64
+import tempfile
+from io import BytesIO
+from torchvision.transforms import ToPILImage
+from torchvision import transforms
+from torch.utils.data import DataLoader
+
+from sedna.datasources import IndexDataParse
+from sedna.core.lifelong_learning import LifelongLearning
+from sedna.common.config import Context
+
+from eval import Validator
+from accuracy import accuracy
+from basemodel import preprocess, val_args, Model
+from dataloaders.utils import Colorize
+from dataloaders import custom_transforms as tr
+from dataloaders.datasets.cityscapes import CityscapesSegmentation
+
+def _load_txt_dataset(dataset_url):
+ # use original dataset url,
+ # see https://github.com/kubeedge/sedna/issues/35
+ original_dataset_url = Context.get_parameters('original_dataset_url')
+ return os.path.join(os.path.dirname(original_dataset_url), dataset_url)
+
+def fetch_data():
+ test_dataset_url = Context.get_parameters("test_dataset_url")
+ test_data = IndexDataParse(data_type="test", func=_load_txt_dataset)
+ test_data.parse(test_dataset_url, use_raw=False)
+ return test_data
+
+def pre_data_process(samples):
+ composed_transforms = transforms.Compose([
+ tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
+ tr.ToTensor()])
+
+ data = BaseDataSource(data_type="test")
+ data.x = [(composed_transforms(samples), "")]
+ return data
+
+def post_process(res, is_unseen_task):
+ if is_unseen_task:
+ res, base64_string = None, None
+ else:
+ res = res[0].tolist()
+
+ type = 0 if not is_unseen_task else 1
+ mesg = {
+ "msg": "",
+ "result": {
+ "type": type,
+ "box": res
+ },
+ "code": 0
+ }
+ return mesg
+
+def image_merge(raw_img, result):
+ raw_img = ToPILImage()(raw_img)
+
+ pre_colors = Colorize()(torch.from_numpy(result))
+ pre_color_image = ToPILImage()(pre_colors[0]) # pre_colors.dtype = float64
+
+ image = raw_img.resize(pre_color_image.size, Image.BILINEAR)
+ image = image.convert('RGBA')
+ label = pre_color_image.convert('RGBA')
+ image = Image.blend(image, label, 0.6)
+ with tempfile.NamedTemporaryFile(suffix='.png') as f:
+ image.save(f.name)
+
+ with open(f.name, 'rb') as open_file:
+ byte_content = open_file.read()
+ base64_bytes = base64.b64encode(byte_content)
+ base64_string = base64_bytes.decode('utf-8')
+ return base64_string
+
+def init_ll_job():
+ estimator = Model()
+ inference_integrate = {
+ "method": "BBoxInferenceIntegrate"
+ }
+ unseen_task_allocation = {
+ "method": "UnseenTaskAllocationDefault"
+ }
+ unseen_sample_recognition = {
+ "method": "SampleRegonitionByRFNet"
+ }
+
+ ll_job = LifelongLearning(
+ estimator,
+ task_definition=None,
+ task_relationship_discovery=None,
+ task_allocation=None,
+ task_remodeling=None,
+ inference_integrate=inference_integrate,
+ task_update_decision=None,
+ unseen_task_allocation=unseen_task_allocation,
+ unseen_sample_recognition=unseen_sample_recognition,
+ unseen_sample_re_recognition=None)
+
+ args = val_args()
+ args.weight_path = "./models/detection_model.pth"
+ args.num_class = 31
+
+ return ll_job, Validator(args, unseen_detection=True)
+
+def predict(ll_job, data=None, validator=None):
+ if data:
+ data = pre_data_process(data)
+ else:
+ data = fetch_data()
+ data.x = preprocess(data.x)
+
+ res, is_unseen_task, _ = ll_job.inference(
+ data, validator=validator, initial=False)
+ return post_process(res, is_unseen_task)
+
+if __name__ == '__main__':
+ ll_job, validator = init_ll_job()
+ print("Inference result:", predict(ll_job, validator=validator))
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_train.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_train.py
new file mode 100644
index 00000000..1c99361a
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/sedna_train.py
@@ -0,0 +1,78 @@
+import os
+os.environ['BACKEND_TYPE'] = 'PYTORCH'
+os.environ["OUTPUT_URL"] = "./cloud_kb/"
+# os.environ['CLOUD_KB_INDEX'] = "./cloud_kb/index.pkl"
+os.environ["TRAIN_DATASET_URL"] = "./data_txt/sedna_data.txt"
+os.environ["KB_SERVER"] = "http://0.0.0.0:9020"
+os.environ["HAS_COMPLETED_INITIAL_TRAINING"] = "false"
+
+from sedna.common.file_ops import FileOps
+from sedna.datasources import IndexDataParse
+from sedna.common.config import Context, BaseConfig
+from sedna.core.lifelong_learning import LifelongLearning
+
+from basemodel import Model
+
+def _load_txt_dataset(dataset_url):
+ # use original dataset url
+ original_dataset_url = Context.get_parameters('original_dataset_url')
+ return os.path.join(os.path.dirname(original_dataset_url), dataset_url)
+
+def train(estimator, train_data):
+ task_definition = {
+ "method": "TaskDefinitionByOrigin",
+ "param": {
+ "origins": ["real", "sim"]
+ }
+ }
+
+ task_allocation = {
+ "method": "TaskAllocationByOrigin",
+ "param": {
+ "origins": ["real", "sim"]
+ }
+ }
+
+ ll_job = LifelongLearning(estimator,
+ task_definition=task_definition,
+ task_relationship_discovery=None,
+ task_allocation=task_allocation,
+ task_remodeling=None,
+ inference_integrate=None,
+ task_update_decision=None,
+ unseen_task_allocation=None,
+ unseen_sample_recognition=None,
+ unseen_sample_re_recognition=None
+ )
+
+ ll_job.train(train_data)
+
+def update(estimator, train_data):
+ ll_job = LifelongLearning(estimator,
+ task_definition=None,
+ task_relationship_discovery=None,
+ task_allocation=None,
+ task_remodeling=None,
+ inference_integrate=None,
+ task_update_decision=None,
+ unseen_task_allocation=None,
+ unseen_sample_recognition=None,
+ unseen_sample_re_recognition=None
+ )
+
+ ll_job.update(train_data)
+
+def run():
+ estimator = Model()
+ train_dataset_url = BaseConfig.train_dataset_url
+ train_data = IndexDataParse(data_type="train")
+ train_data.parse(train_dataset_url, use_raw=False)
+
+ is_completed_initilization = str(Context.get_parameters("HAS_COMPLETED_INITIAL_TRAINING", "false")).lower()
+ if is_completed_initilization == "false":
+ train(estimator, train_data)
+ else:
+ update(estimator, train_data)
+
+if __name__ == '__main__':
+ run()
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/test.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/test.py
new file mode 100644
index 00000000..7350632b
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/test.py
@@ -0,0 +1,25 @@
+import numpy as np
+import seaborn as sns
+import pandas as pd
+import matplotlib.pyplot as plt
+
+CPA_results = np.load("./cpa_results.npy").T
+ratios = [0.3, 0.5, 0.6, 0.7, 0.8, 0.9]
+ratio_counts = np.zeros((len(CPA_results), len(ratios)), dtype=float)
+
+for i in range(len(CPA_results)):
+ for j in range(len(ratios)):
+ result = CPA_results[i]
+ result = result[result <= ratios[j]]
+
+ ratio_counts[i][j] = len(result) / 275
+
+plt.figure(figsize=(45, 10))
+ratio_counts = pd.DataFrame(data=ratio_counts.T, index=ratios)
+sns.heatmap(data=ratio_counts, annot=True, cmap="YlGnBu", annot_kws={'fontsize': 15})
+plt.xticks(fontsize=20)
+plt.yticks(fontsize=25)
+plt.xlabel("Test images", fontsize=25)
+plt.ylabel("Ratio of PA ranges", fontsize=25)
+plt.savefig("./figs/ratio_count.png")
+plt.show()
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/train.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/train.py
new file mode 100644
index 00000000..ecd87553
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/train.py
@@ -0,0 +1,361 @@
+import argparse
+import os
+import numpy as np
+from tqdm import tqdm
+import torch
+import copy
+
+from mypath import Path
+from dataloaders import make_data_loader
+
+from models.erfnet_RA_parallel import Net as Net_RAP
+
+from utils.loss import SegmentationLosses
+from models.replicate import patch_replication_callback
+from utils.calculate_weights import calculate_weigths_labels
+from utils.lr_scheduler import LR_Scheduler
+from utils.saver import Saver
+from utils.summaries import TensorboardSummary
+from utils.metrics import Evaluator
+from sedna.datasources import BaseDataSource
+
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+
+class Trainer(object):
+ def __init__(self, args, train_data=None, valid_data=None):
+ self.args = args
+ # Define Saver
+ self.saver = Saver(args)
+ self.saver.save_experiment_config()
+ # Define Tensorboard Summary
+ self.summary = TensorboardSummary(self.saver.experiment_dir)
+ self.writer = self.summary.create_summary()
+ # denormalize for detph image
+ self.mean_depth = torch.as_tensor(0.12176, dtype=torch.float32, device='cpu')
+ self.std_depth = torch.as_tensor(0.09752, dtype=torch.float32, device='cpu')
+
+ self.nclass = args.num_class # [13, 30, 30]
+ self.current_domain = min(args.current_domain, 2) # current domain start from 0 and maximum is 2
+ self.next_domain = args.next_domain # next_domain start from 1
+
+ if self.current_domain <= 0:
+ self.current_class = [self.nclass[0]]
+ elif self.current_domain == 1:
+ self.current_class = self.nclass[:2]
+ elif self.current_domain >= 2:
+ self.current_class = self.nclass
+ else:
+ pass
+
+ # Define Dataloader
+ kwargs = {'num_workers': args.workers, 'pin_memory': True}
+ self.train_loader, self.val_loader, self.test_loader, _ = make_data_loader(args, train_data=train_data,
+ valid_data=valid_data, **kwargs)
+
+ self.print_domain_info()
+
+ # Define network
+ model = Net_RAP(num_classes=self.current_class, nb_tasks=self.current_domain + 1, cur_task=self.current_domain)
+ model_old = Net_RAP(num_classes=self.current_class, nb_tasks=self.current_domain, cur_task=max(self.current_domain-1, 0))
+ args.current_domain = self.next_domain
+ args.next_domain += 1
+
+ optimizer = torch.optim.Adam(model.parameters(), lr=args.lr,
+ weight_decay=args.weight_decay)
+ # Define Criterion
+ # whether to use class balanced weights
+ if args.use_balanced_weights:
+ classes_weights_path = os.path.join(Path.db_root_dir(args.dataset), args.dataset+'_classes_weights.npy')
+ if os.path.isfile(classes_weights_path):
+ weight = np.load(classes_weights_path)
+ else:
+ weight = calculate_weigths_labels(args.dataset, self.train_loader, self.nclass)
+ weight = torch.from_numpy(weight.astype(np.float32))
+ else:
+ weight = None
+ # Define loss function
+ self.criterion = SegmentationLosses(weight=weight, cuda=args.cuda, gpu_ids=args.gpu_ids).build_loss(mode=args.loss_type)
+ self.model, self.model_old, self.optimizer = model, model_old, optimizer
+ # Define Evaluator
+ self.evaluator = Evaluator(self.nclass[self.current_domain])
+ # # Define lr scheduler
+ self.scheduler = LR_Scheduler(args.lr_scheduler, args.lr, args.epochs, len(self.train_loader))
+ # Using cuda
+ if args.cuda:
+ self.model = torch.nn.DataParallel(self.model)
+ # patch_replication_callback(self.model)
+ self.model = self.model.cuda(args.gpu_ids)
+ self.gpu_ids = args.gpu_ids
+ # Resuming checkpoint
+ self.best_pred = 0.0
+ if args.resume is not None:
+ if not os.path.isfile(args.resume):
+ raise RuntimeError("=> no checkpoint found at '{}'" .format(args.resume))
+ print(f"Training: load model from {args.resume}")
+ checkpoint = torch.load(args.resume, map_location=torch.device('cuda:0'))
+ args.start_epoch = checkpoint['epoch']
+
+ self.model.load_state_dict(checkpoint['state_dict'], False)
+
+ if not args.ft:
+ self.optimizer.load_state_dict(checkpoint['optimizer'])
+ self.best_pred = checkpoint['best_pred']
+ print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']))
+
+ # Clear start epoch if fine-tuning
+ if args.ft:
+ args.start_epoch = 0
+
+ def get_weight(self):
+ print("get weight")
+ current_model = copy.deepcopy(self.model)
+ return current_model.parameters()
+
+ def set_weight(self, weights):
+ length = len(weights)
+ print("set weight", length)
+ print("model:", self.args.resume)
+ tau = 0.2
+ if length == 1:
+ for param, target_param in zip(weights[0], self.model.parameters()):
+ target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
+ elif length == 2:
+ for param1, param2, target_param in zip(weights[0], weights[1], self.model.parameters()):
+ target_param.data.copy_(0.5 * tau * param1.data + 0.5 * tau * param2.data + (1 - tau) * target_param.data)
+
+ def my_training(self, epoch):
+ train_loss = 0.0
+ print(self.optimizer.state_dict()['param_groups'][0]['lr'])
+ current_model = copy.deepcopy(self.model)
+ self.model.train()
+ tbar = tqdm(self.train_loader)
+ num_img_tr = len(self.train_loader)
+
+ for i, sample in enumerate(tbar):
+ if self.args.depth:
+ image, depth, target = sample['image'], sample['depth'], sample['label']
+ #print(target.shape)
+ else:
+ image, target = sample['image'], sample['label']
+ print(image.shape)
+ if self.args.cuda:
+ image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids)
+ if self.args.depth:
+ depth = depth.cuda(self.args.gpu_ids)
+ self.scheduler(self.optimizer, i, epoch, self.best_pred)
+ self.optimizer.zero_grad()
+
+ if self.args.depth:
+ output = self.model(image, depth)
+ else:
+ output = self.model(image)
+ target[target > self.nclass[2]-1] = 255
+ loss = self.criterion(output, target)
+ loss.backward()
+ self.optimizer.step()
+ #print(self.optimizer.state_dict()['param_groups'][0]['lr'])
+ train_loss += loss.item()
+ tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))
+ self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch)
+ # Show 10 * 3 inference results each epoch
+ if i % (num_img_tr // 10 + 1) == 0:
+ global_step = i + num_img_tr * epoch
+ if self.args.depth:
+ self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step)
+
+ depth_display = depth[0].cpu().unsqueeze(0)
+ depth_display = depth_display.mul_(self.std_depth).add_(self.mean_depth)
+ depth_display = depth_display.numpy()
+ depth_display = depth_display*255
+ depth_display = depth_display.astype(np.uint8)
+ self.writer.add_image('Depth', depth_display, global_step)
+
+ else:
+ self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step)
+
+ self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch)
+ print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
+ print('Loss: %.3f' % train_loss)
+ tau = 0.3
+ flag = True
+ for param, target_param in zip(current_model.parameters(), self.model.parameters()):
+ if flag:
+ flag = False
+ target_param.data.copy_(tau * param.data + (1 - tau) * target_param.data)
+
+ del current_model
+ return train_loss
+
+ def training(self, epoch):
+ train_loss = 0.0
+ print(self.optimizer.state_dict()['param_groups'][0]['lr'])
+
+ self.model.train()
+ self.model_old.eval()
+
+ for name, m in self.model_old.named_parameters():
+ m.requires_grad = False
+
+ for name, m in self.model.named_parameters():
+ if 'decoder' in name:
+ if 'decoder.{}'.format(self.current_domain) in name:
+ m.requires_grad = True
+ else:
+ m.requires_grad = False
+
+ elif 'encoder' in name:
+ if 'bn' in name or 'parallel_conv' in name:
+ if '.{}.weight'.format(self.current_domain) in name or '.{}.bias'.format(self.current_domain) in name:
+ m.requires_grad = True
+ else:
+ m.requires_grad = False
+
+ tbar = tqdm(self.train_loader)
+ num_img_tr = len(self.train_loader)
+
+ for i, sample in enumerate(tbar):
+ if self.args.depth:
+ image, depth, target = sample['image'], sample['depth'], sample['label']
+ #print(target.shape)
+ else:
+ image, target = sample['image'], sample['label']
+ # print(image.shape)
+ if self.args.cuda:
+ image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids)
+ if self.args.depth:
+ depth = depth.cuda(self.args.gpu_ids)
+ self.scheduler(self.optimizer, i, epoch, self.best_pred)
+ self.optimizer.zero_grad()
+
+ if self.args.depth:
+ output = self.model(image, depth)
+ else:
+ output = self.model(image, self.current_domain)
+
+ output = torch.tensor(output, dtype=torch.float32)
+ target[target > self.nclass[self.current_domain]-1] = 255
+
+ target = self.my_to_label(target)
+ target = self.my_relabel(target, 255, self.nclass[self.current_domain] - 1)
+
+ target = target.squeeze(0)
+ target = target.cuda(self.gpu_ids)
+
+ outputs_prev_task = self.model(image, max(self.current_domain-1, 0))
+ loss = self.criterion(output, target)
+
+ loss.requires_grad_(True)
+ loss.backward()
+ self.optimizer.step()
+
+ train_loss += loss.item()
+ tbar.set_description('Train loss: %.3f' % (train_loss / (i + 1)))
+ self.writer.add_scalar('train/total_loss_iter', loss.item(), i + num_img_tr * epoch)
+
+ # Show 10 * 3 inference results each epoch
+ if i % (num_img_tr // 10 + 1) == 0:
+ global_step = i + num_img_tr * epoch
+ if self.args.depth:
+ self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step)
+
+ depth_display = depth[0].cpu().unsqueeze(0)
+ depth_display = depth_display.mul_(self.std_depth).add_(self.mean_depth)
+ depth_display = depth_display.numpy()
+ depth_display = depth_display*255
+ depth_display = depth_display.astype(np.uint8)
+ self.writer.add_image('Depth', depth_display, global_step)
+
+ else:
+ self.summary.visualize_image(self.writer, self.args.dataset, image, target, output, global_step)
+
+ self.writer.add_scalar('train/total_loss_epoch', train_loss, epoch)
+ print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
+ print('Loss: %.3f' % train_loss)
+
+ # save checkpoint every epoch
+ checkpoint_path = self.saver.save_checkpoint({
+ 'epoch': epoch + 1,
+ 'state_dict': self.model.state_dict(),
+ 'optimizer': self.optimizer.state_dict(),
+ 'best_pred': self.best_pred,
+ }, True)
+ return train_loss
+
+ def validation(self, epoch):
+ self.model.eval()
+ self.evaluator.reset()
+ tbar = tqdm(self.val_loader, desc='\r')
+ test_loss = 0.0
+ for i, (sample, img_path) in enumerate(tbar):
+ if self.args.depth:
+ image, depth, target = sample['image'], sample['depth'], sample['label']
+ else:
+ image, target = sample['image'], sample['label']
+ # print(f"val image is {image}")
+ if self.args.cuda:
+ image, target = image.cuda(self.args.gpu_ids), target.cuda(self.args.gpu_ids)
+ if self.args.depth:
+ depth = depth.cuda(self.args.gpu_ids)
+ with torch.no_grad():
+ if self.args.depth:
+ output = self.model(image, depth)
+ else:
+ output = self.model(image)
+ target[target > self.nclass[2]-1] = 255
+ loss = self.criterion(output, target)
+ test_loss += loss.item()
+ tbar.set_description('Test loss: %.3f' % (test_loss / (i + 1)))
+ pred = output.data.cpu().numpy()
+ target = target.cpu().numpy()
+ pred = np.argmax(pred, axis=1)
+ # Add batch sample into evaluator
+ self.evaluator.add_batch(target, pred)
+
+ # Fast test during the training
+ Acc = self.evaluator.Pixel_Accuracy()
+ Acc_class = self.evaluator.Pixel_Accuracy_Class()
+ mIoU = self.evaluator.Mean_Intersection_over_Union()
+ FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()
+ self.writer.add_scalar('val/total_loss_epoch', test_loss, epoch)
+ self.writer.add_scalar('val/mIoU', mIoU, epoch)
+ self.writer.add_scalar('val/Acc', Acc, epoch)
+ self.writer.add_scalar('val/Acc_class', Acc_class, epoch)
+ self.writer.add_scalar('val/fwIoU', FWIoU, epoch)
+ print('Validation:')
+ print('[Epoch: %d, numImages: %5d]' % (epoch, i * self.args.batch_size + image.data.shape[0]))
+ print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(Acc, Acc_class, mIoU, FWIoU))
+ print('Loss: %.3f' % test_loss)
+
+ new_pred = mIoU
+ if new_pred > self.best_pred:
+ is_best = True
+ self.best_pred = new_pred
+ self.saver.save_checkpoint({
+ 'epoch': epoch + 1,
+ 'state_dict': self.model.state_dict(),
+ 'optimizer': self.optimizer.state_dict(),
+ 'best_pred': self.best_pred,
+ }, is_best)
+
+
+ def print_domain_info(self):
+
+ domain_map = {
+ 0: "Synthia",
+ 1: "CityScapes",
+ 2: "Cloud-Robotics"
+ }
+
+ domain_name = domain_map.get(self.current_domain, "Unknown Domain")
+
+ print("We are in domain", self.current_domain, "which is", domain_name)
+
+ def my_relabel(self, tensor, olabel, nlabel):
+ tensor[tensor == olabel] = nlabel
+ return tensor
+
+ def my_to_label(self, image):
+ image = image.cpu()
+ return torch.from_numpy(np.array(image)).long().unsqueeze(0)
+
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/__init__.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/args.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/args.py
new file mode 100644
index 00000000..6b445914
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/args.py
@@ -0,0 +1,61 @@
+class TrainArgs:
+ def __init__(self, **kwargs):
+ self.depth = False
+ self.dataset = 'cityscapes'
+ self.workers = 4
+ self.base_size = 1024
+ self.crop_size = 768
+ self.loss_type = 'ce'
+ self.epochs = kwargs.get("epochs", 1)
+ self.start_epoch = 0
+
+ self.num_class = [13, 30, 30]
+ self.current_domain = 0
+ self.next_domain = 1
+
+ self.state = None
+ self.batch_size = 2
+ self.val_batch_size = 1
+ self.use_balanced_weights = False
+
+ self.lr = kwargs.get("learning_rate", 1e-4)
+ self.lr_scheduler = 'cos'
+ self.momentum = 0.9
+ self.weight_decay = 2.5e-5
+ self.no_cuda = False
+ self.gpu_ids = 0
+
+ self.seed = 1
+ self.resume = None
+ self.checkname = 'erfnet_RA_parallel'
+ self.ft = True
+ self.eval_interval = kwargs.get("eval_interval", 50)
+ self.no_val = kwargs.get("no_val", True)
+ self.cuda = True
+ self.savedir = '/home/QXY/dataset/save'
+
+class ValArgs:
+ def __init__(self, **kwargs):
+ self.dataset = 'cityscapes'
+ self.workers = 0
+ self.base_size = 1024
+ self.crop_size = 768
+ self.batch_size = 6
+ self.val_batch_size = 1
+ self.test_batch_size = 1
+
+ self.num_class = [13, 30, 30]
+ self.current_domain = 0
+ self.next_domain = 1
+
+ self.no_cuda = False
+ self.gpu_ids = 0
+ self.checkname = None
+ self.weight_path = "./models/530_exp3_2.pth"
+ self.save_predicted_image = False
+ self.color_label_save_path = './test/color'
+ self.merge_label_save_path = './test/merge'
+ self.label_save_path = './test/label'
+ self.merge = True
+ self.depth = False
+ self.cuda = True
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/calculate_weights.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/calculate_weights.py
new file mode 100644
index 00000000..2c2c9821
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/calculate_weights.py
@@ -0,0 +1,29 @@
+import os
+from tqdm import tqdm
+import numpy as np
+from mypath import Path
+
+def calculate_weigths_labels(dataset, dataloader, num_classes):
+ # Create an instance from the data loader
+ z = np.zeros((num_classes,))
+ # Initialize tqdm
+ tqdm_batch = tqdm(dataloader)
+ print('Calculating classes weights')
+ for sample in tqdm_batch:
+ y = sample['label']
+ y = y.detach().cpu().numpy()
+ mask = (y >= 0) & (y < num_classes)
+ labels = y[mask].astype(np.uint8)
+ count_l = np.bincount(labels, minlength=num_classes)
+ z += count_l
+ tqdm_batch.close()
+ total_frequency = np.sum(z)
+ class_weights = []
+ for frequency in z:
+ class_weight = 1 / (np.log(1.02 + (frequency / total_frequency)))
+ class_weights.append(class_weight)
+ ret = np.array(class_weights)
+ classes_weights_path = os.path.join(Path.db_root_dir(dataset), dataset+'_classes_weights.npy')
+ np.save(classes_weights_path, ret)
+
+ return ret
\ No newline at end of file
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/iouEval.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/iouEval.py
new file mode 100644
index 00000000..d3801a9e
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/iouEval.py
@@ -0,0 +1,141 @@
+import torch
+
+class iouEval:
+
+ def __init__(self, nClasses, ignoreIndex=20):
+
+ self.nClasses = nClasses
+ self.ignoreIndex = ignoreIndex if nClasses > ignoreIndex else -1 # if ignoreIndex is larger than nClasses, consider no ignoreIndex
+ self.reset()
+
+ def reset(self):
+ classes = self.nClasses if self.ignoreIndex == -1 else self.nClasses - 1
+ self.tp = torch.zeros(classes).double()
+ self.fp = torch.zeros(classes).double()
+ self.fn = torch.zeros(classes).double()
+ self.cdp_obstacle = torch.zeros(1).double()
+ self.tp_obstacle = torch.zeros(1).double()
+ self.idp_obstacle = torch.zeros(1).double()
+ self.tp_nonobstacle = torch.zeros(1).double()
+ # self.cdi = torch.zeros(1).double()
+
+ def addBatch(self, x, y): # x=preds, y=targets
+ # sizes should be "batch_size x nClasses x H x W"
+ # cdi = 0
+
+ # print ("X is cuda: ", x.is_cuda)
+ # print ("Y is cuda: ", y.is_cuda)
+
+ if (x.is_cuda or y.is_cuda):
+ x = x.cuda()
+ y = y.cuda()
+
+ # if size is "batch_size x 1 x H x W" scatter to onehot
+ if (x.size(1) == 1):
+ x_onehot = torch.zeros(x.size(0), self.nClasses, x.size(2), x.size(3))
+ if x.is_cuda:
+ x_onehot = x_onehot.cuda()
+ x_onehot.scatter_(1, x, 1).float() # dim index src 按照列用1替换0，索引为x
+ else:
+ x_onehot = x.float()
+
+ if (y.size(1) == 1):
+ y_onehot = torch.zeros(y.size(0), self.nClasses, y.size(2), y.size(3))
+ if y.is_cuda:
+ y_onehot = y_onehot.cuda()
+ y_onehot.scatter_(1, y, 1).float()
+ else:
+ y_onehot = y.float()
+
+ if (self.ignoreIndex != -1):
+ ignores = y_onehot[:, self.ignoreIndex].unsqueeze(1) # 加一维
+ x_onehot = x_onehot[:, :self.ignoreIndex] # ignoreIndex后的都不要
+ y_onehot = y_onehot[:, :self.ignoreIndex]
+ else:
+ ignores = 0
+
+
+ tpmult = x_onehot * y_onehot # times prediction and gt coincide is 1
+ tp = torch.sum(torch.sum(torch.sum(tpmult, dim=0, keepdim=True), dim=2, keepdim=True), dim=3,
+ keepdim=True).squeeze()
+ fpmult = x_onehot * (
+ 1 - y_onehot - ignores) # times prediction says its that class and gt says its not (subtracting cases when its ignore label!)
+ fp = torch.sum(torch.sum(torch.sum(fpmult, dim=0, keepdim=True), dim=2, keepdim=True), dim=3,
+ keepdim=True).squeeze()
+ fnmult = (1 - x_onehot) * (y_onehot) # times prediction says its not that class and gt says it is
+ fn = torch.sum(torch.sum(torch.sum(fnmult, dim=0, keepdim=True), dim=2, keepdim=True), dim=3,
+ keepdim=True).squeeze()
+
+ self.tp += tp.double().cpu()
+ self.fp += fp.double().cpu()
+ self.fn += fn.double().cpu()
+
+ cdp_obstacle = tpmult[:, 19].sum() # obstacle index 19
+ tp_obstacle = y_onehot[:, 19].sum()
+
+ idp_obstacle = (x_onehot[:, 19] - tpmult[:, 19]).sum()
+ tp_nonobstacle = (-1*y_onehot+1).sum()
+
+ # for i in range(0, x.size(0)):
+ # if tpmult[i].sum()/(y_onehot[i].sum() + 1e-15) >= 0.5:
+ # cdi += 1
+
+
+ self.cdp_obstacle += cdp_obstacle.double().cpu()
+ self.tp_obstacle += tp_obstacle.double().cpu()
+ self.idp_obstacle += idp_obstacle.double().cpu()
+ self.tp_nonobstacle += tp_nonobstacle.double().cpu()
+ # self.cdi += cdi.double().cpu()
+
+
+
+ def getIoU(self):
+ num = self.tp
+ den = self.tp + self.fp + self.fn + 1e-15
+ iou = num / den
+ iou_not_zero = list(filter(lambda x: x != 0, iou))
+ # print(len(iou_not_zero))
+ iou_mean = sum(iou_not_zero) / len(iou_not_zero)
+ tfp = self.tp + self.fp + 1e-15
+ acc = num / tfp
+ acc_not_zero = list(filter(lambda x: x != 0, acc))
+ acc_mean = sum(acc_not_zero) / len(acc_not_zero)
+
+ return iou_mean, iou, acc_mean, acc # returns "iou mean", "iou per class"
+
+ def getObstacleEval(self):
+
+ pdr_obstacle = self.cdp_obstacle / (self.tp_obstacle+1e-15)
+ pfp_obstacle = self.idp_obstacle / (self.tp_nonobstacle+1e-15)
+
+ return pdr_obstacle, pfp_obstacle
+
+
+# Class for colors
+class colors:
+ RED = '\033[31;1m'
+ GREEN = '\033[32;1m'
+ YELLOW = '\033[33;1m'
+ BLUE = '\033[34;1m'
+ MAGENTA = '\033[35;1m'
+ CYAN = '\033[36;1m'
+ BOLD = '\033[1m'
+ UNDERLINE = '\033[4m'
+ ENDC = '\033[0m'
+
+
+# Colored value output if colorized flag is activated.
+def getColorEntry(val):
+ if not isinstance(val, float):
+ return colors.ENDC
+ if (val < .20):
+ return colors.RED
+ elif (val < .40):
+ return colors.YELLOW
+ elif (val < .60):
+ return colors.BLUE
+ elif (val < .80):
+ return colors.CYAN
+ else:
+ return colors.GREEN
+
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/loss.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/loss.py
new file mode 100644
index 00000000..6f184503
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/loss.py
@@ -0,0 +1,59 @@
+import torch
+import torch.nn as nn
+
+class SegmentationLosses(object):
+ def __init__(self, weight=None, size_average=True, batch_average=True, ignore_index=255, cuda=False, gpu_ids=0): # ignore_index=255
+ self.ignore_index = ignore_index
+ self.weight = weight
+ self.size_average = size_average
+ self.batch_average = batch_average
+ self.cuda = cuda
+ self.gpu_ids = gpu_ids
+
+ def build_loss(self, mode='ce'):
+ """Choices: ['ce' or 'focal']"""
+ if mode == 'ce':
+ return self.CrossEntropyLoss
+ elif mode == 'focal':
+ return self.FocalLoss
+ else:
+ raise NotImplementedError
+
+ def CrossEntropyLoss(self, logit, target):
+ criterion = nn.CrossEntropyLoss()
+ if self.cuda:
+ criterion = criterion.cuda(self.gpu_ids)
+
+ loss = criterion(logit, target.long())
+
+ return loss
+
+ def FocalLoss(self, logit, target, gamma=2, alpha=0.5):
+ n, c, h, w = logit.size()
+ criterion = nn.CrossEntropyLoss(weight=self.weight, ignore_index=self.ignore_index,
+ size_average=self.size_average)
+ if self.cuda:
+ criterion = criterion.cuda(self.gpu_ids)
+
+ logpt = -criterion(logit, target.long())
+ pt = torch.exp(logpt)
+ if alpha is not None:
+ logpt *= alpha
+ loss = -((1 - pt) ** gamma) * logpt
+
+ if self.batch_average:
+ loss /= n
+
+ return loss
+
+if __name__ == "__main__":
+ loss = SegmentationLosses(cuda=True)
+ a = torch.rand(1, 3, 7, 7).cuda()
+ b = torch.rand(1, 7, 7).cuda()
+ print(loss.CrossEntropyLoss(a, b).item())
+ print(loss.FocalLoss(a, b, gamma=0, alpha=None).item())
+ print(loss.FocalLoss(a, b, gamma=2, alpha=0.5).item())
+
+
+
+
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/lr_scheduler.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/lr_scheduler.py
new file mode 100644
index 00000000..47124028
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/lr_scheduler.py
@@ -0,0 +1,70 @@
+##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+## Created by: Hang Zhang
+## ECE Department, Rutgers University
+## Email: zhang.hang@rutgers.edu
+## Copyright (c) 2017
+##
+## This source code is licensed under the MIT-style license found in the
+## LICENSE file in the root directory of this source tree
+##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+import math
+
+class LR_Scheduler(object):
+ """Learning Rate Scheduler
+
+ Step mode: ``lr = baselr * 0.1 ^ {floor(epoch-1 / lr_step)}``
+
+ Cosine mode: ``lr = baselr * 0.5 * (1 + cos(iter/maxiter))``
+
+ Poly mode: ``lr = baselr * (1 - iter/maxiter) ^ 0.9``
+
+ Args:
+ args:
+ :attr:`args.lr_scheduler` lr scheduler mode (`cos`, `poly`),
+ :attr:`args.lr` base learning rate, :attr:`args.epochs` number of epochs,
+ :attr:`args.lr_step`
+
+ iters_per_epoch: number of iterations per epoch
+ """
+ def __init__(self, mode, base_lr, num_epochs, iters_per_epoch=0,
+ lr_step=0, warmup_epochs=0):
+ self.mode = mode
+ print('Using {} LR Scheduler!'.format(self.mode))
+ self.lr = base_lr
+ if mode == 'step':
+ assert lr_step
+ self.lr_step = lr_step
+ self.iters_per_epoch = iters_per_epoch
+ self.N = num_epochs * iters_per_epoch
+ self.epoch = -1
+ self.warmup_iters = warmup_epochs * iters_per_epoch
+
+ def __call__(self, optimizer, i, epoch, best_pred):
+ T = epoch * self.iters_per_epoch + i
+ if self.mode == 'cos':
+ lr = 0.5 * self.lr * (1 + math.cos(1.0 * T / self.N * math.pi))
+ elif self.mode == 'poly':
+ lr = self.lr * pow((1 - 1.0 * T / self.N), 2)
+ elif self.mode == 'step':
+ lr = self.lr * (0.1 ** (epoch // self.lr_step))
+ else:
+ raise NotImplemented
+ # warm up lr schedule
+ if self.warmup_iters > 0 and T < self.warmup_iters:
+ lr = lr * 1.0 * T / self.warmup_iters
+ if epoch > self.epoch:
+ print('\n=>Epoches %i, learning rate = %.4f, \
+ previous best = %.4f' % (epoch, lr, best_pred))
+ self.epoch = epoch
+ assert lr >= 0
+ self._adjust_learning_rate(optimizer, lr)
+
+ def _adjust_learning_rate(self, optimizer, lr):
+ if len(optimizer.param_groups) == 1:
+ optimizer.param_groups[0]['lr'] = lr * 4
+ else:
+ # enlarge the lr at the head
+ optimizer.param_groups[0]['lr'] = lr * 4
+ for i in range(1, len(optimizer.param_groups)):
+ optimizer.param_groups[i]['lr'] = lr
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/metrics.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/metrics.py
new file mode 100644
index 00000000..2900ec2b
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/metrics.py
@@ -0,0 +1,78 @@
+import numpy as np
+
+
+class Evaluator(object):
+ def __init__(self, num_class):
+ self.num_class = num_class
+ self.confusion_matrix = np.zeros((self.num_class,)*2) # shape:(num_class, num_class)
+
+ def Pixel_Accuracy(self):
+ Acc = np.diag(self.confusion_matrix).sum() / self.confusion_matrix.sum()
+ return Acc
+
+ def Pixel_Accuracy_Class_Curb(self):
+ Acc = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1)
+ Acc = np.nanmean(Acc[:2])
+ return Acc
+
+
+ def Pixel_Accuracy_Class(self):
+ Acc = np.diag(self.confusion_matrix) / self.confusion_matrix.sum(axis=1)
+ Acc = np.nanmean(Acc)
+ return Acc
+
+ def Mean_Intersection_over_Union(self):
+ MIoU = np.diag(self.confusion_matrix) / (
+ np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) -
+ np.diag(self.confusion_matrix))
+ MIoU = np.nanmean(MIoU)
+ return MIoU
+
+ def Mean_Intersection_over_Union_Curb(self):
+ MIoU = np.diag(self.confusion_matrix) / (
+ np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) -
+ np.diag(self.confusion_matrix))
+ MIoU = np.nanmean(MIoU[:2])
+ return MIoU
+
+ def Frequency_Weighted_Intersection_over_Union(self):
+ freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix)
+ iu = np.diag(self.confusion_matrix) / (
+ np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) -
+ np.diag(self.confusion_matrix))
+
+ FWIoU = (freq[freq > 0] * iu[freq > 0]).sum()
+ CFWIoU = freq[freq > 0] * iu[freq > 0]
+ return FWIoU
+
+ def Frequency_Weighted_Intersection_over_Union_Curb(self):
+ freq = np.sum(self.confusion_matrix, axis=1) / np.sum(self.confusion_matrix)
+ iu = np.diag(self.confusion_matrix) / (
+ np.sum(self.confusion_matrix, axis=1) + np.sum(self.confusion_matrix, axis=0) -
+ np.diag(self.confusion_matrix))
+ CFWIoU = freq[freq > 0] * iu[freq > 0]
+
+ return np.nanmean(CFWIoU[:2])
+
+ def _generate_matrix(self, gt_image, pre_image):
+ mask = (gt_image >= 0) & (gt_image < self.num_class)
+ label = self.num_class * gt_image[mask].astype('int') + pre_image[mask]
+ count = np.bincount(label, minlength=self.num_class**2)
+ confusion_matrix = count.reshape(self.num_class, self.num_class)
+ return confusion_matrix
+
+ def add_batch(self, gt_image, pre_image):
+ gt_image = np.array(gt_image)
+ pre_image = np.array(pre_image)
+ print(gt_image.shape, pre_image.shape)
+ if gt_image.shape != pre_image.shape:
+ pre_image = pre_image[0]
+ assert gt_image.shape == pre_image.shape
+ self.confusion_matrix += self._generate_matrix(gt_image, pre_image)
+
+ def reset(self):
+ self.confusion_matrix = np.zeros((self.num_class,) * 2)
+
+
+
+
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/saver.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/saver.py
new file mode 100644
index 00000000..03866432
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/saver.py
@@ -0,0 +1,68 @@
+import os
+import time
+import shutil
+import tempfile
+import torch
+from collections import OrderedDict
+import glob
+
+class Saver(object):
+
+ def __init__(self, args):
+ self.args = args
+ self.directory = os.path.join('/tmp', args.dataset, args.checkname)
+ self.runs = sorted(glob.glob(os.path.join(self.directory, 'experiment_*')))
+ run_id = int(self.runs[-1].split('_')[-1]) + 1 if self.runs else 0
+
+ self.experiment_dir = os.path.join(self.directory, 'experiment_{}'.format(str(run_id)))
+ if not os.path.exists(self.experiment_dir):
+ os.makedirs(self.experiment_dir)
+
+ def save_checkpoint(self, state, is_best): # filename from .pth.tar change to .pth?
+ """Saves checkpoint to disk"""
+ filename = f'checkpoint_{time.time()}.pth'
+ checkpoint_path = os.path.join(self.experiment_dir, filename)
+ torch.save(state, checkpoint_path)
+ if is_best:
+ best_pred = state['best_pred']
+ with open(os.path.join(self.experiment_dir, 'best_pred.txt'), 'w') as f:
+ f.write(str(best_pred))
+ if self.runs:
+ previous_miou = [0.0]
+ for run in self.runs:
+ run_id = run.split('_')[-1]
+ path = os.path.join(self.directory, 'experiment_{}'.format(str(run_id)), 'best_pred.txt')
+ if os.path.exists(path):
+ with open(path, 'r') as f:
+ miou = float(f.readline())
+ previous_miou.append(miou)
+ else:
+ continue
+ max_miou = max(previous_miou)
+ if best_pred > max_miou:
+ checkpoint_path_best = os.path.join(self.directory, 'model_best.pth')
+ shutil.copyfile(checkpoint_path, checkpoint_path_best)
+ checkpoint_path = checkpoint_path_best
+ else:
+ checkpoint_path_best = os.path.join(self.directory, 'model_best.pth')
+ shutil.copyfile(checkpoint_path, checkpoint_path_best)
+ checkpoint_path = checkpoint_path_best
+
+ return checkpoint_path
+
+ def save_experiment_config(self):
+ logfile = os.path.join(self.experiment_dir, 'parameters.txt')
+ log_file = open(logfile, 'w')
+ p = OrderedDict()
+ p['datset'] = self.args.dataset
+ # p['out_stride'] = self.args.out_stride
+ p['lr'] = self.args.lr
+ p['lr_scheduler'] = self.args.lr_scheduler
+ p['loss_type'] = self.args.loss_type
+ p['epoch'] = self.args.epochs
+ p['base_size'] = self.args.base_size
+ p['crop_size'] = self.args.crop_size
+
+ for key, val in p.items():
+ log_file.write(key + ':' + str(val) + '\n')
+ log_file.close()
\ No newline at end of file
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/summaries.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/summaries.py
new file mode 100644
index 00000000..04bcdb82
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/ERFNet/utils/summaries.py
@@ -0,0 +1,39 @@
+import os
+import torch
+from torchvision.utils import make_grid
+# from tensorboardX import SummaryWriter
+from torch.utils.tensorboard import SummaryWriter
+from dataloaders.utils import decode_seg_map_sequence
+
+class TensorboardSummary(object):
+ def __init__(self, directory):
+ self.directory = directory
+
+ def create_summary(self):
+ writer = SummaryWriter(log_dir=os.path.join(self.directory))
+ return writer
+
+ def visualize_image(self, writer, dataset, image, target, output, global_step, depth=None):
+ if depth is None:
+ grid_image = make_grid(image[:3].clone().cpu().data, 3, normalize=True)
+ writer.add_image('Image', grid_image, global_step)
+
+ grid_image = make_grid(decode_seg_map_sequence(torch.max(output[:3], 1)[1].detach().cpu().numpy(),
+ dataset=dataset), 3, normalize=False, range=(0, 255))
+ writer.add_image('Predicted label', grid_image, global_step)
+ grid_image = make_grid(decode_seg_map_sequence(torch.squeeze(target[:3], 1).detach().cpu().numpy(),
+ dataset=dataset), 3, normalize=False, range=(0, 255))
+ writer.add_image('Groundtruth label', grid_image, global_step)
+ else:
+ grid_image = make_grid(image[:3].clone().cpu().data, 4, normalize=True)
+ writer.add_image('Image', grid_image, global_step)
+
+ grid_image = make_grid(depth[:3].clone().cpu().data, 4, normalize=True) # normalize=False?
+ writer.add_image('Depth', grid_image, global_step)
+
+ grid_image = make_grid(decode_seg_map_sequence(torch.max(output[:3], 1)[1].detach().cpu().numpy(),
+ dataset=dataset), 4, normalize=False, range=(0, 255))
+ writer.add_image('Predicted label', grid_image, global_step)
+ grid_image = make_grid(decode_seg_map_sequence(torch.squeeze(target[:3], 1).detach().cpu().numpy(),
+ dataset=dataset), 4, normalize=False, range=(0, 255))
+ writer.add_image('Groundtruth label', grid_image, global_step)
\ No newline at end of file
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/basemodel.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/basemodel.py
new file mode 100644
index 00000000..97aca3de
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/basemodel.py
@@ -0,0 +1,142 @@
+import os
+import gc
+import numpy as np
+import torch
+from torch.utils.data import DataLoader
+from sedna.common.class_factory import ClassType, ClassFactory
+from sedna.common.config import Context
+from sedna.common.file_ops import FileOps
+from sedna.common.log import LOGGER
+from PIL import Image
+from torchvision import transforms
+
+from ERFNet.train import Trainer
+from ERFNet.eval import Validator, load_my_state_dict
+from ERFNet.dataloaders import custom_transforms as tr
+from ERFNet.dataloaders import make_data_loader
+from ERFNet.utils.args import TrainArgs, ValArgs
+
+# set backend
+os.environ['BACKEND_TYPE'] = 'PYTORCH'
+
+os.environ["OMP_NUM_THREADS"] = "1"
+os.environ["MKL_NUM_THREADS"] = "1"
+
+@ClassFactory.register(ClassType.GENERAL, alias="BaseModel")
+class BaseModel:
+ def __init__(self, **kwargs):
+ self.train_args = TrainArgs(**kwargs)
+ self.trainer = None
+
+ self.val_args = ValArgs(**kwargs)
+ label_save_dir = Context.get_parameters("INFERENCE_RESULT_DIR", "./inference_results")
+ self.val_args.color_label_save_path = os.path.join(label_save_dir, "color")
+ self.val_args.merge_label_save_path = os.path.join(label_save_dir, "merge")
+ self.val_args.label_save_path = os.path.join(label_save_dir, "label")
+ self.validator = Validator(self.val_args)
+
+ def get_weights(self):
+ return self.trainer.get_weight()
+
+ def set_weights(self, weights):
+ self.trainer.set_weight(weights)
+
+ epoch_num = 0
+ print("Total epoch: ", epoch_num)
+ loss_all = []
+ for epoch in range(epoch_num):
+ train_loss = self.trainer.my_training(epoch)
+ loss_all.append(train_loss)
+
+ def train(self, train_data, valid_data=None, **kwargs):
+ self.trainer = Trainer(self.train_args, train_data=train_data)
+ print("Total epoches:", self.trainer.args.epochs)
+ loss_all = []
+ for epoch in range(
+ self.trainer.args.start_epoch,
+ self.trainer.args.epochs):
+ if epoch == 0 and self.trainer.val_loader:
+ self.trainer.validation(epoch)
+ loss = self.trainer.training(epoch)
+ loss_all.append(loss)
+ if self.trainer.args.no_val and (
+ epoch %
+ self.trainer.args.eval_interval == (
+ self.trainer.args.eval_interval -
+ 1) or epoch == self.trainer.args.epochs -
+ 1):
+ is_best = False
+ self.train_model_url = self.trainer.saver.save_checkpoint({
+ 'epoch': epoch + 1,
+ 'state_dict': self.trainer.model.state_dict(),
+ 'optimizer': self.trainer.optimizer.state_dict(),
+ 'best_pred': self.trainer.best_pred,
+ }, is_best)
+
+ self.trainer.writer.close()
+ return self.train_model_url
+
+ def predict(self, data, **kwargs):
+ if len(data) > 10:
+ print("predict start for big data")
+ my_kwargs = {'num_workers': self.val_args.workers, 'pin_memory': True}
+ _, _, self.validator.test_loader, _ = make_data_loader(self.val_args, test_data=data, **my_kwargs)
+ else:
+ print("predict start for small data")
+ if not isinstance(data[0][0], dict):
+ data = self._preprocess(data)
+ if type(data) is np.ndarray:
+ data = data.tolist()
+ self.validator.test_loader = DataLoader(data, batch_size=self.val_args.test_batch_size, shuffle=False,
+ pin_memory=True)
+
+ return self.validator.validate()
+
+ def evaluate(self, data, **kwargs):
+ self.val_args.save_predicted_image = kwargs.get("save_predicted_image", True)
+ samples = self._preprocess(data.x)
+ predictions = self.predict(samples)
+ metric_name, metric_func = kwargs.get("metric")
+ if callable(metric_func):
+ return metric_func(data.y, predictions)
+ else:
+ raise Exception(f"not found model metric func(name={metric_name}) in model eval phase")
+
+ def load(self, model_url, **kwargs):
+ if model_url:
+ print("load model url: ",model_url)
+ self.validator.new_state_dict = torch.load(model_url, map_location=torch.device("cpu"))
+ self.train_args.resume = model_url
+ else:
+ raise Exception("model url does not exist.")
+ self.validator.model = load_my_state_dict(self.validator.model, self.validator.new_state_dict['state_dict'])
+
+ def save(self, model_path=None):
+ if not model_path:
+ LOGGER.warning(f"Not specify model path.")
+ return self.train_model_url
+
+ return FileOps.upload(self.train_model_url, model_path)
+
+ def _preprocess(self, image_urls):
+ transformed_images = []
+ for paths in image_urls:
+ if len(paths) == 2:
+ img_path, depth_path = paths
+ _img = Image.open(img_path).convert('RGB')
+ _depth = Image.open(depth_path)
+ else:
+ img_path = paths[0]
+ _img = Image.open(img_path).convert('RGB')
+ _depth = _img
+
+ sample = {'image': _img, 'depth': _depth, 'label': _img}
+ del _img
+ gc.collect()
+ composed_transforms = transforms.Compose([
+ tr.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
+ tr.ToTensor()])
+
+ transformed_images.append((composed_transforms(sample), img_path))
+
+ return transformed_images
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_allocation_by_domain.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_allocation_by_domain.py
new file mode 100644
index 00000000..dc4d9522
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_allocation_by_domain.py
@@ -0,0 +1,46 @@
+from sedna.datasources import BaseDataSource
+from sedna.common.class_factory import ClassFactory, ClassType
+
+__all__ = ('TaskAllocationByDomain',)
+
+
+@ClassFactory.register(ClassType.STP, alias="TaskAllocationByDomain")
+class TaskAllocationByOrigin:
+ """
+ Corresponding to `TaskDefinitionByOrigin`
+
+ Parameters
+ ----------
+ task_extractor : Dict
+ used to match target tasks
+ origins: List[Metadata]
+ metadata is usually a class feature
+ label with finite values.
+ """
+
+ def __init__(self, **kwargs):
+ self.default_origin = kwargs.get("default", None)
+
+ def __call__(self, task_extractor, samples: BaseDataSource):
+ self.task_extractor = {"Synthia": 0, "Cityscapes": 1, "Cloud-Robotics": 2} # Mapping of origins to task indices
+
+ if self.default_origin:
+ return samples, [int(self.task_extractor.get(self.default_origin))] * len(samples.x)
+
+ categories = ["Cityscapes", "Synthia", "Cloud-Robotics"] # List of all possible origins
+
+ sample_origins = []
+ for _x in samples.x:
+ sample_origin = None
+ for category in categories:
+ if category in _x[0]:
+ sample_origin = category
+ break
+ if sample_origin is None:
+ # If none of the categories match, assign a default origin
+ sample_origin = self.default_origin if self.default_origin else categories[0]
+ sample_origins.append(sample_origin)
+
+ allocations = [int(self.task_extractor.get(sample_origin)) for sample_origin in sample_origins]
+
+ return samples, allocations
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_definition_by_domain.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_definition_by_domain.py
new file mode 100644
index 00000000..eba0f74f
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_definition_by_domain.py
@@ -0,0 +1,56 @@
+from typing import List, Any, Tuple
+
+from sedna.datasources import BaseDataSource
+from sedna.common.class_factory import ClassType, ClassFactory
+from sedna.algorithms.seen_task_learning.artifact import Task
+
+__all__ = ('TaskDefinitionByDomain',)
+
+
+@ClassFactory.register(ClassType.STP, alias="TaskDefinitionByDomain")
+class TaskDefinitionByOrigin:
+ """
+ Dividing datasets based on their origins.
+
+ Parameters
+ ----------
+ origins： List[Metadata]
+ metadata is usually a class feature label with finite values.
+ """
+
+ def __init__(self, **kwargs):
+ self.origins = kwargs.get("origins", ["Cityscapes", "Synthia", "Cloud-Robotics"])
+
+ def __call__(self,
+ samples: BaseDataSource, **kwargs) -> Tuple[List[Task],
+ Any,
+ BaseDataSource]:
+ categories = self.origins
+
+ tasks = []
+ d_type = samples.data_type
+ x_data = samples.x
+ y_data = samples.y
+
+ task_index = dict(zip(categories, range(len(categories))))
+
+ data_sources = {category: BaseDataSource(data_type=d_type) for category in categories}
+ print(data_sources)
+
+ for category in data_sources.values():
+ category.x = []
+ category.y = []
+
+ for i in range(samples.num_examples()):
+ for category in categories:
+ if category in x_data[i]:
+ data_sources[category].x.append(x_data[i])
+ data_sources[category].y.append(y_data[i])
+ break
+
+ for category, data_source in data_sources.items():
+ task_name = f"{category}_semantic_segmentation_model"
+ task_obj = Task(entry=task_name, samples=data_source, meta_attr=category)
+ tasks.append(task_obj)
+
+ return tasks, task_index, samples
\ No newline at end of file
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/test_algorithm.yaml b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/test_algorithm.yaml
new file mode 100644
index 00000000..d39634c3
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/test_algorithm.yaml
@@ -0,0 +1,63 @@
+algorithm:
+ # paradigm type; string type;
+ # currently the options of value are as follows:
+ # 1> "singletasklearning"
+ # 2> "incrementallearning"
+ # 3> "lifelonglearning"
+ paradigm_type: "lifelonglearning"
+ lifelong_learning_data_setting:
+ # ratio of training dataset; float type;
+ # the default value is 0.8.
+ train_ratio: 0.9
+ # the method of splitting dataset; string type; optional;
+ # currently the options of value are as follows:
+ # 1> "default": the dataset is evenly divided based train_ratio;
+ # splitting_method: "default"
+ splitting_method: "fwt_splitting"
+
+ # algorithm module configuration in the paradigm; list type;
+ modules:
+ # type of algorithm module; string type;
+ # currently the options of value are as follows:
+ # 1> "basemodel": contains important interfaces such as train、 eval、 predict and more; required module;
+ - type: "basemodel"
+ # name of python module; string type;
+ # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking;
+ name: "BaseModel"
+ # the url address of python module; string type;
+ url: "./examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/basemodel.py"
+ # hyperparameters configuration for the python module; list type;
+ hyperparameters:
+ # name of the hyperparameter; string type;
+ - learning_rate:
+ values:
+ - 0.0001
+ - epochs:
+ values:
+ - 1
+ # 2> "task_definition": define lifelong task ; optional module;
+ - type: "task_definition"
+ # name of python module; string type;
+ name: "TaskDefinitionByDomain"
+ # the url address of python module; string type;
+ url: "./examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_definition_by_domain.py"
+ # hyperparameters configuration for the python module; list type;
+ hyperparameters:
+ # name of the hyperparameter; string type;
+ # origins of data; value is ["real", "sim"], this means that data from real camera and simulator.
+ - origins:
+ values:
+ - ["Cityscapes", "Synthia", "Cloud-Robotics"]
+ # 3> "task_allocation": allocate lifelong task ; optional module;
+ - type: "task_allocation"
+ # name of python module; string type;
+ name: "TaskAllocationByDomain"
+ # the url address of python module; string type;
+ url: "./examples/class_increment_semantic_segmentation/lifelong_learning_bench/testalgorithms/erfnet/task_allocation_by_domain.py"
+ # hyperparameters configuration for the python module; list type;
+ hyperparameters:
+ # name of the hyperparameter; string type;
+ # origins of data; value is ["real", "sim"], this means that data from real camera and simulator.
+ - origins:
+ values:
+ - ["Cityscapes", "Synthia", "Cloud-Robotics"]
\ No newline at end of file
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/accuracy.py b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/accuracy.py
new file mode 100644
index 00000000..4dd63613
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/accuracy.py
@@ -0,0 +1,54 @@
+# Copyright 2022 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from tqdm import tqdm
+
+from sedna.common.class_factory import ClassType, ClassFactory
+
+from ERFNet.dataloaders import make_data_loader
+from ERFNet.utils.metrics import Evaluator
+from ERFNet.utils.args import ValArgs
+
+__all__ = ('accuracy')
+
+
+@ClassFactory.register(ClassType.GENERAL, alias="accuracy")
+def accuracy(y_true, y_pred, **kwargs):
+ args = ValArgs()
+ _, _, test_loader, num_class = make_data_loader(args, test_data=y_true)
+ evaluator = Evaluator(num_class)
+ #print(y_true)
+ tbar = tqdm(test_loader, desc='\r')
+ for i, (sample, img_path) in enumerate(tbar):
+ if args.depth:
+ image, depth, target = sample['image'], sample['depth'], sample['label']
+ else:
+ image, target = sample['image'], sample['label']
+ if args.cuda:
+ image, target = image.cuda(args.gpu_ids), target.cuda(args.gpu_ids)
+ if args.depth:
+ depth = depth.cuda(args.gpu_ids)
+
+ target[target > evaluator.num_class-1] = 255
+ target = target.cpu().numpy()
+ evaluator.add_batch(target, y_pred[i])
+
+ # Test during the training
+ # Acc = evaluator.Pixel_Accuracy()
+ CPA = evaluator.Pixel_Accuracy_Class()
+ mIoU = evaluator.Mean_Intersection_over_Union()
+ FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union()
+
+ print("CPA:{}, mIoU:{}, fwIoU: {}".format(CPA, mIoU, FWIoU))
+ return mIoU
diff --git a/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/testenv.yaml b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/testenv.yaml
new file mode 100644
index 00000000..d00b4aa6
--- /dev/null
+++ b/examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/testenv.yaml
@@ -0,0 +1,38 @@
+testenv:
+ # dataset configuration
+ dataset:
+ # the url address of train dataset index; string type;
+ train_url: "/home/QXY/dataset/mdil-ss/train/mdil-ss-train-index-small.txt"
+ # the url address of test dataset index; string type;
+ test_url: "/home/QXY/dataset/mdil-ss/test/mdil-ss-test-index-small.txt"
+
+ # model eval configuration of incremental learning;
+ model_eval:
+ # metric used for model evaluation
+ model_metric:
+ # metric name; string type;
+ name: "accuracy"
+ # the url address of python file
+ url: "./examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/accuracy.py"
+ mode: "no-inference"
+
+ # condition of triggering inference model to update
+ # threshold of the condition; types are float/int
+ threshold: 0
+ # operator of the condition; string type;
+ # values are ">=", ">", "<=", "<" and "=";
+ operator: "<"
+
+ # metrics configuration for test case's evaluation; list type;
+ metrics:
+ # metric name; string type;
+ - name: "accuracy"
+ # the url address of python file
+ url: "./examples/class_increment_semantic_segmentation/lifelong_learning_bench/testenv/accuracy.py"
+ - name: "samples_transfer_ratio"
+ - name: "BWT"
+ - name: "FWT"
+ - name: "Matrix"
+
+ # incremental rounds setting; int type; default value is 2;
+ incremental_rounds: 3
\ No newline at end of file