diff --git a/Cross-cutting-with-face/SimilarImageFinder.py b/Cross-cutting-with-face/SimilarImageFinder.py index 75deb42..ccb6aaf 100644 --- a/Cross-cutting-with-face/SimilarImageFinder.py +++ b/Cross-cutting-with-face/SimilarImageFinder.py @@ -1,31 +1,33 @@ from yoloface import face_detector from util.distance import findEuclideanDistance from itertools import combinations -from util.VideoCapture import VideoCapture from util.plot import plot_by_cv2, plot_crop_face import torch from similarImageFinder_functions import * from verifier import FaceVerifier, VGGFace +import os class SimilarImageFinder: - def __init__(self, detector, video_name, video_path, period=30, video_capture=True): - self.video_name = video_name - if video_capture: - # 영상 -> 이미지 - VideoCapture(period, video_name, video_path) + def __init__(self, detector, dataset_path: os.PathLike, detection=True): + self.dataset_path = dataset_path + self.df = None + if detection: # 같은 프레임의 이미지들 중에서 적어도 2번 이상 사람이 감지된 프레임만 추출해서 data frame 생성 - df = self._make_df_detection_by_detector(detector, video_name) + df = self._make_df_detection_by_detector(detector, dataset_path) else: - df = pd.read_parquet(f'./dataset/{video_name}/detection_data/yoloface_data.parquet') + if os.path.isfile(self.dataset_path + '/filtered_detection_data.parquet'): + df = pd.read_parquet(self.dataset_path + '/filtered_detection_data.parquet') + else: + print('make detection dataframe first') # dataframe 이 없음 # 유사한 이미지 탐색 similarity_list = self.find_similarity_images(df) # 이미지 출력 - plot_by_cv2(similarity_list, video_name, 30) + plot_by_cv2(dataset_path, similarity_list, 30) - def _make_df_detection_by_detector(self, detector, video_name): + def _make_df_detection_by_detector(self, detector, dataset_path): """ Face Detection 데이터 프레임을 만드는 함수입니다. - :param: detector ['opencv', 'ssd', 'mtcnn', 'retinaface', 'yoloface] + :param: detector 'yoloface' :return: """ if detector == 'yoloface': @@ -33,19 +35,22 @@ def _make_df_detection_by_detector(self, detector, video_name): torch.tensor(1).cuda() yolo_detector = face_detector.YoloDetector(target_size=720, gpu=0, min_face=90) print('gpu 사용') - df_detection = detect_images_by_gpu(yolo_detector, video_name, batch_size=16) + df_detection = detect_images_by_gpu(yolo_detector, dataset_path, batch_size=16) except: yolo_detector = face_detector.YoloDetector(target_size=720, gpu=-1, min_face=90) print('cpu 사용, 시간이 오래걸리기 때문에 gpu를 이용하는 것을 권장합니다.') - df_detection = detect_images_by_cpu(yolo_detector, video_name) + df_detection = detect_images_by_cpu(yolo_detector, dataset_path) else: print('아직 미구현, Deepface Detector 는 얼굴을 추출하기 때문에 customizing 필요합니다') # frame_num 순으로 정렬 df_detection = df_detection.sort_values(by='frame_num') + # 데이터프레임 저장 + df_detection.to_parquet(self.dataset_path + '/detection_data.parquet', engine='pyarrow') + + self.df = df_detection.copy() # filter 된 dataframe df_filtered = filter_df(df_detection) - # 데이터프레임 저장 - df_filtered.to_parquet(f'./dataset/{video_name}/detection_data/yoloface_data.parquet', engine='pyarrow') + df_filtered.to_parquet(self.dataset_path + '/filtered_detection_data.parquet', engine='pyarrow') return df_filtered @@ -55,51 +60,58 @@ def find_similarity_images(self, df_filtered): for frame in df_filtered.index.unique(): df_temp = df_filtered.loc[frame] # 감지된 사람 수 - detect_person_num = df_temp['detect_person_num'].iloc[0] - # 영상 번호 리스트 - video_num_list = list(df_temp['video_num']) + detect_person_num_list = df_temp['detect_person_num'].unique() + # 선택된 영상 번호 selected_video = None # 거리 dis_min = 100000000 - for selected_video_nums in list(combinations(video_num_list, 2)): - videonum1, videonum2 = selected_video_nums[0], selected_video_nums[1] - boxes = list(df_temp.loc[(df_temp['video_num'] == videonum1) | (df_temp['video_num'] == videonum2)]['boxes']) - imgs_path = [f'./dataset/{self.video_name}/frame/{frame}/{videonum1}.jpg', - f'./dataset/{self.video_name}/frame/{frame}/{videonum2}.jpg'] + for n_person in detect_person_num_list: + df_temp2 = df_temp[df_temp['detect_person_num'] == n_person] + # 영상 번호 리스트 + video_id_list = list(df_temp2['video_id']) + + for selected_video_ids in list(combinations(video_id_list, 2)): + videoid1, videoid2 = selected_video_ids[0], selected_video_ids[1] + boxes = list(df_temp2.loc[(df_temp2['video_id'] == videoid1) | (df_temp2['video_id'] == videoid2)]['boxes']) + imgs_path = [self.dataset_path + f'/{videoid1}/{frame}.jpeg', + self.dataset_path + f'/{videoid2}/{frame}.jpeg'] - # 2개의 이미지에서 가장 큰 얼굴 간의 비율, face verification 을 위한 crop face image - area_fraction, crop_face1, crop_face2 = get_max_area_fraction_and_crop_faces(boxes, imgs_path) + # 2개의 이미지에서 가장 큰 얼굴 간의 비율, face verification 을 위한 crop face image + area_fraction, crop_face1, crop_face2 = get_max_area_fraction_and_crop_faces(boxes, imgs_path) - # #Verified Test : 검증 확인 - # verified = FaceVerifier.verify(crop_face1, crop_face2, model)['verified'] - # #check not verified face - # if not verified: plot_crop_face(crop_face1, crop_face2) - # # check verified face - # if verified: plot_crop_face(crop_face1, crop_face2) + # #Verified Test : 검증 확인 + # verified = FaceVerifier.verify(crop_face1, crop_face2, model)['verified'] + # #check not verified face + # if not verified: plot_crop_face(crop_face1, crop_face2) + # # check verified face + # if verified: plot_crop_face(crop_face1, crop_face2) - # compare area_fraction and face recognition - if 0.8 < area_fraction < 1.2 and FaceVerifier.verify(crop_face1, crop_face2, model)['verified']: - landmarks = list( - df_temp.loc[(df_temp['video_num'] == videonum1) | (df_temp['video_num'] == videonum2)]['landmarks']) + # compare area_fraction and face recognition + if 0.8 < area_fraction < 1.2 and FaceVerifier.verify(crop_face1, crop_face2, model)['verified']: + landmarks = list( + df_temp2.loc[(df_temp2['video_id'] == videoid1) | (df_temp2['video_id'] == videoid2)]['landmarks'] + ) - dis = findEuclideanDistance(landmarks[0], landmarks[1], detect_person_num) + dis = findEuclideanDistance(landmarks[0], landmarks[1], n_person) - if dis < dis_min: - dis_min = dis - selected_video = selected_video_nums + if dis < dis_min: + dis_min = dis + selected_video = selected_video_ids if selected_video is not None: similarity_list.append((frame, selected_video, dis_min)) similarity_list = sorted(similarity_list, key=lambda x: x[2]) + print(similarity_list) return similarity_list if __name__ == '__main__': - video_path = 'C:/Users/JungSupLim/Desktop/video' + dataset_path = './dataset/tomboy' # video capture 생략하고 싶으면 False - SimilarImageFinder('yoloface', 'idle_tomboy3', video_path, period=30, video_capture=True) + # SimilarImageFinder('yoloface', 'idle_tomboy3', video_path, period=30, video_capture=True) + SimilarImageFinder('yoloface', dataset_path, detection=False) diff --git a/Cross-cutting-with-face/similarImageFinder_functions.py b/Cross-cutting-with-face/similarImageFinder_functions.py index b3f606c..565c43a 100644 --- a/Cross-cutting-with-face/similarImageFinder_functions.py +++ b/Cross-cutting-with-face/similarImageFinder_functions.py @@ -2,17 +2,11 @@ import numpy as np import pandas as pd from PIL import Image +import os -def get_all_images(video_name): - """ - dataset 폴더 안에 있는 모든 이미지 경로를 반환하는 함수입니다. - """ - images = [] - frames = glob.glob(f'./dataset/{video_name}/frame/*') - for frame in frames: - for image in glob.glob(frame + '/*.jpg'): - images.append(image.replace('\\', '/')) +def get_all_images(dataset_path): + images = glob.glob(os.path.join(dataset_path, '**/*.jpeg')) return images def _zip_person(boxes, points, gpu=True): @@ -66,13 +60,13 @@ def generate_batch(lst, batch_size): for i in range(0, len(lst), batch_size): yield lst[i: i + batch_size] -def detect_images_by_gpu(detector, video_name, batch_size=32): +def detect_images_by_gpu(detector, dataset_path, batch_size=32): """ image batch 와 gpu 를 이용해 이미지들을 분석하는 함수 입니다. :return df_detection(감지한 """ - df_detection = pd.DataFrame(columns=['frame_num', 'video_num', 'detect_person_num', 'boxes', 'landmarks']) - images = get_all_images(video_name) + df_detection = pd.DataFrame(columns=['frame_num', 'video_id', 'detect_person_num', 'boxes', 'landmarks']) + images = get_all_images(dataset_path) image_path_batches = [x for x in generate_batch(images, batch_size=batch_size)] for image_path_batch in image_path_batches: @@ -87,36 +81,37 @@ def detect_images_by_gpu(detector, video_name, batch_size=32): for detected_image in list(zip(boxes,points, image_paths)): # image 별로 boxes = detected_image[0] points = detected_image[1] - image_path = detected_image[2] + image_path = detected_image[2].replace('\\','/') # corresponding window path # 영상 번호 - video_num = image_path.split('/')[-1][:-4] # [:-4] -> .jpg remove + video_id = image_path.split('/')[-2] # frame 번호 - frame_num = image_path.split('/')[-2] + frame_num = image_path.split('/')[-1][:-5] # [:-5] -> .jpeg remove people = _sort_by_x1(_zip_person(boxes,points)) if len(people) > 0: data = { - 'frame_num': int(frame_num), - 'video_num': int(video_num), + 'frame_num': frame_num, + 'video_id': video_id, 'detect_person_num': len(people), 'boxes': [person[0] for person in people], 'landmarks': [person[1] for person in people] } df_detection = df_detection.append(data, ignore_index=True) - print(f'{frame_num}번 째 frame : {video_num} 영상 이미지 저장') + print(f'{frame_num}번 째 frame : {video_id} 영상 이미지 저장') return df_detection -def detect_images_by_cpu(detector, video_name): +def detect_images_by_cpu(detector, dataset_path): """ cpu 를 이용한 """ - df_detection = pd.DataFrame(columns=['frame_num', 'video_num', 'detect_person_num', 'boxes', 'landmarks']) - images = get_all_images(video_name) # 모든 이미지 경로를 가져옵니다. + df_detection = pd.DataFrame(columns=['frame_num', 'video_id', 'detect_person_num', 'boxes', 'landmarks']) + images = get_all_images(dataset_path) # 모든 이미지 경로를 가져옵니다. for image in images: + image = image.replace('\\', '/') # 영상 번호 - video_num = image.split('/')[-1][:-4] # [:-4] -> .jpg remove + video_id = image.split('/')[-2] # frame 번호 - frame_num = image.split('/')[-2] + frame_num = image.split('/')[-1][:-5] # [:-5] -> .jpeg remove # 이미지 경로 -> ndarray image = np.array(Image.open(image)) # get boundary box and landmarks @@ -126,14 +121,14 @@ def detect_images_by_cpu(detector, video_name): # 1명 이상인 경우 / 1명만 검출하고 싶으면 == 1 로 변경 if len(people) > 0: data = { - 'frame_num': int(frame_num), - 'video_num': int(video_num), + 'frame_num': frame_num, + 'video_id': video_id, 'detect_person_num': len(people), 'boxes': [person[0] for person in people], 'landmarks': [person[1] for person in people] } df_detection = df_detection.append(data, ignore_index=True) - print(f'{frame_num}번 째 frame : {video_num} 영상 이미지 저장') + print(f'{frame_num}번 째 frame : {video_id} 영상 이미지 저장') return df_detection def filter_df(df): @@ -147,7 +142,7 @@ def filter_df(df): # 빈 df_filtered 생성 df_filtered = pd.DataFrame( - columns=['frame_num', 'video_num', 'detect_person_num', 'boxes', 'landmarks'] + columns=['frame_num', 'video_id', 'detect_person_num', 'boxes', 'landmarks'] ).set_index('frame_num') for frame_num in frame_index: diff --git a/Cross-cutting-with-face/util/plot.py b/Cross-cutting-with-face/util/plot.py index bbce0b2..28c6bc9 100644 --- a/Cross-cutting-with-face/util/plot.py +++ b/Cross-cutting-with-face/util/plot.py @@ -3,7 +3,7 @@ from PIL import Image import numpy as np -def plot_by_matplotlib(similarity_list, video_name, plot_limit=20): +def plot_by_matplotlib(path, similarity_list, video_name, plot_limit=20): n = 0 for frame, selected_video, _ in similarity_list: fig = plt.figure() @@ -11,34 +11,32 @@ def plot_by_matplotlib(similarity_list, video_name, plot_limit=20): if n > plot_limit: break xlabels = ['selected_image1', 'selected_image2'] i = 1 - selected_video_num1 = selected_video[0] - selected_video_num2 = selected_video[1] ax = fig.add_subplot(1, 2, i) - ax.imshow(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video_num1}.jpg')) + ax.imshow(Image.open(path + f'/{selected_video[0]}/{frame}.jpeg')) ax.set_xlabel(xlabels[0]) ax.set_xticks([]), ax.set_yticks([]) ax = fig.add_subplot(1, 2, i + 1) - ax.imshow(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video_num2}.jpg')) + ax.imshow(Image.open(path + f'/{selected_video[0]}/{frame}.jpeg')) ax.set_xlabel(xlabels[1]) ax.set_xticks([]), ax.set_yticks([]) n += 1 fig.show() -def plot_by_cv2(similarity_list, video_name, plot_limit=20): +def plot_by_cv2(path, similarity_list, plot_limit=20): n = 0 i = 1 for frame, selected_video, _ in similarity_list: if n > plot_limit: break - img1 = np.array(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video[0]}.jpg').resize((480, 270))) + img1 = np.array(Image.open(path + f'/{selected_video[0]}/{frame}.jpeg').resize((480, 270))) img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB) - img2 = np.array(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video[1]}.jpg').resize((480, 270))) + img2 = np.array(Image.open(path + f'/{selected_video[1]}/{frame}.jpeg').resize((480, 270))) img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB) combine_imgs = np.concatenate((img1, img2), axis=0) - cv2.imshow(f'{i}번째로 제일 유사한 Frame', combine_imgs) + cv2.imshow(f'{i} Similarly Frame', combine_imgs) cv2.waitKey() i += 1 n += 1