Jungsup: Edit SimilarImageFinder - Corresponding with loader.video2frame

Cross-cutting-with-face/SimilarImageFinder.py

@@ -1,51 +1,56 @@
 from yoloface import face_detector
 from util.distance import findEuclideanDistance
 from itertools import combinations
-from util.VideoCapture import VideoCapture
 from util.plot import plot_by_cv2, plot_crop_face
 import torch
 from similarImageFinder_functions import *
 from verifier import FaceVerifier, VGGFace
+import os
 
 class SimilarImageFinder:
-    def __init__(self, detector, video_name, video_path, period=30, video_capture=True):
-        self.video_name = video_name
-        if video_capture:
-            # 영상 -> 이미지
-            VideoCapture(period, video_name, video_path)
+    def __init__(self, detector, dataset_path: os.PathLike, detection=True):
+        self.dataset_path = dataset_path
+        self.df = None
+        if detection:
             # 같은 프레임의 이미지들 중에서 적어도 2번 이상 사람이 감지된 프레임만 추출해서 data frame 생성
-            df = self._make_df_detection_by_detector(detector, video_name)
+            df = self._make_df_detection_by_detector(detector, dataset_path)
         else:
-            df = pd.read_parquet(f'./dataset/{video_name}/detection_data/yoloface_data.parquet')
+            if os.path.isfile(self.dataset_path + '/filtered_detection_data.parquet'):
+                df = pd.read_parquet(self.dataset_path + '/filtered_detection_data.parquet')
+            else:
+                print('make detection dataframe first') # dataframe 이 없음
         # 유사한 이미지 탐색
         similarity_list = self.find_similarity_images(df)
         # 이미지 출력
-        plot_by_cv2(similarity_list, video_name, 30)
+        plot_by_cv2(dataset_path, similarity_list, 30)
 
-    def _make_df_detection_by_detector(self, detector, video_name):
+    def _make_df_detection_by_detector(self, detector, dataset_path):
         """
         Face Detection 데이터 프레임을 만드는 함수입니다.
-        :param: detector ['opencv', 'ssd', 'mtcnn', 'retinaface', 'yoloface]
+        :param: detector 'yoloface'
         :return:
         """
         if detector == 'yoloface':
             try:
                 torch.tensor(1).cuda()
                 yolo_detector = face_detector.YoloDetector(target_size=720, gpu=0, min_face=90)
                 print('gpu 사용')
-                df_detection = detect_images_by_gpu(yolo_detector, video_name, batch_size=16)
+                df_detection = detect_images_by_gpu(yolo_detector, dataset_path, batch_size=16)
             except:
                 yolo_detector = face_detector.YoloDetector(target_size=720, gpu=-1, min_face=90)
                 print('cpu 사용, 시간이 오래걸리기 때문에 gpu를 이용하는 것을 권장합니다.')
-                df_detection = detect_images_by_cpu(yolo_detector, video_name)
+                df_detection = detect_images_by_cpu(yolo_detector, dataset_path)
         else:
             print('아직 미구현, Deepface Detector 는 얼굴을 추출하기 때문에 customizing 필요합니다')
         # frame_num 순으로 정렬
         df_detection = df_detection.sort_values(by='frame_num')
+        # 데이터프레임 저장
+        df_detection.to_parquet(self.dataset_path + '/detection_data.parquet', engine='pyarrow')
+
+        self.df = df_detection.copy()
         # filter 된 dataframe
         df_filtered = filter_df(df_detection)
-        # 데이터프레임 저장
-        df_filtered.to_parquet(f'./dataset/{video_name}/detection_data/yoloface_data.parquet', engine='pyarrow')
+        df_filtered.to_parquet(self.dataset_path + '/filtered_detection_data.parquet', engine='pyarrow')
 
         return df_filtered
 
@@ -55,51 +60,58 @@ def find_similarity_images(self, df_filtered):
         for frame in df_filtered.index.unique():
             df_temp = df_filtered.loc[frame]
             # 감지된 사람 수
-            detect_person_num = df_temp['detect_person_num'].iloc[0]
-            # 영상 번호 리스트
-            video_num_list = list(df_temp['video_num'])
+            detect_person_num_list = df_temp['detect_person_num'].unique()
+
             # 선택된 영상 번호
             selected_video = None
             # 거리
             dis_min = 100000000
 
-            for selected_video_nums in list(combinations(video_num_list, 2)):
-                videonum1, videonum2 = selected_video_nums[0], selected_video_nums[1]
-                boxes = list(df_temp.loc[(df_temp['video_num'] == videonum1) | (df_temp['video_num'] == videonum2)]['boxes'])
-                imgs_path = [f'./dataset/{self.video_name}/frame/{frame}/{videonum1}.jpg',
-                             f'./dataset/{self.video_name}/frame/{frame}/{videonum2}.jpg']
+            for n_person in detect_person_num_list:
+                df_temp2 = df_temp[df_temp['detect_person_num'] == n_person]
+                # 영상 번호 리스트
+                video_id_list = list(df_temp2['video_id'])
+
+                for selected_video_ids in list(combinations(video_id_list, 2)):
+                    videoid1, videoid2 = selected_video_ids[0], selected_video_ids[1]
+                    boxes = list(df_temp2.loc[(df_temp2['video_id'] == videoid1) | (df_temp2['video_id'] == videoid2)]['boxes'])
+                    imgs_path = [self.dataset_path + f'/{videoid1}/{frame}.jpeg',
+                                 self.dataset_path + f'/{videoid2}/{frame}.jpeg']
 
-                # 2개의 이미지에서 가장 큰 얼굴 간의 비율, face verification 을 위한 crop face image
-                area_fraction, crop_face1, crop_face2 = get_max_area_fraction_and_crop_faces(boxes, imgs_path)
+                    # 2개의 이미지에서 가장 큰 얼굴 간의 비율, face verification 을 위한 crop face image
+                    area_fraction, crop_face1, crop_face2 = get_max_area_fraction_and_crop_faces(boxes, imgs_path)
 
-                # #Verified Test : 검증 확인
-                # verified = FaceVerifier.verify(crop_face1, crop_face2, model)['verified']
-                # #check not verified face
-                # if not verified: plot_crop_face(crop_face1, crop_face2)
-                # # check verified face
-                # if verified: plot_crop_face(crop_face1, crop_face2)
+                    # #Verified Test : 검증 확인
+                    # verified = FaceVerifier.verify(crop_face1, crop_face2, model)['verified']
+                    # #check not verified face
+                    # if not verified: plot_crop_face(crop_face1, crop_face2)
+                    # # check verified face
+                    # if verified: plot_crop_face(crop_face1, crop_face2)
 
-                # compare area_fraction and face recognition
-                if 0.8 < area_fraction < 1.2 and FaceVerifier.verify(crop_face1, crop_face2, model)['verified']:
-                    landmarks = list(
-                        df_temp.loc[(df_temp['video_num'] == videonum1) | (df_temp['video_num'] == videonum2)]['landmarks'])
+                    # compare area_fraction and face recognition
+                    if 0.8 < area_fraction < 1.2 and FaceVerifier.verify(crop_face1, crop_face2, model)['verified']:
+                        landmarks = list(
+                            df_temp2.loc[(df_temp2['video_id'] == videoid1) | (df_temp2['video_id'] == videoid2)]['landmarks']
+                        )
 
-                    dis = findEuclideanDistance(landmarks[0], landmarks[1], detect_person_num)
+                        dis = findEuclideanDistance(landmarks[0], landmarks[1], n_person)
 
-                    if dis < dis_min:
-                        dis_min = dis
-                        selected_video = selected_video_nums
+                        if dis < dis_min:
+                            dis_min = dis
+                            selected_video = selected_video_ids
 
             if selected_video is not None:
                 similarity_list.append((frame, selected_video, dis_min))
 
         similarity_list = sorted(similarity_list, key=lambda x: x[2])
+        print(similarity_list)
 
         return similarity_list
 
 
 if __name__ == '__main__':
-    video_path = 'C:/Users/JungSupLim/Desktop/video'
+    dataset_path = './dataset/tomboy'
     # video capture 생략하고 싶으면 False
-    SimilarImageFinder('yoloface', 'idle_tomboy3', video_path, period=30, video_capture=True)
+    # SimilarImageFinder('yoloface', 'idle_tomboy3', video_path, period=30, video_capture=True)
+    SimilarImageFinder('yoloface', dataset_path, detection=False)
 

Cross-cutting-with-face/similarImageFinder_functions.py

@@ -2,17 +2,11 @@
 import numpy as np
 import pandas as pd
 from PIL import Image
+import os
 
 
-def get_all_images(video_name):
-    """
-    dataset 폴더 안에 있는 모든 이미지 경로를 반환하는 함수입니다.
-    """
-    images = []
-    frames = glob.glob(f'./dataset/{video_name}/frame/*')
-    for frame in frames:
-        for image in glob.glob(frame + '/*.jpg'):
-            images.append(image.replace('\\', '/'))
+def get_all_images(dataset_path):
+    images = glob.glob(os.path.join(dataset_path, '**/*.jpeg'))
     return images
 
 def _zip_person(boxes, points, gpu=True):
@@ -66,13 +60,13 @@ def generate_batch(lst, batch_size):
     for i in range(0, len(lst), batch_size):
         yield lst[i: i + batch_size]
 
-def detect_images_by_gpu(detector, video_name, batch_size=32):
+def detect_images_by_gpu(detector, dataset_path, batch_size=32):
     """
     image batch 와 gpu 를 이용해 이미지들을 분석하는 함수 입니다.
     :return df_detection(감지한
     """
-    df_detection = pd.DataFrame(columns=['frame_num', 'video_num', 'detect_person_num', 'boxes', 'landmarks'])
-    images = get_all_images(video_name)
+    df_detection = pd.DataFrame(columns=['frame_num', 'video_id', 'detect_person_num', 'boxes', 'landmarks'])
+    images = get_all_images(dataset_path)
     image_path_batches = [x for x in generate_batch(images, batch_size=batch_size)]
 
     for image_path_batch in image_path_batches:
@@ -87,36 +81,37 @@ def detect_images_by_gpu(detector, video_name, batch_size=32):
         for detected_image in list(zip(boxes,points, image_paths)): # image 별로
             boxes = detected_image[0]
             points = detected_image[1]
-            image_path = detected_image[2]
+            image_path = detected_image[2].replace('\\','/') # corresponding window path
             # 영상 번호
-            video_num = image_path.split('/')[-1][:-4]  # [:-4] -> .jpg remove
+            video_id = image_path.split('/')[-2]
             # frame 번호
-            frame_num = image_path.split('/')[-2]
+            frame_num = image_path.split('/')[-1][:-5] # [:-5] -> .jpeg remove
             people = _sort_by_x1(_zip_person(boxes,points))
 
             if len(people) > 0:
                 data = {
-                    'frame_num': int(frame_num),
-                    'video_num': int(video_num),
+                    'frame_num': frame_num,
+                    'video_id': video_id,
                     'detect_person_num': len(people),
                     'boxes': [person[0] for person in people],
                     'landmarks': [person[1] for person in people]
                 }
                 df_detection = df_detection.append(data, ignore_index=True)
-                print(f'{frame_num}번 째 frame : {video_num} 영상 이미지 저장')
+                print(f'{frame_num}번 째 frame : {video_id} 영상 이미지 저장')
     return df_detection
 
-def detect_images_by_cpu(detector, video_name):
+def detect_images_by_cpu(detector, dataset_path):
     """
     cpu 를 이용한
     """
-    df_detection = pd.DataFrame(columns=['frame_num', 'video_num', 'detect_person_num', 'boxes', 'landmarks'])
-    images = get_all_images(video_name)  # 모든 이미지 경로를 가져옵니다.
+    df_detection = pd.DataFrame(columns=['frame_num', 'video_id', 'detect_person_num', 'boxes', 'landmarks'])
+    images = get_all_images(dataset_path)  # 모든 이미지 경로를 가져옵니다.
     for image in images:
+        image = image.replace('\\', '/')
         # 영상 번호
-        video_num = image.split('/')[-1][:-4]  # [:-4] -> .jpg remove
+        video_id = image.split('/')[-2]
         # frame 번호
-        frame_num = image.split('/')[-2]
+        frame_num = image.split('/')[-1][:-5]  # [:-5] -> .jpeg remove
         # 이미지 경로 -> ndarray
         image = np.array(Image.open(image))
         # get boundary box and landmarks
@@ -126,14 +121,14 @@ def detect_images_by_cpu(detector, video_name):
         # 1명 이상인 경우 / 1명만 검출하고 싶으면 == 1 로 변경
         if len(people) > 0:
             data = {
-                'frame_num': int(frame_num),
-                'video_num': int(video_num),
+                'frame_num': frame_num,
+                'video_id': video_id,
                 'detect_person_num': len(people),
                 'boxes': [person[0] for person in people],
                 'landmarks': [person[1] for person in people]
             }
             df_detection = df_detection.append(data, ignore_index=True)
-            print(f'{frame_num}번 째 frame : {video_num} 영상 이미지 저장')
+            print(f'{frame_num}번 째 frame : {video_id} 영상 이미지 저장')
     return df_detection
 
 def filter_df(df):
@@ -147,7 +142,7 @@ def filter_df(df):
 
     # 빈 df_filtered 생성
     df_filtered = pd.DataFrame(
-        columns=['frame_num', 'video_num', 'detect_person_num', 'boxes', 'landmarks']
+        columns=['frame_num', 'video_id', 'detect_person_num', 'boxes', 'landmarks']
     ).set_index('frame_num')
 
     for frame_num in frame_index:

Cross-cutting-with-face/util/plot.py

@@ -3,42 +3,40 @@
 from PIL import Image
 import numpy as np
 
-def plot_by_matplotlib(similarity_list, video_name, plot_limit=20):
+def plot_by_matplotlib(path, similarity_list, video_name, plot_limit=20):
     n = 0
     for frame, selected_video, _ in similarity_list:
         fig = plt.figure()
         # plot_limit 까지 plot
         if n > plot_limit: break
         xlabels = ['selected_image1', 'selected_image2']
         i = 1
-        selected_video_num1 = selected_video[0]
-        selected_video_num2 = selected_video[1]
 
         ax = fig.add_subplot(1, 2, i)
-        ax.imshow(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video_num1}.jpg'))
+        ax.imshow(Image.open(path + f'/{selected_video[0]}/{frame}.jpeg'))
         ax.set_xlabel(xlabels[0])
         ax.set_xticks([]), ax.set_yticks([])
 
         ax = fig.add_subplot(1, 2, i + 1)
-        ax.imshow(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video_num2}.jpg'))
+        ax.imshow(Image.open(path + f'/{selected_video[0]}/{frame}.jpeg'))
         ax.set_xlabel(xlabels[1])
         ax.set_xticks([]), ax.set_yticks([])
         n += 1
         fig.show()
 
 
-def plot_by_cv2(similarity_list, video_name, plot_limit=20):
+def plot_by_cv2(path, similarity_list, plot_limit=20):
     n = 0
     i = 1
     for frame, selected_video, _ in similarity_list:
         if n > plot_limit: break
-        img1 = np.array(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video[0]}.jpg').resize((480, 270)))
+        img1 = np.array(Image.open(path + f'/{selected_video[0]}/{frame}.jpeg').resize((480, 270)))
         img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)
-        img2 = np.array(Image.open(f'./dataset/{video_name}/frame/{frame}/{selected_video[1]}.jpg').resize((480, 270)))
+        img2 = np.array(Image.open(path + f'/{selected_video[1]}/{frame}.jpeg').resize((480, 270)))
         img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
 
         combine_imgs = np.concatenate((img1, img2), axis=0)
-        cv2.imshow(f'{i}번째로 제일 유사한 Frame', combine_imgs)
+        cv2.imshow(f'{i} Similarly Frame', combine_imgs)
         cv2.waitKey()
         i += 1
         n += 1