fixed a problem in extract_images

.gitignore

@@ -1,7 +1,6 @@
 /static
 /0_static
 /test
-/test_images
 /__pycache__
 /results.html
 /results_filtered.html
@@ -12,4 +11,5 @@
 /features
 /features_pickle
 /extract_features_old.py
-/feature_distance.py
+/feature_distance.py
+/sorting.py

docker_commnands

@@ -23,7 +23,7 @@ docker network create --driver bridge ndd_subnet
 # --entrypoint /bin/bash
 docker build -f Dockerfile_server_ndd -t jacobloe/server_ndd:0.1 .
 docker run --rm -it -v $(pwd)/static/PATH_TO_FEATURES/:/root/static -v /home/.keras/:/root/.keras -v $(pwd)/features_pickle/:/root/features_pickle/ -p 9000:9000 --network ndd_subnet --name server_ndd jacobloe/server_ndd:0.1 --num_cores 8
-docker run --rm -it -v $(pwd)/static/features_videos/:/root/static -v /home/.keras/:/root/.keras -v $(pwd)/features_pickle/:/root/features_pickle/ -p 9000:9000 --network ndd_subnet --name server_ndd jacobloe/server_ndd:0.2 --num_cores 8
+docker run --rm -it -v $(pwd)/static/features_videos/:/root/static -v /home/.keras/:/root/.keras -v $(pwd)/features_pickle/:/root/features_pickle/ -p 9000:9000 --network ndd_subnet --name server_ndd jacobloe/server_ndd:0.4 --num_cores 8
 
 
 #docker client_flask.py

extract_features.py

@@ -47,15 +47,13 @@ def main(features_path):
                                 os.path.join(features_path, os.path.relpath(p, cp))
                             )[0])[0])[0]
                           for p in list_images_path]
-    # print(list_features_path)
     model = load_model()
     done = 0
     while done < len(list_features_path):  # repeat until all frames in the list have been processed correctly
         if done != 0:
             done = 0
         print('-------------------------------------------------------')
         for i_path, f_path in tqdm(zip(list_images_path, list_features_path), total=len(list_images_path)):
-        # for i_path, f_path in zip(list_images_path, list_features_path):
             feature_name = os.path.split(i_path)[1][:-4]  # get the name of the image, remove the file extension
 
             shot = os.path.split(os.path.split(i_path)[0])[1]  # get the name of the shot for the image
@@ -65,14 +63,6 @@ def main(features_path):
             done_name = '.done' + feature_name
             done_path = os.path.join(fp, done_name)
 
-            # print((not os.path.isfile(path) and not os.path.isfile(done_path)),
-            #       (os.path.isfile(path) and os.path.isfile(done_path)),
-            #       (os.path.isfile(path) and not os.path.isfile(done_path)),
-            #       (not os.path.isfile(path) and os.path.isfile(done_path)))
-            # print(path)
-            # print(done_path)
-            # print('--------')
-
             if not os.path.isfile(path) and not os.path.isfile(done_path):  # if neither the feature nor a .done-file exist, start extracting the feature
                 if not os.path.isdir(fp):  # create the directory to save the features
                     os.makedirs(fp)

extract_images.py

@@ -71,11 +71,12 @@ def main(videos_path, features_path):
                 # create a hidden file to signal that the image-extraction for a movie is done
                 open(os.path.join(frames_dir, '.done'), 'a').close()
                 done += 1  # count the instances of the image-extraction done correctly
-            elif os.path.isfile(os.path.join(frames_dir, '.done')):
+            elif os.path.isfile(os.path.join(frames_dir, '.done')):     # do nothing if a .done-file exists
                 done += 1  # count the instances of the image-extraction done correctly
                 print('image-extraction was already done for {}'.format(os.path.split(v_path)[1]))
+            # if the folder already exists but the .done-file doesn't, delete the folder
             elif os.path.isdir(os.path.join(frames_dir)) and not os.path.isfile(os.path.join(frames_dir, '.done')):
-                shutil.rmtree(f_path)
+                shutil.rmtree(frames_dir)
                 print('image-extraction was not done correctly for {}'.format(os.path.split(v_path)[1]))
 
 #########################################################################################################

server_ndd.py

@@ -4,8 +4,6 @@
 import datetime
 import logging
 
-from scipy.spatial.distance import sqeuclidean
-from sklearn.metrics import pairwise_distances
 from extract_features import extract_features, load_model
 
 from sklearn.metrics.pairwise import euclidean_distances
@@ -32,7 +30,7 @@
 logger.propagate = False    # prevent log messages from appearing twice
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--num_cores", type=int, default=48, help="specify the number cpu cores used for distance calculation, default value is 48")
+parser.add_argument("--num_cores", type=int, default=4, help="specify the number cpu cores used for distance calculation, default value is 4")
 args = parser.parse_args()
 
 inception_model = load_model()
@@ -87,6 +85,7 @@ def get_features(features_path):
 
 archive_features = np.asarray(features_server['feature_list'])
 
+
 def compute_batch(start_idx, batch_size, Y):
 
     global archive_features
@@ -133,7 +132,6 @@ def do_POST(s):
 
         # calculate the distance for all features
         logger.info('calculating the distance for all features')
-        # distances = pairwise_distances(features_server['feature_list'], target_feature, metric=sqeuclidean, n_jobs=args.num_cores)
         logger.debug(archive_features.shape)
 
         f_size = archive_features.shape[0]   # the amount of features
@@ -143,7 +141,6 @@ def do_POST(s):
 
         compute_batch_ = functools.partial(compute_batch, batch_size=batch_size, Y=target_feature)
 
-        # logger.info("computing")
         with mp.Pool(nproc) as pool:
             distances = list(pool.imap(compute_batch_, [i for i in range(0, f_size, batch_size)]))
 
@@ -162,7 +159,6 @@ def do_POST(s):
         hits = 0    # hits is incremented whenever a distance is added to filtered_distances, if hits is higher than num_results
         shot_hits = set()  # saves the name of the shots that have been added to filtered_distances
         index = 0
-        aa = []
 
         logger.info('filter distances')
         while (hits < num_results) and (index < (len(lowest_distances)-1)):  # repeat filtering until num_results results are found or there are no distances in the list anymore
@@ -188,7 +184,6 @@ def do_POST(s):
             for dis, sv, sbf, ft, fp in filtered_distances]
         )
 
-        # logger.debug(concepts)
         s.send_response(200)
         s.send_header("Content-type", "application/json")
         s.end_headers()
@@ -214,4 +209,3 @@ def do_POST(s):
     except KeyboardInterrupt:
         print('setting up server failed failed')
         pass
-    # httpd.server_close()