From e4e4fc27e7ee9145657e58e3d4ef3e5ad2b37811 Mon Sep 17 00:00:00 2001 From: JacobLoe Date: Mon, 21 Oct 2019 15:29:54 +0200 Subject: [PATCH] fixed a problem in extract_images --- .gitignore | 4 ++-- docker_commnands | 2 +- extract_features.py | 10 ---------- extract_images.py | 5 +++-- server_ndd.py | 10 ++-------- 5 files changed, 8 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 84178283..49a455a8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ /static /0_static /test -/test_images /__pycache__ /results.html /results_filtered.html @@ -12,4 +11,5 @@ /features /features_pickle /extract_features_old.py -/feature_distance.py \ No newline at end of file +/feature_distance.py +/sorting.py \ No newline at end of file diff --git a/docker_commnands b/docker_commnands index 60634015..4e385ef7 100755 --- a/docker_commnands +++ b/docker_commnands @@ -23,7 +23,7 @@ docker network create --driver bridge ndd_subnet # --entrypoint /bin/bash docker build -f Dockerfile_server_ndd -t jacobloe/server_ndd:0.1 . docker run --rm -it -v $(pwd)/static/PATH_TO_FEATURES/:/root/static -v /home/.keras/:/root/.keras -v $(pwd)/features_pickle/:/root/features_pickle/ -p 9000:9000 --network ndd_subnet --name server_ndd jacobloe/server_ndd:0.1 --num_cores 8 -docker run --rm -it -v $(pwd)/static/features_videos/:/root/static -v /home/.keras/:/root/.keras -v $(pwd)/features_pickle/:/root/features_pickle/ -p 9000:9000 --network ndd_subnet --name server_ndd jacobloe/server_ndd:0.2 --num_cores 8 +docker run --rm -it -v $(pwd)/static/features_videos/:/root/static -v /home/.keras/:/root/.keras -v $(pwd)/features_pickle/:/root/features_pickle/ -p 9000:9000 --network ndd_subnet --name server_ndd jacobloe/server_ndd:0.4 --num_cores 8 #docker client_flask.py diff --git a/extract_features.py b/extract_features.py index 2fb6f6ff..cc66b564 100755 --- a/extract_features.py +++ b/extract_features.py @@ -47,7 +47,6 @@ def main(features_path): os.path.join(features_path, os.path.relpath(p, cp)) )[0])[0])[0] for p in list_images_path] - # print(list_features_path) model = load_model() done = 0 while done < len(list_features_path): # repeat until all frames in the list have been processed correctly @@ -55,7 +54,6 @@ def main(features_path): done = 0 print('-------------------------------------------------------') for i_path, f_path in tqdm(zip(list_images_path, list_features_path), total=len(list_images_path)): - # for i_path, f_path in zip(list_images_path, list_features_path): feature_name = os.path.split(i_path)[1][:-4] # get the name of the image, remove the file extension shot = os.path.split(os.path.split(i_path)[0])[1] # get the name of the shot for the image @@ -65,14 +63,6 @@ def main(features_path): done_name = '.done' + feature_name done_path = os.path.join(fp, done_name) - # print((not os.path.isfile(path) and not os.path.isfile(done_path)), - # (os.path.isfile(path) and os.path.isfile(done_path)), - # (os.path.isfile(path) and not os.path.isfile(done_path)), - # (not os.path.isfile(path) and os.path.isfile(done_path))) - # print(path) - # print(done_path) - # print('--------') - if not os.path.isfile(path) and not os.path.isfile(done_path): # if neither the feature nor a .done-file exist, start extracting the feature if not os.path.isdir(fp): # create the directory to save the features os.makedirs(fp) diff --git a/extract_images.py b/extract_images.py index e1a5950f..53cfaff4 100755 --- a/extract_images.py +++ b/extract_images.py @@ -71,11 +71,12 @@ def main(videos_path, features_path): # create a hidden file to signal that the image-extraction for a movie is done open(os.path.join(frames_dir, '.done'), 'a').close() done += 1 # count the instances of the image-extraction done correctly - elif os.path.isfile(os.path.join(frames_dir, '.done')): + elif os.path.isfile(os.path.join(frames_dir, '.done')): # do nothing if a .done-file exists done += 1 # count the instances of the image-extraction done correctly print('image-extraction was already done for {}'.format(os.path.split(v_path)[1])) + # if the folder already exists but the .done-file doesn't, delete the folder elif os.path.isdir(os.path.join(frames_dir)) and not os.path.isfile(os.path.join(frames_dir, '.done')): - shutil.rmtree(f_path) + shutil.rmtree(frames_dir) print('image-extraction was not done correctly for {}'.format(os.path.split(v_path)[1])) ######################################################################################################### diff --git a/server_ndd.py b/server_ndd.py index 33131e90..921e5304 100755 --- a/server_ndd.py +++ b/server_ndd.py @@ -4,8 +4,6 @@ import datetime import logging -from scipy.spatial.distance import sqeuclidean -from sklearn.metrics import pairwise_distances from extract_features import extract_features, load_model from sklearn.metrics.pairwise import euclidean_distances @@ -32,7 +30,7 @@ logger.propagate = False # prevent log messages from appearing twice parser = argparse.ArgumentParser() -parser.add_argument("--num_cores", type=int, default=48, help="specify the number cpu cores used for distance calculation, default value is 48") +parser.add_argument("--num_cores", type=int, default=4, help="specify the number cpu cores used for distance calculation, default value is 4") args = parser.parse_args() inception_model = load_model() @@ -87,6 +85,7 @@ def get_features(features_path): archive_features = np.asarray(features_server['feature_list']) + def compute_batch(start_idx, batch_size, Y): global archive_features @@ -133,7 +132,6 @@ def do_POST(s): # calculate the distance for all features logger.info('calculating the distance for all features') - # distances = pairwise_distances(features_server['feature_list'], target_feature, metric=sqeuclidean, n_jobs=args.num_cores) logger.debug(archive_features.shape) f_size = archive_features.shape[0] # the amount of features @@ -143,7 +141,6 @@ def do_POST(s): compute_batch_ = functools.partial(compute_batch, batch_size=batch_size, Y=target_feature) - # logger.info("computing") with mp.Pool(nproc) as pool: distances = list(pool.imap(compute_batch_, [i for i in range(0, f_size, batch_size)])) @@ -162,7 +159,6 @@ def do_POST(s): hits = 0 # hits is incremented whenever a distance is added to filtered_distances, if hits is higher than num_results shot_hits = set() # saves the name of the shots that have been added to filtered_distances index = 0 - aa = [] logger.info('filter distances') while (hits < num_results) and (index < (len(lowest_distances)-1)): # repeat filtering until num_results results are found or there are no distances in the list anymore @@ -188,7 +184,6 @@ def do_POST(s): for dis, sv, sbf, ft, fp in filtered_distances] ) - # logger.debug(concepts) s.send_response(200) s.send_header("Content-type", "application/json") s.end_headers() @@ -214,4 +209,3 @@ def do_POST(s): except KeyboardInterrupt: print('setting up server failed failed') pass - # httpd.server_close()