From 3214b2b11e4ae48938c659a8e93c5bfa5b3eeac2 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sat, 30 Nov 2019 17:32:39 +0000 Subject: [PATCH 01/19] initial working version - tflite export, model not tested --- converter/converter.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/converter/converter.py b/converter/converter.py index b407163..b590b61 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -19,9 +19,10 @@ import tensorflow as tf from tensorflow.python.framework import graph_util -from tensorflow.python.framework.graph_util import convert_variables_to_constants +from tensorflow.compat.v1.graph_util import convert_variables_to_constants from tensorflow.tools.graph_transforms import TransformGraph from tensorflow.python.tools import optimize_for_inference_lib +from tensorflow.compat.v1.lite import TFLiteConverter ################################################################################ @@ -73,13 +74,23 @@ def convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename, # freeze and removes nodes which are not related to feedforward prediction - minimal_graph = convert_variables_to_constants(sess, sess.graph_def, [output_layer_name]) + minimal_graph = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph_def, [output_layer_name]) graph_def = optimize_for_inference_lib.optimize_for_inference(minimal_graph, [input_layer_name], [output_layer_name], tf.float32.as_datatype_enum) graph_def = TransformGraph(graph_def, [input_layer_name], [output_layer_name], ["sort_by_execution_order"]) with tf.gfile.GFile(pbfilename, 'wb') as f: f.write(graph_def.SerializeToString()) + # convert also to tflite format + + img = tf.placeholder(name=input_layer_name, dtype=tf.float32, shape=(1, 224, 224, 3)) + val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.]) + out = tf.identity(val, name=output_layer_name) + + converter = TFLiteConverter.from_session(sess, input_tensors=[img], output_tensors=[out]) + tflite_model = converter.convert() + open("converted_model.tflite", "wb").write(tflite_model) + # write model to logs dir so we can visualize it as: # tensorboard --logdir="logs" From 6c91cd79ab7cadd7c1bc6772530181f4b003d958 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sat, 30 Nov 2019 17:36:21 +0000 Subject: [PATCH 02/19] add comment on source --- converter/converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converter/converter.py b/converter/converter.py index b590b61..6f26c49 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -81,7 +81,7 @@ def convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename, with tf.gfile.GFile(pbfilename, 'wb') as f: f.write(graph_def.SerializeToString()) - # convert also to tflite format + # convert also to tflite format (taken from: https://www.tensorflow.org/lite/guide/inference) img = tf.placeholder(name=input_layer_name, dtype=tf.float32, shape=(1, 224, 224, 3)) val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.]) From 259af7224dcfaeb4911aa684605ebdfbc867c5d9 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sat, 30 Nov 2019 23:53:25 +0000 Subject: [PATCH 03/19] add seperate routine for conevrsion to tflite format --- converter/converter.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/converter/converter.py b/converter/converter.py index 6f26c49..fd5b81a 100644 --- a/converter/converter.py +++ b/converter/converter.py @@ -1,7 +1,7 @@ ################################################################################ # Example : perform conversion from tflearn checkpoint format to TensorFlow -# protocol buffer (.pb) binary format files (for import into other tools) +# protocol buffer (.pb) binary format and also .tflite files (for import into other tools) # Copyright (c) 2019 Toby Breckon, Durham University, UK @@ -37,14 +37,14 @@ ################################################################################ # convert a loaded model definition by loading a checkpoint from a given path # retaining the network between the specified input and output layers -# outputs to pbfilename as a binary .pb protocol buffer format files +# outputs to pbfilename as a binary .pb protocol buffer format file # e.g. for FireNet # model = construct_firenet (224, 224, False) # path = "models/FireNet/firenet"; # path to tflearn checkpoint including filestem # input_layer_name = 'InputData/X' # input layer of network # output_layer_name= 'FullyConnected_2/Softmax' # output layer of network -# pbfilename = "firenet.pb" # output pb format filename +# filename = "firenet.pb" # output filename def convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename, verbose=False): @@ -74,23 +74,13 @@ def convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename, # freeze and removes nodes which are not related to feedforward prediction - minimal_graph = tf.compat.v1.graph_util.convert_variables_to_constants(sess, sess.graph_def, [output_layer_name]) + minimal_graph = convert_variables_to_constants(sess, sess.graph_def, [output_layer_name]) graph_def = optimize_for_inference_lib.optimize_for_inference(minimal_graph, [input_layer_name], [output_layer_name], tf.float32.as_datatype_enum) graph_def = TransformGraph(graph_def, [input_layer_name], [output_layer_name], ["sort_by_execution_order"]) with tf.gfile.GFile(pbfilename, 'wb') as f: f.write(graph_def.SerializeToString()) - # convert also to tflite format (taken from: https://www.tensorflow.org/lite/guide/inference) - - img = tf.placeholder(name=input_layer_name, dtype=tf.float32, shape=(1, 224, 224, 3)) - val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.]) - out = tf.identity(val, name=output_layer_name) - - converter = TFLiteConverter.from_session(sess, input_tensors=[img], output_tensors=[out]) - tflite_model = converter.convert() - open("converted_model.tflite", "wb").write(tflite_model) - # write model to logs dir so we can visualize it as: # tensorboard --logdir="logs" @@ -106,3 +96,21 @@ def convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename, os.remove('checkpoint') ################################################################################ +# convert a binary .pb protocol buffer format model to tflite format + +# e.g. for FireNet +# pbfilename = "firenet.pb" +# input_layer_name = 'InputData/X' # input layer of network +# output_layer_name= 'FullyConnected_2/Softmax' # output layer of network + +def convert_to_tflite(pbfilename, input_layer_name, output_layer_name): + + input_tensor={input_layer_name:[1,224,224,3]} + + print("[INFO] tflite model to " + pbfilename.replace(".pb",".tflite") + " ...") + + converter = tf.lite.TFLiteConverter.from_frozen_graph(pbfilename, [input_layer_name], [output_layer_name], input_tensor) + tflite_model = converter.convert() + open(pbfilename.replace(".pb",".tflite"), "wb").write(tflite_model) + +################################################################################ From 3dc2f7149689d02bceba97dc76bd8581f1f6218f Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sat, 30 Nov 2019 23:55:00 +0000 Subject: [PATCH 04/19] now covers both .pb and .tflite format conversion --- converter/firenet-conversion.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 converter/firenet-conversion.py diff --git a/converter/firenet-conversion.py b/converter/firenet-conversion.py new file mode 100644 index 0000000..feddc36 --- /dev/null +++ b/converter/firenet-conversion.py @@ -0,0 +1,41 @@ +################################################################################ + +# Example : perform conversion of FireNet tflearn model to TensorFlow protocol +# buffer (.pb) binary format and tflife format files (for import into other tools, example OpenCV) + +# Copyright (c) 2019 Toby Breckon, Durham University, UK + +# License : https://github.com/tobybreckon/fire-detection-cnn/blob/master/LICENSE + +# Acknowledgements: some portions - tensorflow tutorial examples and URL below + +################################################################################ + +import glob,os +import sys +sys.path.append('..') + +################################################################################ + +from firenet import construct_firenet +from converter import convert_to_pb +from converter import convert_to_tflite + +################################################################################ + +if __name__ == '__main__': + + # construct and re-export model (so that is excludes the training layers) + + model = construct_firenet (224, 224, False) + print("[INFO] Constructed FireNet ...") + + path = "../models/FireNet/firenet"; # path to tflearn checkpoint including filestem + input_layer_name = 'InputData/X' # input layer of network + output_layer_name= 'FullyConnected_2/Softmax' # output layer of network + filename = "firenet.pb" # output pb format filename + + convert_to_pb(model, path, input_layer_name, output_layer_name, filename) + convert_to_tflite(filename, input_layer_name, output_layer_name) + +################################################################################ From ce26fb54d7fc2fc62b6245fe2e0fe45414ef2012 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sat, 30 Nov 2019 23:58:42 +0000 Subject: [PATCH 05/19] wrap dropout inside training flags also --- firenet.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/firenet.py b/firenet.py index 672187c..ac748f5 100644 --- a/firenet.py +++ b/firenet.py @@ -45,10 +45,12 @@ def construct_firenet (x,y, training=False): network = local_response_normalization(network) network = fully_connected(network, 4096, activation='tanh') - network = dropout(network, 0.5) + if(training): + network = dropout(network, 0.5) network = fully_connected(network, 4096, activation='tanh') - network = dropout(network, 0.5) + if(training): + network = dropout(network, 0.5) network = fully_connected(network, 2, activation='softmax') From 9c1a5abee382182d76684002f0c34ae353c4855d Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sat, 30 Nov 2019 23:59:19 +0000 Subject: [PATCH 06/19] update header --- converter/firenet-to-protobuf.py | 39 -------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 converter/firenet-to-protobuf.py diff --git a/converter/firenet-to-protobuf.py b/converter/firenet-to-protobuf.py deleted file mode 100644 index dc1789b..0000000 --- a/converter/firenet-to-protobuf.py +++ /dev/null @@ -1,39 +0,0 @@ -################################################################################ - -# Example : perform conversion of FireNet tflearn model to TensorFlow protocol -# buffer (.pb) binary format files (for import into other tools, example OpenCV) - -# Copyright (c) 2019 Toby Breckon, Durham University, UK - -# License : https://github.com/tobybreckon/fire-detection-cnn/blob/master/LICENSE - -# Acknowledgements: some portions - tensorflow tutorial examples and URL below - -################################################################################ - -import glob,os -import sys -sys.path.append('..') - -################################################################################ - -from firenet import construct_firenet -from converter import convert_to_pb - -################################################################################ - -if __name__ == '__main__': - - # construct and re-export model (so that is excludes the training layers) - - model = construct_firenet (224, 224, False) - print("[INFO] Constructed FireNet ...") - - path = "../models/FireNet/firenet"; # path to tflearn checkpoint including filestem - input_layer_name = 'InputData/X' # input layer of network - output_layer_name= 'FullyConnected_2/Softmax' # output layer of network - pbfilename = "firenet.pb" # output pb format filename - - convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename) - -################################################################################ From 90b060186d32896b7385e87cfe6813347766f7d2 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sun, 1 Dec 2019 23:50:57 +0000 Subject: [PATCH 07/19] initial version --- converter/firenet-validation.py | 131 ++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 converter/firenet-validation.py diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py new file mode 100644 index 0000000..ec1f89f --- /dev/null +++ b/converter/firenet-validation.py @@ -0,0 +1,131 @@ +################################################################################ + +# Example : perform validation of FireNet models in TFLearn, PB and TFLite formats + +# Copyright (c) 2019 - Toby Breckon, Durham University, UK + +# License : https://github.com/tobybreckon/fire-detection-cnn/blob/master/LICENSE + +################################################################################ + +import cv2 +import os +import sys +import math + +################################################################################ + +import tflearn +from tflearn.layers.core import * +from tflearn.layers.conv import * +from tflearn.layers.normalization import * +from tflearn.layers.estimator import regression + +################################################################################ + +sys.path.append('..') +from firenet import construct_firenet + +################################################################################ + +# tflearn - load model + +print("Load tflearn model from: ../models/FireNet ...", end = '') +model_tflearn = construct_firenet (224, 224, training=False) +model_tflearn.load(os.path.join("../models/FireNet", "firenet"),weights_only=True) +print("OK") + +################################################################################ + +# tf protocol buffer - load model (into opencv) + +print("Load protocolbuf (pb) model from: firenet.pb ...", end = '') +tensorflow_pb_model = cv2.dnn.readNetFromTensorflow('firenet.pb'); +print("OK") + +################################################################################ + +# tflite - load model + +print("Load protocolbuf (pb) model from: firenet.tflite ...", end = '') +tflife_model = tf.lite.Interpreter(model_path="firenet.tflite") +tflife_model.allocate_tensors() +print("OK") + +# Get input and output tensors. +tflife_input_details = tflife_model.get_input_details() +tflife_output_details = tflife_model.get_output_details() +print(tflife_input_details[0]['shape']) + +################################################################################ + +# load video file + +video = cv2.VideoCapture("../models/test.mp4") +print("Load test video from ../models/test.mp4 ...") + +# get video properties + +width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)); +height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + +frame_counter = 0; +keepProcessing = True; + +while (keepProcessing): + + # get video frame from file, handle end of file + + ret, frame = video.read() + if not ret: + print("... end of video file reached"); + keepProcessing = False; + + print("frame: " + str(frame_counter), end = '') + frame_counter = frame_counter + 1 + + # re-size image to network input size and perform prediction + + small_frame = cv2.resize(frame, (224, 224), cv2.INTER_AREA) + + ############################################################################ + + # input to networks is: 224x224x3 colour image with channel ordering as {B,G,R} + # as is the opencv norm, not {R,G,B} and pixel value range 0->255 for each channel + + np.set_printoptions(precision=6) + + # perform predictiion with tflearn model + + output_tflearn = model_tflearn.predict([small_frame]) + print("\t: TFLearn (original): ", end = '') + print(output_tflearn, end = '') + + # perform prediction with protocolbuf model via opencv + + tensorflow_pb_model.setInput(cv2.dnn.blobFromImage(small_frame, size=(224, 224), swapRB=False, crop=False)) + output_tensorflow_pb = tensorflow_pb_model.forward() + + print("\t: Tensorflow .pb (via opencv): ", end = '') + print(output_tensorflow_pb, end = '') + + # perform prediction with tflite model via TensorFlow + + tflife_input_data = np.reshape(np.float32(small_frame), (1, 224, 224, 3)) + tflife_model.set_tensor(tflife_input_details[0]['index'], tflife_input_data) + + tflife_model.invoke() + + output_tflite = tflife_model.get_tensor(tflife_output_details[0]['index']) + print("\t: TFLite (via tensorflow): ", end = '') + print(output_tflite, end = '') + + print() + + try: + np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, 7) + np.testing.assert_almost_equal(output_tflearn, output_tflite, 7) + except AssertionError: + print("Warning: not equal to 7 decimal places"); + +################################################################################ From e986dbd9b4a7098e41739e3bcbfc103348517278 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Sun, 1 Dec 2019 23:56:49 +0000 Subject: [PATCH 08/19] add pass/fail, move comment on RGB/BGR --- converter/firenet-validation.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index ec1f89f..829c765 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -86,13 +86,13 @@ # re-size image to network input size and perform prediction + # input to networks is: 224x224x3 colour image with channel ordering as {B,G,R} + # as is the opencv norm, not {R,G,B} and pixel value range 0->255 for each channel + small_frame = cv2.resize(frame, (224, 224), cv2.INTER_AREA) ############################################################################ - # input to networks is: 224x224x3 colour image with channel ordering as {B,G,R} - # as is the opencv norm, not {R,G,B} and pixel value range 0->255 for each channel - np.set_printoptions(precision=6) # perform predictiion with tflearn model @@ -120,12 +120,11 @@ print("\t: TFLite (via tensorflow): ", end = '') print(output_tflite, end = '') - print() - try: np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, 7) np.testing.assert_almost_equal(output_tflearn, output_tflite, 7) + print(": all equal test - PASS") except AssertionError: - print("Warning: not equal to 7 decimal places"); + print(" all equal test - FAIL") ################################################################################ From 0d390f7eaca2aac8ff02debfe3f4ad3ddc3e21f6 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Mon, 2 Dec 2019 00:01:47 +0000 Subject: [PATCH 09/19] reove printout of shape for tflite input --- converter/firenet-validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index 829c765..d1e190d 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -55,7 +55,6 @@ # Get input and output tensors. tflife_input_details = tflife_model.get_input_details() tflife_output_details = tflife_model.get_output_details() -print(tflife_input_details[0]['shape']) ################################################################################ From 4fb6bdffc35acb309d98c1ec2d6353e4e6241059 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Mon, 2 Dec 2019 00:02:19 +0000 Subject: [PATCH 10/19] update to reflect convertor changes --- README.md | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f2e2aac..bc79fed 100644 --- a/README.md +++ b/README.md @@ -78,12 +78,13 @@ $ python superpixel-inceptionV1OnFire.py models/test.mp4 ## Instructions to use pre-trained models with other frameworks: -To convert the supplied pre-trained models from TFLearn checkpoint format to protocol buffer (.pb) format (used by [OpenCV](http://www.opencv.org) DNN, [TensorFlow](https://www.tensorflow.org/), ...) do: +To convert the supplied pre-trained models from TFLearn checkpoint format to protocol buffer (.pb) format (used by [OpenCV](http://www.opencv.org) DNN, [TensorFlow](https://www.tensorflow.org/), ...) and +also tflite (firenet only at present, used with [TensorFlow](https://www.tensorflow.org/)) do: ``` $ cd converter -$ python firenet-to-protobuf.py +$ python firenet-conversion.py $ python inceptionV1OnFire-to-protobuf.py ``` @@ -95,7 +96,22 @@ $ python test-pb-opencv.py (N.B. for the superpixel network, the test script just checks loading and inference with the ```.pb``` loaded model but does not supply an actual superpixel image - just any test image, hence inference fails to detect the fire correctly for the example only). -**To convert to to other frameworks** (such as PyTorch, MXNet, Keras, ...) from this tensorflow format (protocol buffer, .pb): - please see the extensive deep neural network model conversion tools offered by the [MMdnn](https://github.com/Microsoft/MMdnn) project. +In addition it creates one ```.tflite``` file is created (at present) for FireNet inside the ```converter``` directory (```firenet.tflite```) which can then be tested with the following validation command across all three model formats: + +``` +$ python firenet-validation.py +Load tflearn model from: ../models/FireNet ...OK +Load protocolbuf (pb) model from: firenet.pb ...OK +Load protocolbuf (pb) model from: firenet.tflite ...OK +Load test video from ../models/test.mp4 ... +frame: 0 : TFLearn (original): [[9.999914e-01 8.576833e-06]] : Tensorflow .pb (via opencv): [[9.999914e-01 8.576866e-06]] : TFLite (via tensorflow): [[9.999914e-01 8.576899e-06]]: all equal test - PASS +frame: 1 : TFLearn (original): [[9.999924e-01 7.609045e-06]] : Tensorflow .pb (via opencv): [[9.999924e-01 7.608987e-06]] : TFLite (via tensorflow): [[9.999924e-01 7.608980e-06]]: all equal test - PASS +frame: 2 : TFLearn (original): [[9.999967e-01 3.373572e-06]] : Tensorflow .pb (via opencv): [[9.999967e-01 3.373559e-06]] : TFLite (via tensorflow): [[9.999967e-01 3.373456e-06]]: all equal test - PASS +frame: 3 : TFLearn (original): [[9.999968e-01 3.165212e-06]] : Tensorflow .pb (via opencv): [[9.999968e-01 3.165221e-06]] : TFLite (via tensorflow): [[9.999968e-01 3.165176e-06]]: all equal test - PASS +... +``` + +**To convert to to other frameworks** (such as PyTorch, MXNet, Keras, ...) from these tensorflow formats: - please see the extensive deep neural network model conversion tools offered by the [MMdnn](https://github.com/Microsoft/MMdnn) project. --- From 8d6aba4f0bf83bbfe18b8608ba4fade888ba8262 Mon Sep 17 00:00:00 2001 From: tobybreckon Date: Mon, 2 Dec 2019 13:55:24 +0000 Subject: [PATCH 11/19] change precision to 5, fix end of video bug, global for precision added --- converter/firenet-validation.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index d1e190d..e5c40f8 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -23,6 +23,10 @@ ################################################################################ +VALIDATE_TO_PRECISION_N = 5 + +################################################################################ + sys.path.append('..') from firenet import construct_firenet @@ -40,7 +44,7 @@ # tf protocol buffer - load model (into opencv) print("Load protocolbuf (pb) model from: firenet.pb ...", end = '') -tensorflow_pb_model = cv2.dnn.readNetFromTensorflow('firenet.pb'); +tensorflow_pb_model = cv2.dnn.readNetFromTensorflow('firenet.pb') print("OK") ################################################################################ @@ -65,20 +69,19 @@ # get video properties -width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)); +width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) -frame_counter = 0; -keepProcessing = True; +frame_counter = 0 -while (keepProcessing): +while (True): # get video frame from file, handle end of file ret, frame = video.read() if not ret: - print("... end of video file reached"); - keepProcessing = False; + print("... end of video file reached") + break print("frame: " + str(frame_counter), end = '') frame_counter = frame_counter + 1 @@ -120,8 +123,8 @@ print(output_tflite, end = '') try: - np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, 7) - np.testing.assert_almost_equal(output_tflearn, output_tflite, 7) + np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, VALIDATE_TO_PRECISION_N) + np.testing.assert_almost_equal(output_tflearn, output_tflite, 3) print(": all equal test - PASS") except AssertionError: print(" all equal test - FAIL") From 5f92908d2125ef78e70928518d3df350351207b4 Mon Sep 17 00:00:00 2001 From: tobybreckon Date: Mon, 2 Dec 2019 15:05:39 +0000 Subject: [PATCH 12/19] add failure counter --- converter/firenet-validation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index e5c40f8..8ec8cb5 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -73,6 +73,7 @@ height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) frame_counter = 0 +fail_counter = 0 while (True): @@ -128,5 +129,8 @@ print(": all equal test - PASS") except AssertionError: print(" all equal test - FAIL") + fail_counter = fail_counter +1; ################################################################################ +print("*** FINAL cross-model validation FAILS (for precision of " + VALIDATE_TO_PRECISION_N + ") = " + fail_counter); +################################################################################ From 1ee53e24df9ae7ab1d5050006e7f833a89db3b26 Mon Sep 17 00:00:00 2001 From: tobybreckon Date: Mon, 2 Dec 2019 15:25:35 +0000 Subject: [PATCH 13/19] fix final message --- converter/firenet-validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index 8ec8cb5..9c0415d 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -132,5 +132,5 @@ fail_counter = fail_counter +1; ################################################################################ -print("*** FINAL cross-model validation FAILS (for precision of " + VALIDATE_TO_PRECISION_N + ") = " + fail_counter); +print("*** FINAL cross-model validation FAILS (for precision of " + str(VALIDATE_TO_PRECISION_N) + ") = " + str(fail_counter)); ################################################################################ From 9a125572492063ca08ed35a78f326b22d8b7a10c Mon Sep 17 00:00:00 2001 From: tobybreckon Date: Mon, 2 Dec 2019 15:26:26 +0000 Subject: [PATCH 14/19] remove semis --- converter/firenet-validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index 9c0415d..6e2f2a5 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -129,8 +129,8 @@ print(": all equal test - PASS") except AssertionError: print(" all equal test - FAIL") - fail_counter = fail_counter +1; + fail_counter = fail_counter +1 ################################################################################ -print("*** FINAL cross-model validation FAILS (for precision of " + str(VALIDATE_TO_PRECISION_N) + ") = " + str(fail_counter)); +print("*** FINAL cross-model validation FAILS (for precision of " + str(VALIDATE_TO_PRECISION_N) + ") = " + str(fail_counter)) ################################################################################ From 7c781a21447c0eb94d0c024153d08aff563da3ba Mon Sep 17 00:00:00 2001 From: tobybreckon Date: Mon, 2 Dec 2019 15:47:34 +0000 Subject: [PATCH 15/19] fix output message for tflite model loading --- README.md | 2 +- converter/firenet-validation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bc79fed..47b39b0 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ In addition it creates one ```.tflite``` file is created (at present) for FireNe $ python firenet-validation.py Load tflearn model from: ../models/FireNet ...OK Load protocolbuf (pb) model from: firenet.pb ...OK -Load protocolbuf (pb) model from: firenet.tflite ...OK +Load tflite model from: firenet.tflite ...OK Load test video from ../models/test.mp4 ... frame: 0 : TFLearn (original): [[9.999914e-01 8.576833e-06]] : Tensorflow .pb (via opencv): [[9.999914e-01 8.576866e-06]] : TFLite (via tensorflow): [[9.999914e-01 8.576899e-06]]: all equal test - PASS frame: 1 : TFLearn (original): [[9.999924e-01 7.609045e-06]] : Tensorflow .pb (via opencv): [[9.999924e-01 7.608987e-06]] : TFLite (via tensorflow): [[9.999924e-01 7.608980e-06]]: all equal test - PASS diff --git a/converter/firenet-validation.py b/converter/firenet-validation.py index 6e2f2a5..fd14e03 100644 --- a/converter/firenet-validation.py +++ b/converter/firenet-validation.py @@ -51,7 +51,7 @@ # tflite - load model -print("Load protocolbuf (pb) model from: firenet.tflite ...", end = '') +print("Load tflite model from: firenet.tflite ...", end = '') tflife_model = tf.lite.Interpreter(model_path="firenet.tflite") tflife_model.allocate_tensors() print("OK") From 5638b492d0d4a69ce8386218835a436fd60b628c Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Fri, 6 Dec 2019 22:55:42 +0000 Subject: [PATCH 16/19] add pb + tflite conversion, make dropout in training only --- ...tobuf.py => inceptionV1OnFire-conversion.py} | 17 +++++++++++------ inceptionV1OnFire.py | 3 ++- 2 files changed, 13 insertions(+), 7 deletions(-) rename converter/{inceptionV1OnFire-to-protobuf.py => inceptionV1OnFire-conversion.py} (73%) diff --git a/converter/inceptionV1OnFire-to-protobuf.py b/converter/inceptionV1OnFire-conversion.py similarity index 73% rename from converter/inceptionV1OnFire-to-protobuf.py rename to converter/inceptionV1OnFire-conversion.py index 50efb53..020baaa 100644 --- a/converter/inceptionV1OnFire-to-protobuf.py +++ b/converter/inceptionV1OnFire-conversion.py @@ -1,7 +1,7 @@ ################################################################################ # Example : perform conversion of inceptionV1OnFire tflearn model to TensorFlow protocol -# buffer (.pb) format files (for import into other tools, example OpenCV DNN) +# buffer (.pb) binary format and tflife format files (for import into other tools, example OpenCV) # Copyright (c) 2019 Toby Breckon, Durham University, UK @@ -20,6 +20,7 @@ from inceptionV1OnFire import construct_inceptionv1onfire from converter import convert_to_pb +from converter import convert_to_tflite ################################################################################ @@ -28,21 +29,25 @@ # construct and re-export model (so that is excludes the training layers) model = construct_inceptionv1onfire (224, 224, False) - print("[INFO] Constructed InceptionV1-OnFire ...") + print("[INFO] Constructed InceptionV1-OnFire (binary, full-frame)...") path = "../models/InceptionV1-OnFire/inceptiononv1onfire"; # path to tflearn checkpoint including filestem input_layer_name = 'InputData/X' # input layer of network output_layer_name= 'FullyConnected/Softmax' # output layer of network - pbfilename = "inceptionv1onfire.pb" # output pb format filename + filename = "inceptionv1onfire.pb" # output pb format filename - convert_to_pb(model, path, input_layer_name, output_layer_name, pbfilename) + convert_to_pb(model, path, input_layer_name, output_layer_name, filename) + convert_to_tflite(filename, input_layer_name, output_layer_name) tf.reset_default_graph() + print("[INFO] Constructed InceptionV1-OnFire (superpixel)...") + model_sp = construct_inceptionv1onfire (224, 224, False) path_sp = "../models/SP-InceptionV1-OnFire/sp-inceptiononv1onfire"; # path to tflearn checkpoint including filestem - pbfilename_sp = "sp-inceptionv1onfire.pb" # output pb format filename + filename_sp = "sp-inceptionv1onfire.pb" # output filename - convert_to_pb(model_sp, path_sp, input_layer_name, output_layer_name, pbfilename_sp) + convert_to_pb(model_sp, path_sp, input_layer_name, output_layer_name, filename_sp) + convert_to_tflite(filename_sp, input_layer_name, output_layer_name) ################################################################################ diff --git a/inceptionV1OnFire.py b/inceptionV1OnFire.py index 1e53864..7e74200 100644 --- a/inceptionV1OnFire.py +++ b/inceptionV1OnFire.py @@ -76,7 +76,8 @@ def construct_inceptionv1onfire (x,y, training=False): inception_4a_output = merge([inception_4a_1_1, inception_4a_3_3, inception_4a_5_5, inception_4a_pool_1_1], mode='concat', axis=3, name='inception_4a_output') pool5_7_7 = avg_pool_2d(inception_4a_output, kernel_size=5, strides=1) - pool5_7_7 = dropout(pool5_7_7, 0.4) + if(training): + pool5_7_7 = dropout(pool5_7_7, 0.4) loss = fully_connected(pool5_7_7, 2,activation='softmax') # if training then add training hyperparameters From b728cc75d446d4d39e65664e248d7ad857893a06 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Fri, 6 Dec 2019 23:07:12 +0000 Subject: [PATCH 17/19] initial version --- converter/inceptionV1OnFire-validation.py | 136 ++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 converter/inceptionV1OnFire-validation.py diff --git a/converter/inceptionV1OnFire-validation.py b/converter/inceptionV1OnFire-validation.py new file mode 100644 index 0000000..fbfa44b --- /dev/null +++ b/converter/inceptionV1OnFire-validation.py @@ -0,0 +1,136 @@ +################################################################################ + +# Example : perform validation of InceptionV1-OnFire models in TFLearn, PB and TFLite formats + +# Copyright (c) 2019 - Toby Breckon, Durham University, UK + +# License : https://github.com/tobybreckon/fire-detection-cnn/blob/master/LICENSE + +################################################################################ + +import cv2 +import os +import sys +import math + +################################################################################ + +import tflearn +from tflearn.layers.core import * +from tflearn.layers.conv import * +from tflearn.layers.normalization import * +from tflearn.layers.estimator import regression + +################################################################################ + +VALIDATE_TO_PRECISION_N = 5 + +################################################################################ + +sys.path.append('..') +from inceptionV1OnFire import construct_inceptionv1onfire + +################################################################################ + +# tflearn - load model + +print("Load tflearn model from: ../models/InceptionV1-OnFire ...", end = '') +model_tflearn = construct_inceptionv1onfire (224, 224, training=False) +model_tflearn.load(os.path.join("../models/InceptionV1-OnFire", "inceptiononv1onfire"),weights_only=True) +print("OK") + +################################################################################ + +# tf protocol buffer - load model (into opencv) + +print("Load protocolbuf (pb) model from: inceptiononv1onfire.pb ...", end = '') +tensorflow_pb_model = cv2.dnn.readNetFromTensorflow('inceptionv1onfire.pb') +print("OK") + +################################################################################ + +# tflite - load model + +print("Load tflite model from: inceptiononv1onfire.tflite ...", end = '') +tflife_model = tf.lite.Interpreter(model_path="inceptionv1onfire.tflite") +tflife_model.allocate_tensors() +print("OK") + +# Get input and output tensors. +tflife_input_details = tflife_model.get_input_details() +tflife_output_details = tflife_model.get_output_details() + +################################################################################ + +# load video file + +video = cv2.VideoCapture("../models/test.mp4") +print("Load test video from ../models/test.mp4 ...") + +# get video properties + +width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) +height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + +frame_counter = 0 +fail_counter = 0 + +while (True): + + # get video frame from file, handle end of file + + ret, frame = video.read() + if not ret: + print("... end of video file reached") + break + + print("frame: " + str(frame_counter), end = '') + frame_counter = frame_counter + 1 + + # re-size image to network input size and perform prediction + + # input to networks is: 224x224x3 colour image with channel ordering as {B,G,R} + # as is the opencv norm, not {R,G,B} and pixel value range 0->255 for each channel + + small_frame = cv2.resize(frame, (224, 224), cv2.INTER_AREA) + + ############################################################################ + + np.set_printoptions(precision=6) + + # perform predictiion with tflearn model + + output_tflearn = model_tflearn.predict([small_frame]) + print("\t: TFLearn (original): ", end = '') + print(output_tflearn, end = '') + + # perform prediction with protocolbuf model via opencv + + tensorflow_pb_model.setInput(cv2.dnn.blobFromImage(small_frame, size=(224, 224), swapRB=False, crop=False)) + output_tensorflow_pb = tensorflow_pb_model.forward() + + print("\t: Tensorflow .pb (via opencv): ", end = '') + print(output_tensorflow_pb, end = '') + + # perform prediction with tflite model via TensorFlow + + tflife_input_data = np.reshape(np.float32(small_frame), (1, 224, 224, 3)) + tflife_model.set_tensor(tflife_input_details[0]['index'], tflife_input_data) + + tflife_model.invoke() + + output_tflite = tflife_model.get_tensor(tflife_output_details[0]['index']) + print("\t: TFLite (via tensorflow): ", end = '') + print(output_tflite, end = '') + + try: + np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, VALIDATE_TO_PRECISION_N) + np.testing.assert_almost_equal(output_tflearn, output_tflite, 3) + print(": all equal test - PASS") + except AssertionError: + print(" all equal test - FAIL") + fail_counter = fail_counter +1 + +################################################################################ +print("*** FINAL cross-model validation FAILS (for precision of " + str(VALIDATE_TO_PRECISION_N) + ") = " + str(fail_counter)) +################################################################################ From 6374399c30c2e925ae60aa95faf2cf9ba67b6573 Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Fri, 6 Dec 2019 23:11:45 +0000 Subject: [PATCH 18/19] initial version - not on superpixels, just full frame --- converter/sp-inceptionV1OnFire-validation.py | 137 +++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 converter/sp-inceptionV1OnFire-validation.py diff --git a/converter/sp-inceptionV1OnFire-validation.py b/converter/sp-inceptionV1OnFire-validation.py new file mode 100644 index 0000000..ddcc2a9 --- /dev/null +++ b/converter/sp-inceptionV1OnFire-validation.py @@ -0,0 +1,137 @@ +################################################################################ + +# Example : perform validation of SP-InceptionV1-OnFire models in TFLearn, PB and TFLite formats +# (do this for whole image, not superpixels, just for proof of consistency) + +# Copyright (c) 2019 - Toby Breckon, Durham University, UK + +# License : https://github.com/tobybreckon/fire-detection-cnn/blob/master/LICENSE + +################################################################################ + +import cv2 +import os +import sys +import math + +################################################################################ + +import tflearn +from tflearn.layers.core import * +from tflearn.layers.conv import * +from tflearn.layers.normalization import * +from tflearn.layers.estimator import regression + +################################################################################ + +VALIDATE_TO_PRECISION_N = 5 + +################################################################################ + +sys.path.append('..') +from inceptionV1OnFire import construct_inceptionv1onfire + +################################################################################ + +# tflearn - load model + +print("Load tflearn model from: ../models/InceptionV1-OnFire ...", end = '') +model_tflearn = construct_inceptionv1onfire (224, 224, training=False) +model_tflearn.load(os.path.join("../models/SP-InceptionV1-OnFire", "sp-inceptiononv1onfire"),weights_only=True) +print("OK") + +################################################################################ + +# tf protocol buffer - load model (into opencv) + +print("Load protocolbuf (pb) model from: inceptiononv1onfire.pb ...", end = '') +tensorflow_pb_model = cv2.dnn.readNetFromTensorflow('sp-inceptionv1onfire.pb') +print("OK") + +################################################################################ + +# tflite - load model + +print("Load tflite model from: inceptiononv1onfire.tflite ...", end = '') +tflife_model = tf.lite.Interpreter(model_path="sp-inceptionv1onfire.tflite") +tflife_model.allocate_tensors() +print("OK") + +# Get input and output tensors. +tflife_input_details = tflife_model.get_input_details() +tflife_output_details = tflife_model.get_output_details() + +################################################################################ + +# load video file + +video = cv2.VideoCapture("../models/test.mp4") +print("Load test video from ../models/test.mp4 ...") + +# get video properties + +width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH)) +height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT)) + +frame_counter = 0 +fail_counter = 0 + +while (True): + + # get video frame from file, handle end of file + + ret, frame = video.read() + if not ret: + print("... end of video file reached") + break + + print("frame: " + str(frame_counter), end = '') + frame_counter = frame_counter + 1 + + # re-size image to network input size and perform prediction + + # input to networks is: 224x224x3 colour image with channel ordering as {B,G,R} + # as is the opencv norm, not {R,G,B} and pixel value range 0->255 for each channel + + small_frame = cv2.resize(frame, (224, 224), cv2.INTER_AREA) + + ############################################################################ + + np.set_printoptions(precision=6) + + # perform predictiion with tflearn model + + output_tflearn = model_tflearn.predict([small_frame]) + print("\t: TFLearn (original): ", end = '') + print(output_tflearn, end = '') + + # perform prediction with protocolbuf model via opencv + + tensorflow_pb_model.setInput(cv2.dnn.blobFromImage(small_frame, size=(224, 224), swapRB=False, crop=False)) + output_tensorflow_pb = tensorflow_pb_model.forward() + + print("\t: Tensorflow .pb (via opencv): ", end = '') + print(output_tensorflow_pb, end = '') + + # perform prediction with tflite model via TensorFlow + + tflife_input_data = np.reshape(np.float32(small_frame), (1, 224, 224, 3)) + tflife_model.set_tensor(tflife_input_details[0]['index'], tflife_input_data) + + tflife_model.invoke() + + output_tflite = tflife_model.get_tensor(tflife_output_details[0]['index']) + print("\t: TFLite (via tensorflow): ", end = '') + print(output_tflite, end = '') + + try: + np.testing.assert_almost_equal(output_tflearn, output_tensorflow_pb, VALIDATE_TO_PRECISION_N) + np.testing.assert_almost_equal(output_tflearn, output_tflite, 3) + print(": all equal test - PASS") + except AssertionError: + print(" all equal test - FAIL") + fail_counter = fail_counter +1 + +################################################################################ +print("*** FINAL cross-model validation FAILS (for precision of " + str(VALIDATE_TO_PRECISION_N) + ") = " + str(fail_counter)) +################################################################################ From d9568b993e10be81daab21ae663e2831a40d2a8c Mon Sep 17 00:00:00 2001 From: Toby Breckon Date: Mon, 16 Dec 2019 17:33:34 +0000 Subject: [PATCH 19/19] update README for tflite conversion --- README.md | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 47b39b0..68def77 100644 --- a/README.md +++ b/README.md @@ -79,24 +79,16 @@ $ python superpixel-inceptionV1OnFire.py models/test.mp4 ## Instructions to use pre-trained models with other frameworks: To convert the supplied pre-trained models from TFLearn checkpoint format to protocol buffer (.pb) format (used by [OpenCV](http://www.opencv.org) DNN, [TensorFlow](https://www.tensorflow.org/), ...) and -also tflite (firenet only at present, used with [TensorFlow](https://www.tensorflow.org/)) do: +also tflite (used with [TensorFlow](https://www.tensorflow.org/)) do: ``` $ cd converter $ python firenet-conversion.py -$ python inceptionV1OnFire-to-protobuf.py +$ python inceptionV1OnFire-conversion.py ``` -This creates three ```.pb``` files inside the ```converter``` directory (```firenet.pb``` / ```inceptionv1onfire.pb```/```sp-inceptionv1onfire.pb```) which can then be tested with the [OpenCV](http://www.opencv.org) DNN module (for example, using OpenCV > 4.1.0-pre) from within the same directory: - -``` -$ python test-pb-opencv.py -``` - -(N.B. for the superpixel network, the test script just checks loading and inference with the ```.pb``` loaded model but does not supply an actual superpixel image - just any test image, hence inference fails to detect the fire correctly for the example only). - -In addition it creates one ```.tflite``` file is created (at present) for FireNet inside the ```converter``` directory (```firenet.tflite```) which can then be tested with the following validation command across all three model formats: +This creates a set of six ```.pb``` and ```.tflite``` files inside the ```converter``` directory (```firenet.xxx``` / ```inceptionv1onfire.xxx```/```sp-inceptionv1onfire.xxx``` for ```xxx``` in ```[pb, tflite]```). These files can then be validated with the [OpenCV](http://www.opencv.org) DNN module (OpenCV > 4.1.0-pre) and [TensorFlow](https://www.tensorflow.org/) against the original (tflearn) from within the same directory, as follows: ``` $ python firenet-validation.py @@ -111,6 +103,8 @@ frame: 3 : TFLearn (original): [[9.999968e-01 3.165212e-06]] : Tensor ... ``` +This can be similarly repeated with the ```inceptionV1OnFire-validation.py``` and ```sp-inceptionV1OnFire-validation.py``` validation scripts (N.B. here the superpixel inceptionV1OnFire network is being validated against the whole image frame rather than superpixels just for simply showing consistent output between the original and converted models). + **To convert to to other frameworks** (such as PyTorch, MXNet, Keras, ...) from these tensorflow formats: - please see the extensive deep neural network model conversion tools offered by the [MMdnn](https://github.com/Microsoft/MMdnn) project. ---