From 883f56d79e8a74f3e5ae612599fc4a5cd9c65c38 Mon Sep 17 00:00:00 2001 From: Weijian Zeng Date: Tue, 28 Jan 2020 16:45:08 -0500 Subject: [PATCH] refactor websockets --- cameras/image_converter.py | 11 + controller_listener.py | 71 ++++++ controls.py | 18 +- main.py | 140 ++++++------ processing/color_calibrate.py | 71 ++++-- web/handlers.py | 199 ----------------- web/handlers/CalibrationFeedWS.py | 41 ++++ web/handlers/CameraFeedWS.py | 41 ++++ web/handlers/ControllerWS.py | 144 +++++++++++++ web/handlers/NonCachingStaticFileHandler.py | 17 ++ web/handlers/ObjectTrackingWS.py | 40 ++++ web/handlers/ProcessedVideoWS.py | 42 ++++ web/handlers/__init__.py | 6 + web/image_stream_handlers.py | 36 ++++ web/old_handlers.py | 20 ++ web/tornado_server.py | 34 +-- web/www/app.js | 24 +-- web/www/calibrate.html | 203 ++++++++++++++++++ web/www/calibrate.js | 134 ++++++++++++ web/www/camera.html | 225 +++++++++----------- web/www/index.html | 19 +- web/www/processed.html | 135 ++++++++++++ web/www/ws_streamer.js | 136 +----------- 23 files changed, 1219 insertions(+), 588 deletions(-) create mode 100644 cameras/image_converter.py create mode 100644 controller_listener.py delete mode 100644 web/handlers.py create mode 100644 web/handlers/CalibrationFeedWS.py create mode 100644 web/handlers/CameraFeedWS.py create mode 100644 web/handlers/ControllerWS.py create mode 100644 web/handlers/NonCachingStaticFileHandler.py create mode 100644 web/handlers/ObjectTrackingWS.py create mode 100644 web/handlers/ProcessedVideoWS.py create mode 100644 web/handlers/__init__.py create mode 100644 web/old_handlers.py create mode 100644 web/www/calibrate.html create mode 100644 web/www/calibrate.js create mode 100644 web/www/processed.html diff --git a/cameras/image_converter.py b/cameras/image_converter.py new file mode 100644 index 0000000..fa8d939 --- /dev/null +++ b/cameras/image_converter.py @@ -0,0 +1,11 @@ +import cv2 +from PIL import Image +from io import BytesIO + +def convert_to_jpg(image): + """TBW.""" + # assumes image is RGB provided + im = Image.fromarray(image) + mem_file = BytesIO() + im.save(mem_file, 'JPEG') + return mem_file.getvalue() diff --git a/controller_listener.py b/controller_listener.py new file mode 100644 index 0000000..2eae4a1 --- /dev/null +++ b/controller_listener.py @@ -0,0 +1,71 @@ +import logging +from controls import main_controller +import websocket +import json +import _thread as thread + +logger = logging.getLogger(__name__) + +def start(websocket_url): + + #websocket.enableTrace(True) + def update_controls(ws, message): + controls = json.loads(message) + + logger.info(controls) + + if 'request_type' in controls: + + if controls['request_type'] == 'calibration': + main_controller.calibration = controls + + + if 'controls' in controls: + + if 'camera_mode' in controls['controls']: + main_controller.camera_mode = controls['controls']['camera_mode'] + + + if 'enable_calibration_feed' in controls: + main_controller.enable_calibration_feed = controls['enable_calibration_feed'] + + if 'enable_camera_feed' in controls: + main_controller.enable_camera_feed = controls['enable_camera_feed'] + + if 'enable_processing_feed' in controls: + main_controller.enable_processing_feed = controls['enable_processing_feed'] + + if 'color_profiles' in controls: + for (camera_mode, profile ) in controls['color_profiles'].items(): + logger.info('updating %s ' % camera_mode) + current_profile = main_controller.color_profiles.get(camera_mode) + current_profile.update(profile) + + if 'color_profile' in controls: + profile = controls['color_profile'] + logger.info('updating %s ' % profile['camera_mode']) + current_profile = main_controller.color_profiles.get(profile['camera_mode']) + current_profile.update(profile) + + #logger.info(main_controller.color_profiles) + + def ws_closed(ws): + logger.info('closed socket') + + def on_error(ws, error): + print(error) + + def on_open(ws): + main_controller.enable_camera = True + + + def start_dashboard_socket(*args): + + dashboard_ws = websocket.WebSocketApp(websocket_url, + on_message = update_controls, + on_close=ws_closed, + on_error = on_error) + dashboard_ws.on_open = on_open + dashboard_ws.run_forever() + + thread.start_new_thread(start_dashboard_socket, ()) diff --git a/controls.py b/controls.py index 1487ebf..1ed330e 100644 --- a/controls.py +++ b/controls.py @@ -10,14 +10,24 @@ class Controls(): def __init__(self): self.enable_camera = True - self.enable_processing = False - self.enable_streaming = True - self.camera_mode = CAMERA_MODE_CALIBRATE + self.enable_camera_feed = False + self.enable_calibration_feed = False + self.enable_processing_feed = True + + + self.camera_mode = CAMERA_MODE_BALL self.enable_feed = True - self.turn_camera_off = False + self.color_profiles = {} + + + self.calibration = {} + def connect(self): controller_listener.connect(self) + def update(message): + print(message) + main_controller = Controls() diff --git a/main.py b/main.py index 79b056d..d3e8629 100644 --- a/main.py +++ b/main.py @@ -6,12 +6,24 @@ from processing import colors import network as networktables + from cameras import logitech_c270, generic +from cameras import Camera +from cameras import image_converter + from profiles import color_profiles from processing import bay_tracker from processing import port_tracker from processing import ball_tracker +from processing import color_calibrate + + +import controls from controls import main_controller +import controller_listener + +from profiles.color_profile import ColorProfile + import _thread as thread import time @@ -27,7 +39,6 @@ from websocket import create_connection import ujson as json -from cameras import Camera # initiate the top level logger logging.basicConfig( @@ -50,84 +61,61 @@ def main(): cap = cv2.VideoCapture(config.video_source_number) - # out_pipeline = gst_utils.get_udp_streamer_pipeline2(config.gstreamer_client_ip, - # config.gstreamer_client_port, - # config.gstreamer_bitrate) + enable_gstreamer_pipeline = False + + out = None + if enable_gstreamer_pipeline: + out_pipeline = gst_utils.get_udp_sender(config.gstreamer_client_ip, config.gstreamer_client_port) - out_pipeline = gst_utils.get_udp_sender(config.gstreamer_client_ip, - config.gstreamer_client_port) + # out_pipeline = gst_utils.get_udp_streamer_pipeline2(config.gstreamer_client_ip, + # config.gstreamer_client_port, + # config.gstreamer_bitrate) + out = cv2.VideoWriter(out_pipeline, 0, + camera.FPS, + (camera.FRAME_WIDTH, camera.FRAME_HEIGHT), + True) + # Set camera properties camera = Camera(cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT), cap.get(cv2.CAP_PROP_FPS)) - # print([camera.FRAME_WIDTH]) - # print([camera.FRAME_HEIGHT]) - # print([camera.FPS]) - - out = cv2.VideoWriter(out_pipeline, 0, - camera.FPS, - (camera.FRAME_WIDTH, camera.FRAME_HEIGHT), - True) - - #TODO: if no camera, exit and msg no camera - time.sleep(5) - - - #websocket.enableTrace(True) - - def update_controls(ws, message): - logger.info(message) - - def ws_closed(ws): - logger.info('closed socket') + color_profile_map = {} + for profile in [controls.CAMERA_MODE_RAW, + controls.CAMERA_MODE_BALL, + controls.CAMERA_MODE_HEXAGON, + controls.CAMERA_MODE_LOADING_BAY]: - def on_error(ws, error): - print(error) + color_profile_map[profile] = ColorProfile(profile) - # tracking_ws = create_connection("wss://localhost:8080/tracking/ws/") - # + main_controller.color_profiles = color_profile_map - def on_open(ws): - def run(*args): - for i in range(3): - time.sleep(1) - ws.send("Hello %d" % i) - time.sleep(1) - ws.close() - print("thread terminating...") - thread.start_new_thread(run, ()) + time.sleep(5) tracking_ws = create_connection("ws://localhost:8080/tracking/ws") + camera_ws = create_connection("ws://localhost:8080/camera/ws") + processed_ws = create_connection("ws://localhost:8080/processed/ws") + calibration_ws = create_connection("ws://localhost:8080/calibration/ws") - def start_dashboard_socket(*args): - dashboard_ws = websocket.WebSocketApp("ws://localhost:8080/dashboard/ws", - on_message = update_controls, - on_close=ws_closed, - on_error = on_error) - dashboard.on_open = on_open - dashboard_ws.run_forever() - - thread.start_new_thread(start_dashboard_socket, ()) + controller_listener.start("ws://localhost:8080/dashboard/ws") logger.info('starting main loop ') - frame_cnt = 0 while(True): frame_cnt += 1 - if True or main_controller.enable_camera: + if main_controller.enable_camera: if not cap.isOpened(): print('opening camera') cap.open(config.video_source_number) - _, frame = cap.read() + _, raw_frame = cap.read() - #frame = filters.resize(frame, camera.FRAME_WIDTH, camera.FRAME_HEIGHT) + rgb_frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2RGB) if main_controller.camera_mode == CAMERA_MODE_RAW: @@ -141,19 +129,40 @@ def start_dashboard_socket(*args): elif main_controller.camera_mode == CAMERA_MODE_BALL: - frame, tracking_data = ball_tracker.process(frame, + color_profile=main_controller.color_profiles[CAMERA_MODE_BALL] + + processed_frame, tracking_data = ball_tracker.process(rgb_frame, camera, - frame_cnt) + frame_cnt, + color_profile) tracking_ws.send(json.dumps(dict(targets=tracking_data))) elif main_controller.camera_mode == CAMERA_MODE_HEXAGON: - frame = port_tracker.process(frame, generic, color_profiles.ReflectiveProfile()) + processed_frame = port_tracker.process(frame, generic, color_profiles.ReflectiveProfile()) + + + if main_controller.enable_camera_feed: + + jpg=image_converter.convert_to_jpg(rgb_frame) + camera_ws.send_binary(jpg) + + if main_controller.enable_calibration_feed: + calibration_frame = raw_frame.copy() - if main_controller.enable_streaming: - cv2.putText(frame, + calibration_frame = color_calibrate.process(calibration_frame, + camera_mode = main_controller.calibration.get('camera_mode', 'RAW'), + color_mode = main_controller.calibration.get('color_mode'), + apply_mask = main_controller.calibration.get('apply_mask', False)) + + jpg=image_converter.convert_to_jpg(calibration_frame) + calibration_ws.send_binary(jpg) + + if main_controller.enable_processing_feed: + + cv2.putText(processed_frame, 'Tracking Mode %s' % main_controller.camera_mode, (10,10), cv2.FONT_HERSHEY_DUPLEX, @@ -162,13 +171,17 @@ def start_dashboard_socket(*args): 1, cv2.LINE_AA) + jpg=image_converter.convert_to_jpg(processed_frame) + processed_ws.send_binary(jpg) - out.write(frame) + # if out is not None: + # out.write(frame) #cv2.imshow('frame', frame ) #v2.waitKey(1) else: + logger.info('waiting for control socket') # IDLE mode #if cap.isOpened(): #print('closing camera') @@ -180,17 +193,6 @@ def start_dashboard_socket(*args): - - -def single_frame(debug=False): - - img = cv2.imread("frc_cube.jpg") - img = cube_tracker.process(img, - generic) - - cv2.imshow('Objects Detected',img) - cv2.waitKey() - if __name__ == '__main__': p = Process(target=start_web.main) p.start() diff --git a/processing/color_calibrate.py b/processing/color_calibrate.py index a7c6a8d..7bb5557 100644 --- a/processing/color_calibrate.py +++ b/processing/color_calibrate.py @@ -7,27 +7,64 @@ """ import cv2 -from processing import colors +from controls import main_controller +from . import colors from processing import cvfilters -def process(img, - camera, - profile): +def process(image, + camera_mode='RAW', + color_mode='rgb', + apply_mask=False): - FRAME_WIDTH = camera.FRAME_WIDTH - FRAME_HEIGHT = camera.FRAME_HEIGHT - # - # original_img = img - # - # img = cvfilters.resize(img, camera.FRAME_WIDTH, camera.FRAME_HEIGHT ) + image = cv2.resize(image, ((int)(640), (int)(400)), 0, 0, cv2.INTER_CUBIC) - rgb_mask = cvfilters.rgb_threshold(img, profile) - # - img = cvfilters.apply_mask(img, rgb_mask) - # - # img = cvfilters.hsv_threshold(img, profile) - # - return img + if camera_mode != 'RAW': + color_profile = main_controller.color_profiles.get(camera_mode) + mask = None + + if color_mode == 'rgb': + + mask = cv2.inRange(image, + (color_profile.red.min, color_profile.green.min, color_profile.blue.min), + (color_profile.red.max, color_profile.green.max, color_profile.blue.max)) + + elif color_mode == 'hsv': + hue = color_profile.hsv_hue + sat = color_profile.hsv_sat + val = color_profile.hsv_val + + hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV) + mask = cv2.inRange(hsv, (hue.min, sat.min, val.min), (hue.max, sat.max, val.max)) + + if mask is not None: + + if apply_mask: + image = cvfilters.apply_mask(image, mask) + image = cv2.erode(image, None, iterations=2) + image = cv2.dilate(image, None, iterations=2) + else: + image = mask + + cv2.putText(image, + 'Mode %s' % camera_mode, + (20,20), + cv2.FONT_HERSHEY_DUPLEX, + .4, + colors.BLUE, + 1, + cv2.LINE_AA) + + if color_mode is not None: + cv2.putText(image, + 'COLOR Mode %s' % color_mode, + (20,40), + cv2.FONT_HERSHEY_DUPLEX, + .4, + colors.BLUE, + 1, + cv2.LINE_AA) + + return image diff --git a/web/handlers.py b/web/handlers.py deleted file mode 100644 index 05102d7..0000000 --- a/web/handlers.py +++ /dev/null @@ -1,199 +0,0 @@ -from os.path import abspath, dirname, join - -import uuid -import logging -import time -import json -import json as json_encode - -from tornado.ioloop import IOLoop -from tornado.web import StaticFileHandler -from tornado.websocket import WebSocketHandler, WebSocketClosedError - -import network as networktables -from controls import main_controller -from profiles.color_profile import ColorProfileEncoder -from .nt_serial import NTSerial - -root_path = abspath(join(dirname(__file__),"../")) - -logger = logging.getLogger("handlers") - -USE_NT_TABLES = False - -class ObjectTrackingWebSocket(WebSocketHandler): - """ - """ - watchers = set() - def open(self): - logger.info("ObjectTracking websocket opened") - ObjectTrackingWebSocket.watchers.add(self) - - def check_origin(self, origin): - """ - Allow CORS requests - """ - return True - - """ - broadcast to clients, assumes its target data - """ - def on_message(self, message): - for waiter in ObjectTrackingWebSocket.watchers: - if waiter == self: - continue - waiter.write_message(message) - - def send_msg(self, msg): - try: - self.write_message(msg, False) - except WebSocketClosedError: - logger.warn("websocket closed when sending message") - - def on_close(self): - logger.info("ObjectTracking websocket closed") - ObjectTrackingWebSocket.watchers.remove(self) - - - -class DashboardWebSocket(WebSocketHandler): - """ - A tornado web handler that forwards values between NetworkTables - and a webpage via a websocket - """ - - watchers = set() - def open(self): - self.uid = str(uuid.uuid4()) - logger.info("Dashboard websocket opened") - - DashboardWebSocket.watchers.add(self) - - self.ioloop = IOLoop.current() - - dashboard = networktables.get() - - ### add listener network tables updates and send back to socket - if USE_NT_TABLES: - self.ntserial = NTSerial(self.send_msg_threadsafe) - - camera = dict(width=self.application.settings['camera'].WIDTH, height=self.application.settings['camera'].HEIGHT) - logger.info(camera) - self.write_message(json_encode.dumps(dict(socket=self.uid, - camera=camera, - color_profiles=self.application.settings['color_profiles']), - cls=ColorProfileEncoder)) - - def check_origin(self, origin): - """ - Allow CORS requests - """ - return True - - def on_message(self, message): - if USE_NT_TABLES: - dashboard = networktables.get() - - logger.info(message) - inputs = json.loads(message) - - if 'controls' in inputs: - controls = inputs['controls'] - if USE_NT_TABLES: - dashboard.putBoolean(networktables.keys.vision_enable_camera, controls['enable_camera']) - dashboard.putValue(networktables.keys.vision_camera_mode, controls['camera_mode']) - - elif 'color_profile' in inputs: - - profile = inputs['color_profile'] - - if 'reset' in inputs: - file_name = 'color_profile_%s.json' % (profile['camera_mode']) - filepath = join(root_path, 'profiles', file_name) - logger.info('loading profile from %s' % filepath) - try: - with open(filepath, mode='rb') as f: - profile = json.loads(f.read()) - except: - logger.error('missing file %s' % filepath ) - - if USE_NT_TABLES: - dashboard.putValue(networktables.keys.vision_color_profile, json.dumps(profile)) - - color_profile = self.application.settings['color_profiles'].get(profile['camera_mode']) - logger.info('updating color profile for %s' % color_profile.camera_mode) - - color_profile.red.min = int(profile['rgb']['r']['min']) - color_profile.red.max = int(profile['rgb']['r']['max']) - - color_profile.green.min = int(profile['rgb']['g']['min']) - color_profile.green.max = int(profile['rgb']['g']['max']) - - color_profile.blue.min = int(profile['rgb']['b']['min']) - color_profile.blue.max = int(profile['rgb']['b']['max']) - - color_profile.hsv_hue.min = int(profile['hsv']['h']['min']) - color_profile.hsv_hue.max = int(profile['hsv']['h']['max']) - - color_profile.hsv_sat.min = int(profile['hsv']['s']['min']) - color_profile.hsv_sat.max = int(profile['hsv']['s']['max']) - - color_profile.hsv_val.min = int(profile['hsv']['v']['min']) - color_profile.hsv_val.max = int(profile['hsv']['v']['max']) - - color_profile.hsl_hue.min = int(profile['hsl']['h']['min']) - color_profile.hsl_hue.max = int(profile['hsl']['h']['max']) - - color_profile.hsl_sat.min = int(profile['hsl']['s']['min']) - color_profile.hsl_sat.max = int(profile['hsl']['s']['max']) - - color_profile.hsl_lum.min = int(profile['hsl']['l']['min']) - color_profile.hsl_lum.max = int(profile['hsl']['l']['max']) - - if 'reset' in inputs: - self.write_message(json_encode.dumps(dict(socket=self.uid, - color_profiles=self.application.settings['color_profiles']), - cls=ColorProfileEncoder)) - - if 'save' in inputs: - file_name = 'color_profile_%s.json' % (profile['camera_mode']) - filepath = join(root_path, 'profiles', file_name) - logger.info('writing profile to %s' % filepath) - with open(filepath, mode='w') as f: - json.dump(profile, f, indent=4) - - - logger.info('broadcasting to %s' % len(DashboardWebSocket.watchers)) - for watcher in DashboardWebSocket.watchers: - watcher.write_message(message) - - def send_msg(self, msg): - try: - self.write_message(msg, False) - except WebSocketClosedError: - logger.warn("%s: websocket closed when sending message" % self.uid) - - ## this is used by NTSerial to send updates to web - def send_msg_threadsafe(self, data): - self.ioloop.add_callback(self.send_msg, data) - - def on_close(self): - logger.info("Dashboard websocket closed %s" % self.uid) - DashboardWebSocket.watchers.remove(self) - - -class NonCachingStaticFileHandler(StaticFileHandler): - """ - This static file handler disables caching, to allow for easy - development of your Dashboard - """ - - # This is broken in tornado, disable it - def check_etag_header(self): - return False - - def set_extra_headers(self, path): - # Disable caching - self.set_header( - "Cache-Control", "no-store, no-cache, must-revalidate, max-age=0" - ) diff --git a/web/handlers/CalibrationFeedWS.py b/web/handlers/CalibrationFeedWS.py new file mode 100644 index 0000000..287abf4 --- /dev/null +++ b/web/handlers/CalibrationFeedWS.py @@ -0,0 +1,41 @@ +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging +import json +logger = logging.getLogger(__name__) + +class CalibrationFeedWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("CalibrationFeed websocket opened %s" % self.uid) + self.write_message(json.dumps({ + 'socketid':self.uid + })) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + if isinstance(message, str): + logger.info(message) + if message == 'open feed': + CalibrationFeedWS.watchers.add(self) + if message == 'close feed': + CalibrationFeedWS.watchers.remove(self) + else: + for waiter in CalibrationFeedWS.watchers: + waiter.write_message(message, binary=True) + + def on_close(self): + logger.info("CalibrationFeed websocket closed %s" % self.uid) + CalibrationFeedWS.watchers.remove(self) diff --git a/web/handlers/CameraFeedWS.py b/web/handlers/CameraFeedWS.py new file mode 100644 index 0000000..cd442a5 --- /dev/null +++ b/web/handlers/CameraFeedWS.py @@ -0,0 +1,41 @@ +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging +import json +logger = logging.getLogger(__name__) + +class CameraFeedWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("CameraFeed websocket opened %s" % self.uid) + self.write_message(json.dumps({ + 'socketid':self.uid + })) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + if isinstance(message, str): + logger.info(message) + if message == 'open feed': + CameraFeedWS.watchers.add(self) + if message == 'close feed': + CameraFeedWS.watchers.remove(self) + else: + for waiter in CameraFeedWS.watchers: + waiter.write_message(message, binary=True) + + def on_close(self): + logger.info("CameraFeed websocket closed %s" % self.uid) + CameraFeedWS.watchers.remove(self) diff --git a/web/handlers/ControllerWS.py b/web/handlers/ControllerWS.py new file mode 100644 index 0000000..a632564 --- /dev/null +++ b/web/handlers/ControllerWS.py @@ -0,0 +1,144 @@ +from os.path import abspath, dirname, join + +import uuid +import logging +import time +import json +import json as json_encode + +from tornado.ioloop import IOLoop +from tornado.websocket import WebSocketHandler, WebSocketClosedError + +from controls import main_controller +from profiles.color_profile import ColorProfileEncoder + +logger = logging.getLogger(__name__) + +root_path = abspath(join(dirname(__file__),"../../")) + +class ControllerWS(WebSocketHandler): + """ + A tornado web handler that forwards values between NetworkTables + and a webpage via a websocket + """ + + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("Controller websocket opened") + + ControllerWS.watchers.add(self) + + self.ioloop = IOLoop.current() + self.write_message(json_encode.dumps(dict(socket=self.uid, + enable_camera_feed=main_controller.enable_camera_feed, + enable_processing_feed=main_controller.enable_processing_feed, + enable_calibration_feed= main_controller.enable_calibration_feed, + color_profiles=self.application.settings['color_profiles']), + cls=ColorProfileEncoder)) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + def on_message(self, message): + + logger.info(message) + controls = json.loads(message) + + if 'request_type' in controls: + + if controls['request_type'] == 'calibration': + main_controller.calibration = controls + + + if 'controls' in controls: + + if 'camera_mode' in controls['controls']: + main_controller.camera_mode = controls['controls']['camera_mode'] + + + if 'enable_calibration_feed' in controls: + main_controller.enable_calibration_feed = controls['enable_calibration_feed'] + + if 'enable_camera_feed' in controls: + main_controller.enable_camera_feed = controls['enable_camera_feed'] + + if 'enable_processing_feed' in controls: + main_controller.enable_processing_feed = controls['enable_processing_feed'] + + if 'color_profiles' in controls: + for (camera_mode, profile ) in controls['color_profiles'].items(): + logger.info('updating %s ' % camera_mode) + current_profile = main_controller.color_profiles.get(camera_mode) + current_profile.update(profile) + + if 'color_profile' in controls: + profile = controls['color_profile'] + logger.info('updating %s ' % profile['camera_mode']) + current_profile = main_controller.color_profiles.get(profile['camera_mode']) + current_profile.update(profile) + + + # color_profile = self.application.settings['color_profiles'].get(profile['camera_mode']) + # logger.info('updating color profile for %s' % color_profile.camera_mode) + # + # color_profile.red.min = int(profile['rgb']['r']['min']) + # color_profile.red.max = int(profile['rgb']['r']['max']) + # + # color_profile.green.min = int(profile['rgb']['g']['min']) + # color_profile.green.max = int(profile['rgb']['g']['max']) + # + # color_profile.blue.min = int(profile['rgb']['b']['min']) + # color_profile.blue.max = int(profile['rgb']['b']['max']) + # + # color_profile.hsv_hue.min = int(profile['hsv']['h']['min']) + # color_profile.hsv_hue.max = int(profile['hsv']['h']['max']) + # + # color_profile.hsv_sat.min = int(profile['hsv']['s']['min']) + # color_profile.hsv_sat.max = int(profile['hsv']['s']['max']) + # + # color_profile.hsv_val.min = int(profile['hsv']['v']['min']) + # color_profile.hsv_val.max = int(profile['hsv']['v']['max']) + # + # color_profile.hsl_hue.min = int(profile['hsl']['h']['min']) + # color_profile.hsl_hue.max = int(profile['hsl']['h']['max']) + # + # color_profile.hsl_sat.min = int(profile['hsl']['s']['min']) + # color_profile.hsl_sat.max = int(profile['hsl']['s']['max']) + # + # color_profile.hsl_lum.min = int(profile['hsl']['l']['min']) + # color_profile.hsl_lum.max = int(profile['hsl']['l']['max']) + + if 'reset' in controls: + self.write_message(json_encode.dumps(dict(socket=self.uid, + color_profiles=self.application.settings['color_profiles']), + cls=ColorProfileEncoder)) + + if 'save' in controls: + file_name = 'color_profile_%s.json' % (profile['camera_mode']) + filepath = join(root_path, 'profiles', file_name) + logger.info('writing profile to %s' % filepath) + with open(filepath, mode='w') as f: + json.dump(profile, f, indent=4) + + + logger.info('broadcasting to %s' % len(ControllerWS.watchers)) + for watcher in ControllerWS.watchers: + watcher.write_message(message) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("%s: websocket closed when sending message" % self.uid) + + ## this is used by NTSerial to send updates to web + def send_msg_threadsafe(self, data): + self.ioloop.add_callback(self.send_msg, data) + + def on_close(self): + logger.info("Controller websocket closed %s" % self.uid) + ControllerWS.watchers.remove(self) diff --git a/web/handlers/NonCachingStaticFileHandler.py b/web/handlers/NonCachingStaticFileHandler.py new file mode 100644 index 0000000..c52b504 --- /dev/null +++ b/web/handlers/NonCachingStaticFileHandler.py @@ -0,0 +1,17 @@ +from tornado.web import StaticFileHandler + +class NonCachingStaticFileHandler(StaticFileHandler): + """ + This static file handler disables caching, to allow for easy + development of your Dashboard + """ + + # This is broken in tornado, disable it + def check_etag_header(self): + return False + + def set_extra_headers(self, path): + # Disable caching + self.set_header( + "Cache-Control", "no-store, no-cache, must-revalidate, max-age=0" + ) diff --git a/web/handlers/ObjectTrackingWS.py b/web/handlers/ObjectTrackingWS.py new file mode 100644 index 0000000..9fddca1 --- /dev/null +++ b/web/handlers/ObjectTrackingWS.py @@ -0,0 +1,40 @@ +from os.path import abspath, dirname, join + +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging + +logger = logging.getLogger(__name__) + +class ObjectTrackingWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + logger.info("ObjectTracking websocket opened") + ObjectTrackingWS.watchers.add(self) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + for waiter in ObjectTrackingWS.watchers: + if waiter == self: + continue + waiter.write_message(message) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("websocket closed when sending message") + + def on_close(self): + logger.info("ObjectTracking websocket closed") + ObjectTrackingWS.watchers.remove(self) diff --git a/web/handlers/ProcessedVideoWS.py b/web/handlers/ProcessedVideoWS.py new file mode 100644 index 0000000..eb1652c --- /dev/null +++ b/web/handlers/ProcessedVideoWS.py @@ -0,0 +1,42 @@ +from os.path import abspath, dirname, join + +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging + +logger = logging.getLogger(__name__) + +class ProcessedVideoWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("ProcessedVideoWS websocket opened %s" % self.uid) + ProcessedVideoWS.watchers.add(self) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + for waiter in ProcessedVideoWS.watchers: + if waiter == self: + continue + waiter.write_message(message, binary=True) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("websocket closed when sending message") + + def on_close(self): + logger.info("ProcessedVideoWS websocket closed %s" % self.uid) + ProcessedVideoWS.watchers.remove(self) diff --git a/web/handlers/__init__.py b/web/handlers/__init__.py new file mode 100644 index 0000000..65241f8 --- /dev/null +++ b/web/handlers/__init__.py @@ -0,0 +1,6 @@ +from .CameraFeedWS import CameraFeedWS +from .ControllerWS import ControllerWS +from .NonCachingStaticFileHandler import NonCachingStaticFileHandler +from .ObjectTrackingWS import ObjectTrackingWS +from .ProcessedVideoWS import ProcessedVideoWS +from .CalibrationFeedWS import CalibrationFeedWS diff --git a/web/image_stream_handlers.py b/web/image_stream_handlers.py index 16c1389..1e5f5b7 100644 --- a/web/image_stream_handlers.py +++ b/web/image_stream_handlers.py @@ -34,6 +34,42 @@ def convert_to_jpg(image): im.save(mem_file, 'JPEG') return mem_file.getvalue() +class CameraFeedHandler(tornado.websocket.WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("CameraFeedHandler websocket opened %s" % self.uid) + CameraFeedHandler.watchers.add(self) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + for waiter in CameraFeedHandler.watchers: + if waiter == self: + continue + waiter.write_message(message, binary=True) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("websocket closed when sending message") + + def on_close(self): + logger.info("image websocket closed %s" % self.uid) + CameraFeedHandler.watchers.remove(self) + + class ImageStreamHandler(tornado.websocket.WebSocketHandler): """TBW.""" diff --git a/web/old_handlers.py b/web/old_handlers.py new file mode 100644 index 0000000..915f3ad --- /dev/null +++ b/web/old_handlers.py @@ -0,0 +1,20 @@ +from os.path import abspath, dirname, join + +import uuid +import logging +import time +import json +import json as json_encode + +from tornado.ioloop import IOLoop +from tornado.web import StaticFileHandler +from tornado.websocket import WebSocketHandler, WebSocketClosedError + +import network as networktables +from controls import main_controller +from profiles.color_profile import ColorProfileEncoder +from .nt_serial import NTSerial + +root_path = abspath(join(dirname(__file__),"../")) + +logger = logging.getLogger("handlers") diff --git a/web/tornado_server.py b/web/tornado_server.py index 28d17a9..3016f37 100644 --- a/web/tornado_server.py +++ b/web/tornado_server.py @@ -3,22 +3,28 @@ from os.path import abspath, dirname, exists, join import config import logging +import os.path from cameras.camera import USBCam from tornado.web import StaticFileHandler -from web.handlers import NonCachingStaticFileHandler, DashboardWebSocket, ObjectTrackingWebSocket -from web.image_stream_handlers import ImagePushStreamHandler + +from web.handlers import NonCachingStaticFileHandler +from web.handlers import ControllerWS +from web.handlers import ObjectTrackingWS +from web.handlers import CameraFeedWS +from web.handlers import ProcessedVideoWS +from web.handlers import CalibrationFeedWS from profiles.color_profile import ColorProfile import controls -logger = logging.getLogger("tornado") +logger = logging.getLogger(__name__) def start(): # setup tornado application with static handler + networktables support www_dir = abspath(join(dirname(__file__), "www")) - lib_dir = abspath(join(dirname(__file__), "www", "lib")) + #lib_dir = abspath(join(dirname(__file__), "www", "lib")) color_profile_map = {} for profile in [controls.CAMERA_MODE_RAW, @@ -28,22 +34,26 @@ def start(): color_profile_map[profile] = ColorProfile(profile) + app = tornado.web.Application( handlers=[ - ("/dashboard/ws", DashboardWebSocket), - ("/tracking/ws", ObjectTrackingWebSocket), - (r"/camera/ws", ImagePushStreamHandler), - (r"/calibrate/()", NonCachingStaticFileHandler, {"path": join(www_dir, "camera.html")}), + ("/dashboard/ws", ControllerWS), + ("/tracking/ws", ObjectTrackingWS), + (r"/camera/ws", CameraFeedWS), + (r"/processed/ws", ProcessedVideoWS), + (r"/calibration/ws", CalibrationFeedWS ), + (r"/calibrate/()", NonCachingStaticFileHandler, {"path": join(www_dir, "calibrate.html")}), + (r"/processing/()", NonCachingStaticFileHandler, {"path": join(www_dir, "processed.html")}), + (r"/camera/()", NonCachingStaticFileHandler, {"path": join(www_dir, "camera.html")}), (r"/()", NonCachingStaticFileHandler, {"path": join(www_dir, "index.html")}), - (r'/lib/(.*)', StaticFileHandler, {"path": lib_dir}), + #(r'/lib/(.*)', StaticFileHandler, {"path": lib_dir}), (r"/(.*)", NonCachingStaticFileHandler, {"path": www_dir}) ], sockets=[], - color_profiles=color_profile_map, - camera=USBCam() + color_profiles=color_profile_map ) - ImagePushStreamHandler.start(application=app) + # ImagePushStreamHandler.start(application=app) # Start the app logger.info("Listening on http://localhost:%s/", config.tornado_server_port) diff --git a/web/www/app.js b/web/www/app.js index 5ad5549..0cd3bcf 100644 --- a/web/www/app.js +++ b/web/www/app.js @@ -9,38 +9,18 @@ new Vue({ // enable_processing: false, camera_mode: 'R' }, - targets: [], - color_profile: null + targets: [] }, mounted: function () { console.log('mounted'); var self = this; + start_camera_stream("/processed/ws", "processed_image"); }, methods: { onTargetUpdate: function(key, value, isNew) { //console.log(value); this.targets = value }, - updateColors: function() { - var self = this; - console.log(self.rgb) - Socket.send({ - 'profile':{ - 'rgb': self.rgb, - 'hsv': self.hsv, - 'hsl': self.hsl - }}) - }, - enableCamera: function () { - var self = this; - if(self.controls.enable_camera == false){ - self.controls.enable_camera = true; - } - else{ - self.controls.enable_camera = false; - } - Socket.send(self.controls) - }, enableRaw: function() { var self = this; self.controls.camera_mode = 'RAW'; diff --git a/web/www/calibrate.html b/web/www/calibrate.html new file mode 100644 index 0000000..d64c540 --- /dev/null +++ b/web/www/calibrate.html @@ -0,0 +1,203 @@ + + + + + + + + + + + Jetson Calibrate + + + + + + + + +
+ + + + + + diff --git a/web/www/calibrate.js b/web/www/calibrate.js new file mode 100644 index 0000000..95418cd --- /dev/null +++ b/web/www/calibrate.js @@ -0,0 +1,134 @@ +"use strict"; + +new Vue({ + el: '#app', + template: '#main-template', + data: { + enable_processing_feed: false, + enable_calibration_feed: false, + color_modes: ['rgb', 'hsv', 'hsl'], + color_profiles: null, + selected_color_mode: null, + selected_profile: null, + controls_ws: null, + apply_mask: false + }, + computed: { + "values": function(){ + let selected_profile = this.selected_profile; + let selected_color_mode = this.selected_color_mode; + + if( selected_profile && selected_color_mode ){ + if( selected_color_mode == 'rgb') { + return [ + { + name: 'R', + range: selected_profile.rgb.r + }, + { + name: 'G', + range: selected_profile.rgb.g + }, + { + name: 'B', + range: selected_profile.rgb.b + } + ] + } + if( selected_color_mode == 'hsl') { + return [ + { + name: 'H', + range: selected_profile.hsl.h + }, + { + name: 'S', + range: selected_profile.hsl.s + }, + { + name: 'V', + range: selected_profile.hsl.l + } + ] + } + if( selected_color_mode == 'hsv') { + return [ + { + name: 'H', + range: selected_profile.hsv.h + }, + { + name: 'S', + range: selected_profile.hsv.s + }, + { + name: 'V', + range: selected_profile.hsv.v + } + ] + } + } + return [] + } + }, + mounted: function () { + var self = this; + self.controls_ws = new_web_socket('/dashboard/ws'); + self.controls_ws.onmessage = function(msg) { + var data = JSON.parse(msg.data) + console.log(data); + if(data.hasOwnProperty('enable_calibration_feed')){ + self.enable_calibration_feed = data.enable_calibration_feed + } + if(data.hasOwnProperty('color_profiles')){ + self.color_profiles = data.color_profiles; + if( self.selected_profile) { + self.selected_profile = self.color_profiles[self.selected_profile.camera_mode] + } + } + //self.$forceUpdate(); + } + start_camera_stream("/calibration/ws", "image"); + }, + methods: { + toggleCalibrationFeed: function() { + this.controls_ws.send(JSON.stringify({request_type: 'contols', + enable_calibration_feed: !this.enable_calibration_feed})) + }, + changeProfile: function(profile) { + this.selected_profile = profile; + this.controls_ws.send(JSON.stringify({request_type: 'calibration', + camera_mode:this.selected_profile.camera_mode, + color_mode: this.selected_color_mode, + apply_mask: this.apply_mask})); + }, + changeApplyMask: function() { + this.controls_ws.send(JSON.stringify({request_type: 'calibration', + camera_mode:this.selected_profile.camera_mode, + color_mode: this.selected_color_mode, + apply_mask: this.apply_mask})); + }, + changeColorMode: function(color_mode) { + this.selected_color_mode = color_mode + this.controls_ws.send(JSON.stringify({ request_type: 'calibration', + camera_mode:this.selected_profile.camera_mode, + color_mode: this.selected_color_mode, + apply_mask: this.apply_mask})); + }, + updateColors: function() { + var self = this; + console.log(self.color_profiles) + self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile})) + }, + saveProfile: function() { + var self = this; + self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile, + 'save': true})) + }, + resetProfile: function() { + var self = this; + self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile, + 'reset': true})) + } + } +}); diff --git a/web/www/camera.html b/web/www/camera.html index 2f4b1f0..4841d07 100644 --- a/web/www/camera.html +++ b/web/www/camera.html @@ -16,155 +16,120 @@ -
+
+ diff --git a/web/www/index.html b/web/www/index.html index 49c1eec..5036c3c 100644 --- a/web/www/index.html +++ b/web/www/index.html @@ -140,18 +140,27 @@ - +
+
+ +
+
+ +
+
+ + - + diff --git a/web/www/processed.html b/web/www/processed.html new file mode 100644 index 0000000..d483801 --- /dev/null +++ b/web/www/processed.html @@ -0,0 +1,135 @@ + + + + + + + + + + + Jetson Calibrate + + + + + + + + +
+ + + + + diff --git a/web/www/ws_streamer.js b/web/www/ws_streamer.js index a4a1c42..c04620f 100644 --- a/web/www/ws_streamer.js +++ b/web/www/ws_streamer.js @@ -11,7 +11,10 @@ function new_web_socket(uri_path) { } -var start_camera_stream = function() { +var start_camera_stream = function( websocket_source, target) { + + var image = document.getElementById(target) + ws_imagestream = new_web_socket( websocket_source); time_0 = (new Date()).getTime(); counter = 0; @@ -26,7 +29,6 @@ var start_camera_stream = function() { } } - ws_imagestream = new_web_socket('/camera/ws'); ws_imagestream.onmessage = function(e) { if (e.data instanceof Blob) { @@ -36,139 +38,13 @@ var start_camera_stream = function() { URL.revokeObjectURL(image.src); } } - if (window.stream_mode == "get") { - setTimeout(function(){ws_imagestream.send('?')}, interval); - } } ws_imagestream.onopen = function() { console.log('connected ws_imagestream...'); - ws_imagestream.send('?'); + ws_imagestream.send("open feed") }; ws_imagestream.onclose = function() { - console.log('closed ws_imagestream'); + console.log('closed feed '); }; - //ws_imagestream.send('?'); }; - -new Vue({ - el: '#app', - template: '#main-template', - data: { - color_modes: ['rgb', 'hsv', 'hsl'], - color_profiles: null, - selected_color_mode: null, - selected_profile: null, - controls_ws: null, - apply_mask: false - }, - computed: { - "values": function(){ - let selected_profile = this.selected_profile; - let selected_color_mode = this.selected_color_mode; - - if( selected_profile && selected_color_mode ){ - if( selected_color_mode == 'rgb') { - return [ - { - name: 'R', - range: selected_profile.rgb.r - }, - { - name: 'G', - range: selected_profile.rgb.g - }, - { - name: 'B', - range: selected_profile.rgb.b - } - ] - } - if( selected_color_mode == 'hsl') { - return [ - { - name: 'H', - range: selected_profile.hsl.h - }, - { - name: 'S', - range: selected_profile.hsl.s - }, - { - name: 'V', - range: selected_profile.hsl.l - } - ] - } - if( selected_color_mode == 'hsv') { - return [ - { - name: 'H', - range: selected_profile.hsv.h - }, - { - name: 'S', - range: selected_profile.hsv.s - }, - { - name: 'V', - range: selected_profile.hsv.v - } - ] - } - } - return [] - } - }, - mounted: function () { - var self = this; - self.controls_ws = new_web_socket('/dashboard/ws'); - self.controls_ws.onmessage = function(msg) { - var data = JSON.parse(msg.data) - console.log(data); - if(data.hasOwnProperty('color_profiles')){ - self.color_profiles = data.color_profiles; - if( self.selected_profile) { - self.selected_profile = self.color_profiles[self.selected_profile.camera_mode] - } - } - //self.$forceUpdate(); - } - start_camera_stream(); - - }, - methods: { - changeProfile: function(profile) { - this.selected_profile = profile; - ws_imagestream.send(JSON.stringify({camera_mode:this.selected_profile.camera_mode, - color_mode: this.selected_color_mode, - apply_mask: this.apply_mask})); - }, - changeApplyMask: function() { - ws_imagestream.send(JSON.stringify({camera_mode:this.selected_profile.camera_mode, - color_mode: this.selected_color_mode, - apply_mask: this.apply_mask})); - }, - changeColorMode: function(color_mode) { - this.selected_color_mode = color_mode - ws_imagestream.send(JSON.stringify({camera_mode:this.selected_profile.camera_mode, - color_mode: this.selected_color_mode, - apply_mask: this.apply_mask})); - }, - updateColors: function() { - var self = this; - console.log(self.color_profiles) - self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile})) - }, - saveProfile: function() { - var self = this; - self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile, - 'save': true})) - }, - resetProfile: function() { - var self = this; - self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile, - 'reset': true})) - } - } -});