diff --git a/cameras/image_converter.py b/cameras/image_converter.py new file mode 100644 index 0000000..fa8d939 --- /dev/null +++ b/cameras/image_converter.py @@ -0,0 +1,11 @@ +import cv2 +from PIL import Image +from io import BytesIO + +def convert_to_jpg(image): + """TBW.""" + # assumes image is RGB provided + im = Image.fromarray(image) + mem_file = BytesIO() + im.save(mem_file, 'JPEG') + return mem_file.getvalue() diff --git a/controller_listener.py b/controller_listener.py new file mode 100644 index 0000000..2eae4a1 --- /dev/null +++ b/controller_listener.py @@ -0,0 +1,71 @@ +import logging +from controls import main_controller +import websocket +import json +import _thread as thread + +logger = logging.getLogger(__name__) + +def start(websocket_url): + + #websocket.enableTrace(True) + def update_controls(ws, message): + controls = json.loads(message) + + logger.info(controls) + + if 'request_type' in controls: + + if controls['request_type'] == 'calibration': + main_controller.calibration = controls + + + if 'controls' in controls: + + if 'camera_mode' in controls['controls']: + main_controller.camera_mode = controls['controls']['camera_mode'] + + + if 'enable_calibration_feed' in controls: + main_controller.enable_calibration_feed = controls['enable_calibration_feed'] + + if 'enable_camera_feed' in controls: + main_controller.enable_camera_feed = controls['enable_camera_feed'] + + if 'enable_processing_feed' in controls: + main_controller.enable_processing_feed = controls['enable_processing_feed'] + + if 'color_profiles' in controls: + for (camera_mode, profile ) in controls['color_profiles'].items(): + logger.info('updating %s ' % camera_mode) + current_profile = main_controller.color_profiles.get(camera_mode) + current_profile.update(profile) + + if 'color_profile' in controls: + profile = controls['color_profile'] + logger.info('updating %s ' % profile['camera_mode']) + current_profile = main_controller.color_profiles.get(profile['camera_mode']) + current_profile.update(profile) + + #logger.info(main_controller.color_profiles) + + def ws_closed(ws): + logger.info('closed socket') + + def on_error(ws, error): + print(error) + + def on_open(ws): + main_controller.enable_camera = True + + + def start_dashboard_socket(*args): + + dashboard_ws = websocket.WebSocketApp(websocket_url, + on_message = update_controls, + on_close=ws_closed, + on_error = on_error) + dashboard_ws.on_open = on_open + dashboard_ws.run_forever() + + thread.start_new_thread(start_dashboard_socket, ()) diff --git a/controls.py b/controls.py index 45f76b3..1ed330e 100644 --- a/controls.py +++ b/controls.py @@ -10,13 +10,24 @@ class Controls(): def __init__(self): self.enable_camera = True - self.enable_processing = False - self.enable_streaming = True + + self.enable_camera_feed = False + self.enable_calibration_feed = False + self.enable_processing_feed = True + + self.camera_mode = CAMERA_MODE_BALL self.enable_feed = True - self.turn_camera_off = False + self.color_profiles = {} + + + self.calibration = {} + def connect(self): controller_listener.connect(self) + def update(message): + print(message) + main_controller = Controls() diff --git a/main.py b/main.py index f4ecc56..284f70f 100644 --- a/main.py +++ b/main.py @@ -6,11 +6,24 @@ from processing import colors import network as networktables + from cameras import logitech_c270, generic +from cameras import Camera +from cameras import image_converter + from profiles import color_profiles from processing import bay_tracker from processing import port_tracker from processing import ball_tracker2 +from processing import color_calibrate + + +import controls +from controls import main_controller +import controller_listener + +from profiles.color_profile import ColorProfile + import _thread as thread import time @@ -26,9 +39,6 @@ from websocket import create_connection import ujson as json -from cameras import Camera -from web.handlers import main_controller - # initiate the top level logger logging.basicConfig( @@ -55,12 +65,15 @@ def main(): cap = cv2.VideoCapture(config.video_source_number) - # out_pipeline = gst_utils.get_udp_streamer_pipeline2(config.gstreamer_client_ip, - # config.gstreamer_client_port, - # config.gstreamer_bitrate) + enable_gstreamer_pipeline = False + + out = None + if enable_gstreamer_pipeline: + out_pipeline = gst_utils.get_udp_sender(config.gstreamer_client_ip, config.gstreamer_client_port) - # out_pipeline = gst_utils.get_udp_sender(config.gstreamer_client_ip, - # config.gstreamer_client_port) + # out_pipeline = gst_utils.get_udp_streamer_pipeline2(config.gstreamer_client_ip, + # config.gstreamer_client_port, + # config.gstreamer_bitrate) # Set camera properties cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) @@ -70,60 +83,34 @@ def main(): cap.set(cv2.CAP_PROP_EXPOSURE, 0.02) cap.set(cv2.CAP_PROP_CONTRAST, 0.0) + out = cv2.VideoWriter(out_pipeline, 0, + camera.FPS, + (camera.FRAME_WIDTH, camera.FRAME_HEIGHT), + True) + # Set camera properties camera = Camera(cap.get(cv2.CAP_PROP_FRAME_WIDTH), cap.get(cv2.CAP_PROP_FRAME_HEIGHT), cap.get(cv2.CAP_PROP_FPS)) + color_profile_map = {} + for profile in [controls.CAMERA_MODE_RAW, + controls.CAMERA_MODE_BALL, + controls.CAMERA_MODE_HEXAGON, + controls.CAMERA_MODE_LOADING_BAY]: - # print([camera.FRAME_WIDTH]) - # print([camera.FRAME_HEIGHT]) - # print([camera.FPS]) - - # out = cv2.VideoWriter(out_pipeline, 0, - # camera.FPS, - # (camera.FRAME_WIDTH, camera.FRAME_HEIGHT), - # True) - - #TODO: if no camera, exit and msg no camera - # time.sleep(1) - - - #websocket.enableTrace(True) + color_profile_map[profile] = ColorProfile(profile) - def update_controls(ws, message): - logger.info(message) + main_controller.color_profiles = color_profile_map - def ws_closed(ws): - logger.info('closed socket') - - def on_error(ws, error): - print(error) - - # tracking_ws = create_connection("wss://localhost:8080/tracking/ws/") - # - - def on_open(ws): - def run(*args): - for i in range(3): - time.sleep(1) - ws.send("Hello %d" % i) - time.sleep(1) - ws.close() - print("thread terminating...") - thread.start_new_thread(run, ()) + time.sleep(5) tracking_ws = create_connection("ws://localhost:8080/tracking/ws") + camera_ws = create_connection("ws://localhost:8080/camera/ws") + processed_ws = create_connection("ws://localhost:8080/processed/ws") + calibration_ws = create_connection("ws://localhost:8080/calibration/ws") - def start_dashboard_socket(*args): - dashboard_ws = websocket.WebSocketApp("ws://localhost:8080/dashboard/ws", - on_message = update_controls, - on_close= ws_closed, - on_error = on_error) - dashboard.on_open = on_open - dashboard_ws.run_forever() - - thread.start_new_thread(start_dashboard_socket, ()) + controller_listener.start("ws://localhost:8080/dashboard/ws") logger.info('starting main loop ') frame_cnt = 0 @@ -131,14 +118,16 @@ def start_dashboard_socket(*args): frame_cnt += 1 - if True or main_controller.enable_camera: + if main_controller.enable_camera: if not cap.isOpened(): print('opening camera') cap.open(config.video_source_number) - _, frame = cap.read() - #frame = filters.resize(frame, camera.FRAME_WIDTH, camera.FRAME_HEIGHT) + _, raw_frame = cap.read() + + rgb_frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2RGB) + if main_controller.camera_mode == CAMERA_MODE_RAW: frame = frame @@ -151,19 +140,40 @@ def start_dashboard_socket(*args): elif main_controller.camera_mode == CAMERA_MODE_BALL: - frame, tracking_data = ball_tracker2.process(frame, generic, frame_cnt) - dashboard.putStringArray(networktables.keys.vision_target_data, tracking_data) + color_profile=main_controller.color_profiles[CAMERA_MODE_BALL] + + processed_frame, tracking_data = ball_tracker.process(rgb_frame, + camera, + frame_cnt, + color_profile) + tracking_ws.send(json.dumps(dict(targets=tracking_data))) elif main_controller.camera_mode == CAMERA_MODE_HEXAGON: - frame, tracking_data = port_tracker.process(frame, generic) - dashboard.putStringArray(networktables.keys.vision_target_data, tracking_data) - tracking_ws.send(json.dumps(dict(targets=tracking_data))) + processed_frame = port_tracker.process(frame, generic, color_profiles.ReflectiveProfile()) + + + if main_controller.enable_camera_feed: + + jpg=image_converter.convert_to_jpg(rgb_frame) + camera_ws.send_binary(jpg) + + if main_controller.enable_calibration_feed: - # logger.info(main_controller.camera_mode) - if main_controller.enable_streaming: - cv2.putText(frame, + calibration_frame = raw_frame.copy() + + calibration_frame = color_calibrate.process(calibration_frame, + camera_mode = main_controller.calibration.get('camera_mode', 'RAW'), + color_mode = main_controller.calibration.get('color_mode'), + apply_mask = main_controller.calibration.get('apply_mask', False)) + + jpg=image_converter.convert_to_jpg(calibration_frame) + calibration_ws.send_binary(jpg) + + if main_controller.enable_processing_feed: + + cv2.putText(processed_frame, 'Tracking Mode %s' % main_controller.camera_mode, (10,10), cv2.FONT_HERSHEY_DUPLEX, @@ -172,13 +182,17 @@ def start_dashboard_socket(*args): 1, cv2.LINE_AA) + jpg=image_converter.convert_to_jpg(processed_frame) + processed_ws.send_binary(jpg) - # out.write(frame) + # if out is not None: + # out.write(frame) cv2.imshow('frame', frame ) #cv2.waitKey(1) else: + logger.info('waiting for control socket') # IDLE mode if cap.isOpened(): print('closing camera') @@ -190,17 +204,6 @@ def start_dashboard_socket(*args): - - -def single_frame(debug=False): - - img = cv2.imread("frc_cube.jpg") - img = cube_tracker.process(img, - generic) - - cv2.imshow('Objects Detected',img) - cv2.waitKey() - if __name__ == '__main__': p = Process(target=start_web.main) p.start() diff --git a/processing/ball_tracker.py b/processing/ball_tracker.py index 9280d98..dacf3ff 100644 --- a/processing/ball_tracker.py +++ b/processing/ball_tracker.py @@ -9,45 +9,45 @@ import math import cv2 from processing import colors -from processing import filters +from processing import cvfilters from processing import shape_util import time +from profiles import color_profile + import network MIN_AREA = 1000 BALL_RADIUS = 3.5 -debug = True +debug = False + -def process(img, camera, profile): +def process(img, camera, frame_cnt, color_profile): global rgb_window_active, hsv_window_active tracking_data = [] + original_img = img.copy() - FRAME_WIDTH = camera.FRAME_WIDTH - FRAME_HEIGHT = camera.FRAME_HEIGHT - - img = filters.resize(img, camera.FRAME_WIDTH, camera.FRAME_HEIGHT) + img = cv2.GaussianBlur(img, (13, 13), 0) - original_img = img + #image = cv2.resize(image, ((int)(640), (int)(400)), 0, 0, cv2.INTER_CUBIC) + #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - rgb_mask = filters.rgb_threshold(img, profile) - - img = filters.apply_mask(img, rgb_mask) - - img = filters.hsv_threshold(img, profile) + img = cvfilters.hsv_threshold(img, color_profile) + img = cv2.erode(img, None, iterations=2) + img = cv2.dilate(img, None, iterations=2) if debug: cv2.imshow('hsv', img) - contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) contour_list = [] + # algorithm for detecting rectangular object (loading bay) for (index, contour) in enumerate(contours): @@ -58,6 +58,7 @@ def process(img, camera, profile): # limit the number of contours to process # + #print('%s area:%s' %(index, area) ) if area > MIN_AREA: contour_list.append(contour) center_mass_x = x + w / 2 @@ -67,12 +68,15 @@ def process(img, camera, profile): (x, y) = (int(x), int(y)) _, _, w, h = cv2.boundingRect(contour) # tests for if its width is around its height which should be true - if(0.9 <= w / h <= 1.10): + + # print('x: %s y:%s ratio:%s' % (w, h, w/h)) + + if True : distance = shape_util.get_distance(w, 2 * radius, camera.FOCAL_LENGTH) #convert distance to inches distance = 6520 * (w ** -1.02) - print(distance * radius ** 2) - + # print(distance * radius ** 2) + # checks if radius of ball is around actual radius if(BALL_RADIUS * 0.9 <= radius <= BALL_RADIUS * 1.10): cv2.circle(original_img, (x, y), int(radius), @@ -87,10 +91,17 @@ def process(img, camera, profile): area_text = 'area:%s width:%s height:%s' % (area, w, h) angle_text = 'angle:%.2f distance:%s' % (angle, distance) + distance = int(distance) + angle = int(angle) + radius = int(radius) + # set tracking_data - tracking_data.append(dict(index=index, - distance=distance, + tracking_data.append(dict(shape='BALL', + radius=radius, + index=index, + dist=int(distance), angle=angle, + frame=frame_cnt, xpos=center_mass_x, ypos=center_mass_y)) @@ -100,13 +111,13 @@ def process(img, camera, profile): cv2.putText(original_img, angle_text, (x, y - 5), font, .4, colors.WHITE, 1, cv2.LINE_AA) cv2.putText(original_img, radius_text, (x, y - 50), font, .4, colors.WHITE, 1, cv2.LINE_AA) - cv2.drawContours(original_img, contours, index, colors.random(), 2) + cv2.drawContours(original_img, contours, index, colors.GREEN, 2) cv2.circle(original_img, (int(center_mass_x), int(center_mass_y)), 5, colors.GREEN, -1) - cv2.line(original_img, (FRAME_WIDTH // 2, FRAME_HEIGHT), (int(center_mass_x), int(center_mass_y)), colors.GREEN, 2) + #cv2.line(original_img, (FRAME_WIDTH // 2, FRAME_HEIGHT), (int(center_mass_x), int(center_mass_y)), colors.GREEN, 2) + + #if debug: - if debug: - - cv2.drawContours(original_img, contours, index, colors.random(), 2) + #cv2.drawContours(original_img, contours, index, colors.random(), 2) #cv2.rectangle(original_img, (x, y), (x + w, y + h), colors.WHITE, 2) # print the rectangle that did not match @@ -114,7 +125,7 @@ def process(img, camera, profile): # # print 'square: %s,%s' % (w,h) # print w/h, h/w - top_center = (FRAME_WIDTH // 2, FRAME_HEIGHT) - bottom_center = (FRAME_WIDTH // 2, 0) - cv2.line(original_img, top_center, bottom_center, colors.WHITE, 4) + #top_center = (FRAME_WIDTH // 2, FRAME_HEIGHT) + #bottom_center = (FRAME_WIDTH // 2, 0) + #cv2.line(original_img, top_center, bottom_center, colors.WHITE, 4) return original_img, tracking_data diff --git a/processing/color_calibrate.py b/processing/color_calibrate.py index a7c6a8d..7bb5557 100644 --- a/processing/color_calibrate.py +++ b/processing/color_calibrate.py @@ -7,27 +7,64 @@ """ import cv2 -from processing import colors +from controls import main_controller +from . import colors from processing import cvfilters -def process(img, - camera, - profile): +def process(image, + camera_mode='RAW', + color_mode='rgb', + apply_mask=False): - FRAME_WIDTH = camera.FRAME_WIDTH - FRAME_HEIGHT = camera.FRAME_HEIGHT - # - # original_img = img - # - # img = cvfilters.resize(img, camera.FRAME_WIDTH, camera.FRAME_HEIGHT ) + image = cv2.resize(image, ((int)(640), (int)(400)), 0, 0, cv2.INTER_CUBIC) - rgb_mask = cvfilters.rgb_threshold(img, profile) - # - img = cvfilters.apply_mask(img, rgb_mask) - # - # img = cvfilters.hsv_threshold(img, profile) - # - return img + if camera_mode != 'RAW': + color_profile = main_controller.color_profiles.get(camera_mode) + mask = None + + if color_mode == 'rgb': + + mask = cv2.inRange(image, + (color_profile.red.min, color_profile.green.min, color_profile.blue.min), + (color_profile.red.max, color_profile.green.max, color_profile.blue.max)) + + elif color_mode == 'hsv': + hue = color_profile.hsv_hue + sat = color_profile.hsv_sat + val = color_profile.hsv_val + + hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV) + mask = cv2.inRange(hsv, (hue.min, sat.min, val.min), (hue.max, sat.max, val.max)) + + if mask is not None: + + if apply_mask: + image = cvfilters.apply_mask(image, mask) + image = cv2.erode(image, None, iterations=2) + image = cv2.dilate(image, None, iterations=2) + else: + image = mask + + cv2.putText(image, + 'Mode %s' % camera_mode, + (20,20), + cv2.FONT_HERSHEY_DUPLEX, + .4, + colors.BLUE, + 1, + cv2.LINE_AA) + + if color_mode is not None: + cv2.putText(image, + 'COLOR Mode %s' % color_mode, + (20,40), + cv2.FONT_HERSHEY_DUPLEX, + .4, + colors.BLUE, + 1, + cv2.LINE_AA) + + return image diff --git a/start_web.py b/start_web.py index e4b414f..ec51649 100755 --- a/start_web.py +++ b/start_web.py @@ -1,23 +1,25 @@ +#!/usr/bin/env python3 + import network as networktables from web import tornado_server import logging -# initiate the top level logger -# logging.basicConfig( -# level=logging.INFO, -# format="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", -# handlers=[ -# logging.StreamHandler() -# ] -# ) +#initiate the top level logger logger = logging.getLogger('app') def main(): - tornado_server.start() print("starting server") if __name__ == '__main__': - main() \ No newline at end of file + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", + handlers=[ + logging.StreamHandler() + ] + ) + main() diff --git a/web/handlers/CalibrationFeedWS.py b/web/handlers/CalibrationFeedWS.py new file mode 100644 index 0000000..287abf4 --- /dev/null +++ b/web/handlers/CalibrationFeedWS.py @@ -0,0 +1,41 @@ +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging +import json +logger = logging.getLogger(__name__) + +class CalibrationFeedWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("CalibrationFeed websocket opened %s" % self.uid) + self.write_message(json.dumps({ + 'socketid':self.uid + })) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + if isinstance(message, str): + logger.info(message) + if message == 'open feed': + CalibrationFeedWS.watchers.add(self) + if message == 'close feed': + CalibrationFeedWS.watchers.remove(self) + else: + for waiter in CalibrationFeedWS.watchers: + waiter.write_message(message, binary=True) + + def on_close(self): + logger.info("CalibrationFeed websocket closed %s" % self.uid) + CalibrationFeedWS.watchers.remove(self) diff --git a/web/handlers/CameraFeedWS.py b/web/handlers/CameraFeedWS.py new file mode 100644 index 0000000..cd442a5 --- /dev/null +++ b/web/handlers/CameraFeedWS.py @@ -0,0 +1,41 @@ +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging +import json +logger = logging.getLogger(__name__) + +class CameraFeedWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("CameraFeed websocket opened %s" % self.uid) + self.write_message(json.dumps({ + 'socketid':self.uid + })) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + if isinstance(message, str): + logger.info(message) + if message == 'open feed': + CameraFeedWS.watchers.add(self) + if message == 'close feed': + CameraFeedWS.watchers.remove(self) + else: + for waiter in CameraFeedWS.watchers: + waiter.write_message(message, binary=True) + + def on_close(self): + logger.info("CameraFeed websocket closed %s" % self.uid) + CameraFeedWS.watchers.remove(self) diff --git a/web/handlers/ControllerWS.py b/web/handlers/ControllerWS.py new file mode 100644 index 0000000..a632564 --- /dev/null +++ b/web/handlers/ControllerWS.py @@ -0,0 +1,144 @@ +from os.path import abspath, dirname, join + +import uuid +import logging +import time +import json +import json as json_encode + +from tornado.ioloop import IOLoop +from tornado.websocket import WebSocketHandler, WebSocketClosedError + +from controls import main_controller +from profiles.color_profile import ColorProfileEncoder + +logger = logging.getLogger(__name__) + +root_path = abspath(join(dirname(__file__),"../../")) + +class ControllerWS(WebSocketHandler): + """ + A tornado web handler that forwards values between NetworkTables + and a webpage via a websocket + """ + + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("Controller websocket opened") + + ControllerWS.watchers.add(self) + + self.ioloop = IOLoop.current() + self.write_message(json_encode.dumps(dict(socket=self.uid, + enable_camera_feed=main_controller.enable_camera_feed, + enable_processing_feed=main_controller.enable_processing_feed, + enable_calibration_feed= main_controller.enable_calibration_feed, + color_profiles=self.application.settings['color_profiles']), + cls=ColorProfileEncoder)) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + def on_message(self, message): + + logger.info(message) + controls = json.loads(message) + + if 'request_type' in controls: + + if controls['request_type'] == 'calibration': + main_controller.calibration = controls + + + if 'controls' in controls: + + if 'camera_mode' in controls['controls']: + main_controller.camera_mode = controls['controls']['camera_mode'] + + + if 'enable_calibration_feed' in controls: + main_controller.enable_calibration_feed = controls['enable_calibration_feed'] + + if 'enable_camera_feed' in controls: + main_controller.enable_camera_feed = controls['enable_camera_feed'] + + if 'enable_processing_feed' in controls: + main_controller.enable_processing_feed = controls['enable_processing_feed'] + + if 'color_profiles' in controls: + for (camera_mode, profile ) in controls['color_profiles'].items(): + logger.info('updating %s ' % camera_mode) + current_profile = main_controller.color_profiles.get(camera_mode) + current_profile.update(profile) + + if 'color_profile' in controls: + profile = controls['color_profile'] + logger.info('updating %s ' % profile['camera_mode']) + current_profile = main_controller.color_profiles.get(profile['camera_mode']) + current_profile.update(profile) + + + # color_profile = self.application.settings['color_profiles'].get(profile['camera_mode']) + # logger.info('updating color profile for %s' % color_profile.camera_mode) + # + # color_profile.red.min = int(profile['rgb']['r']['min']) + # color_profile.red.max = int(profile['rgb']['r']['max']) + # + # color_profile.green.min = int(profile['rgb']['g']['min']) + # color_profile.green.max = int(profile['rgb']['g']['max']) + # + # color_profile.blue.min = int(profile['rgb']['b']['min']) + # color_profile.blue.max = int(profile['rgb']['b']['max']) + # + # color_profile.hsv_hue.min = int(profile['hsv']['h']['min']) + # color_profile.hsv_hue.max = int(profile['hsv']['h']['max']) + # + # color_profile.hsv_sat.min = int(profile['hsv']['s']['min']) + # color_profile.hsv_sat.max = int(profile['hsv']['s']['max']) + # + # color_profile.hsv_val.min = int(profile['hsv']['v']['min']) + # color_profile.hsv_val.max = int(profile['hsv']['v']['max']) + # + # color_profile.hsl_hue.min = int(profile['hsl']['h']['min']) + # color_profile.hsl_hue.max = int(profile['hsl']['h']['max']) + # + # color_profile.hsl_sat.min = int(profile['hsl']['s']['min']) + # color_profile.hsl_sat.max = int(profile['hsl']['s']['max']) + # + # color_profile.hsl_lum.min = int(profile['hsl']['l']['min']) + # color_profile.hsl_lum.max = int(profile['hsl']['l']['max']) + + if 'reset' in controls: + self.write_message(json_encode.dumps(dict(socket=self.uid, + color_profiles=self.application.settings['color_profiles']), + cls=ColorProfileEncoder)) + + if 'save' in controls: + file_name = 'color_profile_%s.json' % (profile['camera_mode']) + filepath = join(root_path, 'profiles', file_name) + logger.info('writing profile to %s' % filepath) + with open(filepath, mode='w') as f: + json.dump(profile, f, indent=4) + + + logger.info('broadcasting to %s' % len(ControllerWS.watchers)) + for watcher in ControllerWS.watchers: + watcher.write_message(message) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("%s: websocket closed when sending message" % self.uid) + + ## this is used by NTSerial to send updates to web + def send_msg_threadsafe(self, data): + self.ioloop.add_callback(self.send_msg, data) + + def on_close(self): + logger.info("Controller websocket closed %s" % self.uid) + ControllerWS.watchers.remove(self) diff --git a/web/handlers/NonCachingStaticFileHandler.py b/web/handlers/NonCachingStaticFileHandler.py new file mode 100644 index 0000000..c52b504 --- /dev/null +++ b/web/handlers/NonCachingStaticFileHandler.py @@ -0,0 +1,17 @@ +from tornado.web import StaticFileHandler + +class NonCachingStaticFileHandler(StaticFileHandler): + """ + This static file handler disables caching, to allow for easy + development of your Dashboard + """ + + # This is broken in tornado, disable it + def check_etag_header(self): + return False + + def set_extra_headers(self, path): + # Disable caching + self.set_header( + "Cache-Control", "no-store, no-cache, must-revalidate, max-age=0" + ) diff --git a/web/handlers/ObjectTrackingWS.py b/web/handlers/ObjectTrackingWS.py new file mode 100644 index 0000000..9fddca1 --- /dev/null +++ b/web/handlers/ObjectTrackingWS.py @@ -0,0 +1,40 @@ +from os.path import abspath, dirname, join + +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging + +logger = logging.getLogger(__name__) + +class ObjectTrackingWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + logger.info("ObjectTracking websocket opened") + ObjectTrackingWS.watchers.add(self) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + for waiter in ObjectTrackingWS.watchers: + if waiter == self: + continue + waiter.write_message(message) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("websocket closed when sending message") + + def on_close(self): + logger.info("ObjectTracking websocket closed") + ObjectTrackingWS.watchers.remove(self) diff --git a/web/handlers/ProcessedVideoWS.py b/web/handlers/ProcessedVideoWS.py new file mode 100644 index 0000000..eb1652c --- /dev/null +++ b/web/handlers/ProcessedVideoWS.py @@ -0,0 +1,42 @@ +from os.path import abspath, dirname, join + +import uuid +from tornado.websocket import WebSocketHandler, WebSocketClosedError +import logging + +logger = logging.getLogger(__name__) + +class ProcessedVideoWS(WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("ProcessedVideoWS websocket opened %s" % self.uid) + ProcessedVideoWS.watchers.add(self) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + for waiter in ProcessedVideoWS.watchers: + if waiter == self: + continue + waiter.write_message(message, binary=True) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("websocket closed when sending message") + + def on_close(self): + logger.info("ProcessedVideoWS websocket closed %s" % self.uid) + ProcessedVideoWS.watchers.remove(self) diff --git a/web/handlers/__init__.py b/web/handlers/__init__.py new file mode 100644 index 0000000..65241f8 --- /dev/null +++ b/web/handlers/__init__.py @@ -0,0 +1,6 @@ +from .CameraFeedWS import CameraFeedWS +from .ControllerWS import ControllerWS +from .NonCachingStaticFileHandler import NonCachingStaticFileHandler +from .ObjectTrackingWS import ObjectTrackingWS +from .ProcessedVideoWS import ProcessedVideoWS +from .CalibrationFeedWS import CalibrationFeedWS diff --git a/web/image_stream_handlers.py b/web/image_stream_handlers.py index 5b4926b..5f8c163 100644 --- a/web/image_stream_handlers.py +++ b/web/image_stream_handlers.py @@ -34,6 +34,42 @@ def convert_to_jpg(image): im.save(mem_file, 'JPEG') return mem_file.getvalue() +class CameraFeedHandler(tornado.websocket.WebSocketHandler): + """ + """ + watchers = set() + def open(self): + self.uid = str(uuid.uuid4()) + logger.info("CameraFeedHandler websocket opened %s" % self.uid) + CameraFeedHandler.watchers.add(self) + + def check_origin(self, origin): + """ + Allow CORS requests + """ + return True + + """ + broadcast to clients, assumes its target data + """ + def on_message(self, message): + # logger.info('pushing image') + for waiter in CameraFeedHandler.watchers: + if waiter == self: + continue + waiter.write_message(message, binary=True) + + def send_msg(self, msg): + try: + self.write_message(msg, False) + except WebSocketClosedError: + logger.warn("websocket closed when sending message") + + def on_close(self): + logger.info("image websocket closed %s" % self.uid) + CameraFeedHandler.watchers.remove(self) + + class ImageStreamHandler(tornado.websocket.WebSocketHandler): """TBW.""" @@ -122,16 +158,57 @@ def read_image_loop(application): if interval > 0: if len(application.settings['sockets']): _, image = cam.read() - image = cv2.resize(image, ((int)(640), (int)(480)), 0, 0, cv2.INTER_CUBIC) + + image = cv2.resize(image, ((int)(640), (int)(400)), 0, 0, cv2.INTER_CUBIC) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + + + + if ImagePushStreamHandler.camera_mode != 'RAW': + + + + color_profile = application.settings['color_profiles'].get(ImagePushStreamHandler.camera_mode) + + mask = None + + + if ImagePushStreamHandler.color_mode == 'rgb': + mask = cv2.inRange(image, + (color_profile.red.min, color_profile.green.min, color_profile.blue.min), + (color_profile.red.max, color_profile.green.max, color_profile.blue.max)) + + elif ImagePushStreamHandler.color_mode == 'hsv': + hue = color_profile.hsv_hue + sat = color_profile.hsv_sat + val = color_profile.hsv_val + + hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV) + mask = cv2.inRange(hsv, (hue.min, sat.min, val.min), (hue.max, sat.max, val.max)) + # + # elif ImagePushStreamHandler.color_mode == 'hsl': + # hue = color_profile.hsl_lum + # sat = color_profile.hsl_lum + # lum = color_profile.hsl_lum + # hsl = cv2.cvtColor(image, cv2.COLOR_BGR2HSL) + # hsl = cv2.inRange(image, (hue.min, sat.min, lum.min), (hue.max, sat.max, lum.max)) + + if mask is not None: + if ImagePushStreamHandler.apply_mask: + image = cvfilters.apply_mask(image, mask) + image = cv2.erode(image, None, iterations=2) + image = cv2.dilate(image, None, iterations=2) + else: + image = mask cv2.putText(image, - 'Mode %s' % ImagePushStreamHandler.camera_mode, - (20,20), - cv2.FONT_HERSHEY_DUPLEX, - .4, - colors.BLUE, - 1, - cv2.LINE_AA) + 'Mode %s' % ImagePushStreamHandler.camera_mode, + (20,20), + cv2.FONT_HERSHEY_DUPLEX, + .4, + colors.BLUE, + 1, + cv2.LINE_AA) if ImagePushStreamHandler.color_mode is not None: cv2.putText(image, @@ -142,45 +219,6 @@ def read_image_loop(application): colors.BLUE, 1, cv2.LINE_AA) - - color_profile = application.settings['color_profiles'].get(ImagePushStreamHandler.camera_mode) - - mask = None - - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - - if ImagePushStreamHandler.color_mode == 'rgb': - image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) - mask = cv2.inRange(image, - (color_profile.red.min, color_profile.green.min, color_profile.blue.min), - (color_profile.red.max, color_profile.green.max, color_profile.blue.max)) - - elif ImagePushStreamHandler.color_mode == 'hsv': - hue = color_profile.hsv_hue - sat = color_profile.hsv_sat - val = color_profile.hsv_val - - logger.info((hue.min, sat.min, val.min)) - logger.info((hue.max, sat.max, val.max)) - - hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) - mask = cv2.inRange(hsv, (hue.min, sat.min, val.min), (hue.max, sat.max, val.max)) - # - # elif ImagePushStreamHandler.color_mode == 'hsl': - # hue = color_profile.hsl_lum - # sat = color_profile.hsl_lum - # lum = color_profile.hsl_lum - # hsl = cv2.cvtColor(image, cv2.COLOR_BGR2HSL) - # hsl = cv2.inRange(image, (hue.min, sat.min, lum.min), (hue.max, sat.max, lum.max)) - - if mask is not None: - if ImagePushStreamHandler.apply_mask: - image = cvfilters.apply_mask(image, mask) - image = cv2.erode(image, None, iterations=2) - image = cv2.dilate(image, None, iterations=2) - else: - image = mask - jpg = convert_to_jpg(image) for ws in application.settings['sockets']: ws.images.append(jpg) diff --git a/web/old_handlers.py b/web/old_handlers.py new file mode 100644 index 0000000..915f3ad --- /dev/null +++ b/web/old_handlers.py @@ -0,0 +1,20 @@ +from os.path import abspath, dirname, join + +import uuid +import logging +import time +import json +import json as json_encode + +from tornado.ioloop import IOLoop +from tornado.web import StaticFileHandler +from tornado.websocket import WebSocketHandler, WebSocketClosedError + +import network as networktables +from controls import main_controller +from profiles.color_profile import ColorProfileEncoder +from .nt_serial import NTSerial + +root_path = abspath(join(dirname(__file__),"../")) + +logger = logging.getLogger("handlers") diff --git a/web/tornado_server.py b/web/tornado_server.py index 28d17a9..3016f37 100644 --- a/web/tornado_server.py +++ b/web/tornado_server.py @@ -3,22 +3,28 @@ from os.path import abspath, dirname, exists, join import config import logging +import os.path from cameras.camera import USBCam from tornado.web import StaticFileHandler -from web.handlers import NonCachingStaticFileHandler, DashboardWebSocket, ObjectTrackingWebSocket -from web.image_stream_handlers import ImagePushStreamHandler + +from web.handlers import NonCachingStaticFileHandler +from web.handlers import ControllerWS +from web.handlers import ObjectTrackingWS +from web.handlers import CameraFeedWS +from web.handlers import ProcessedVideoWS +from web.handlers import CalibrationFeedWS from profiles.color_profile import ColorProfile import controls -logger = logging.getLogger("tornado") +logger = logging.getLogger(__name__) def start(): # setup tornado application with static handler + networktables support www_dir = abspath(join(dirname(__file__), "www")) - lib_dir = abspath(join(dirname(__file__), "www", "lib")) + #lib_dir = abspath(join(dirname(__file__), "www", "lib")) color_profile_map = {} for profile in [controls.CAMERA_MODE_RAW, @@ -28,22 +34,26 @@ def start(): color_profile_map[profile] = ColorProfile(profile) + app = tornado.web.Application( handlers=[ - ("/dashboard/ws", DashboardWebSocket), - ("/tracking/ws", ObjectTrackingWebSocket), - (r"/camera/ws", ImagePushStreamHandler), - (r"/calibrate/()", NonCachingStaticFileHandler, {"path": join(www_dir, "camera.html")}), + ("/dashboard/ws", ControllerWS), + ("/tracking/ws", ObjectTrackingWS), + (r"/camera/ws", CameraFeedWS), + (r"/processed/ws", ProcessedVideoWS), + (r"/calibration/ws", CalibrationFeedWS ), + (r"/calibrate/()", NonCachingStaticFileHandler, {"path": join(www_dir, "calibrate.html")}), + (r"/processing/()", NonCachingStaticFileHandler, {"path": join(www_dir, "processed.html")}), + (r"/camera/()", NonCachingStaticFileHandler, {"path": join(www_dir, "camera.html")}), (r"/()", NonCachingStaticFileHandler, {"path": join(www_dir, "index.html")}), - (r'/lib/(.*)', StaticFileHandler, {"path": lib_dir}), + #(r'/lib/(.*)', StaticFileHandler, {"path": lib_dir}), (r"/(.*)", NonCachingStaticFileHandler, {"path": www_dir}) ], sockets=[], - color_profiles=color_profile_map, - camera=USBCam() + color_profiles=color_profile_map ) - ImagePushStreamHandler.start(application=app) + # ImagePushStreamHandler.start(application=app) # Start the app logger.info("Listening on http://localhost:%s/", config.tornado_server_port) diff --git a/web/www/app.js b/web/www/app.js index a56e241..bc8e83d 100644 --- a/web/www/app.js +++ b/web/www/app.js @@ -9,38 +9,18 @@ new Vue({ // enable_processing: false, camera_mode: 'BAY' }, - targets: [], - color_profile: null + targets: [] }, mounted: function () { console.log('mounted'); var self = this; + start_camera_stream("/processed/ws", "processed_image"); }, methods: { onTargetUpdate: function(key, value, isNew) { //console.log(value); this.targets = value }, - updateColors: function() { - var self = this; - console.log(self.rgb) - Socket.send({ - 'profile':{ - 'rgb': self.rgb, - 'hsv': self.hsv, - 'hsl': self.hsl - }}) - }, - enableCamera: function () { - var self = this; - if(self.controls.enable_camera == false){ - self.controls.enable_camera = true; - } - else{ - self.controls.enable_camera = false; - } - Socket.send(self.controls) - }, enableRaw: function() { var self = this; self.controls.camera_mode = 'RAW'; diff --git a/web/www/calibrate.html b/web/www/calibrate.html new file mode 100644 index 0000000..d64c540 --- /dev/null +++ b/web/www/calibrate.html @@ -0,0 +1,203 @@ + + + + + + + + + + + Jetson Calibrate + + + + + + + + +
+ + + + + + diff --git a/web/www/calibrate.js b/web/www/calibrate.js new file mode 100644 index 0000000..95418cd --- /dev/null +++ b/web/www/calibrate.js @@ -0,0 +1,134 @@ +"use strict"; + +new Vue({ + el: '#app', + template: '#main-template', + data: { + enable_processing_feed: false, + enable_calibration_feed: false, + color_modes: ['rgb', 'hsv', 'hsl'], + color_profiles: null, + selected_color_mode: null, + selected_profile: null, + controls_ws: null, + apply_mask: false + }, + computed: { + "values": function(){ + let selected_profile = this.selected_profile; + let selected_color_mode = this.selected_color_mode; + + if( selected_profile && selected_color_mode ){ + if( selected_color_mode == 'rgb') { + return [ + { + name: 'R', + range: selected_profile.rgb.r + }, + { + name: 'G', + range: selected_profile.rgb.g + }, + { + name: 'B', + range: selected_profile.rgb.b + } + ] + } + if( selected_color_mode == 'hsl') { + return [ + { + name: 'H', + range: selected_profile.hsl.h + }, + { + name: 'S', + range: selected_profile.hsl.s + }, + { + name: 'V', + range: selected_profile.hsl.l + } + ] + } + if( selected_color_mode == 'hsv') { + return [ + { + name: 'H', + range: selected_profile.hsv.h + }, + { + name: 'S', + range: selected_profile.hsv.s + }, + { + name: 'V', + range: selected_profile.hsv.v + } + ] + } + } + return [] + } + }, + mounted: function () { + var self = this; + self.controls_ws = new_web_socket('/dashboard/ws'); + self.controls_ws.onmessage = function(msg) { + var data = JSON.parse(msg.data) + console.log(data); + if(data.hasOwnProperty('enable_calibration_feed')){ + self.enable_calibration_feed = data.enable_calibration_feed + } + if(data.hasOwnProperty('color_profiles')){ + self.color_profiles = data.color_profiles; + if( self.selected_profile) { + self.selected_profile = self.color_profiles[self.selected_profile.camera_mode] + } + } + //self.$forceUpdate(); + } + start_camera_stream("/calibration/ws", "image"); + }, + methods: { + toggleCalibrationFeed: function() { + this.controls_ws.send(JSON.stringify({request_type: 'contols', + enable_calibration_feed: !this.enable_calibration_feed})) + }, + changeProfile: function(profile) { + this.selected_profile = profile; + this.controls_ws.send(JSON.stringify({request_type: 'calibration', + camera_mode:this.selected_profile.camera_mode, + color_mode: this.selected_color_mode, + apply_mask: this.apply_mask})); + }, + changeApplyMask: function() { + this.controls_ws.send(JSON.stringify({request_type: 'calibration', + camera_mode:this.selected_profile.camera_mode, + color_mode: this.selected_color_mode, + apply_mask: this.apply_mask})); + }, + changeColorMode: function(color_mode) { + this.selected_color_mode = color_mode + this.controls_ws.send(JSON.stringify({ request_type: 'calibration', + camera_mode:this.selected_profile.camera_mode, + color_mode: this.selected_color_mode, + apply_mask: this.apply_mask})); + }, + updateColors: function() { + var self = this; + console.log(self.color_profiles) + self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile})) + }, + saveProfile: function() { + var self = this; + self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile, + 'save': true})) + }, + resetProfile: function() { + var self = this; + self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile, + 'reset': true})) + } + } +}); diff --git a/web/www/camera.html b/web/www/camera.html index 920f050..4841d07 100644 --- a/web/www/camera.html +++ b/web/www/camera.html @@ -8,88 +8,128 @@ - Tornado Image Streamer + Jetson Calibrate -Actual: -
+ -
-
-
-
- -
-
-
+
+
+
+ +
+
+ +
+
+
+
+ +
+
+
-
+ + + + + + +
+ diff --git a/web/www/index.html b/web/www/index.html index 053739f..6135197 100644 --- a/web/www/index.html +++ b/web/www/index.html @@ -145,18 +145,27 @@ - +
+
+ +
+
+ +
+
+ + - + diff --git a/web/www/processed.html b/web/www/processed.html new file mode 100644 index 0000000..d483801 --- /dev/null +++ b/web/www/processed.html @@ -0,0 +1,135 @@ + + + + + + + + + + + Jetson Calibrate + + + + + + + + +
+ + + + + diff --git a/web/www/ws_streamer.js b/web/www/ws_streamer.js index a053573..c04620f 100644 --- a/web/www/ws_streamer.js +++ b/web/www/ws_streamer.js @@ -10,113 +10,11 @@ function new_web_socket(uri_path) { return ws; } -new Vue({ - el: '#app', - template: '#main-template', - data: { - color_modes: ['rgb', 'hsv', 'hsl'], - color_profiles: null, - selected_color_mode: null, - selected_profile: null, - controls_ws: null, - apply_mask: false - }, - computed: { - "values": function(){ - let selected_profile = this.selected_profile; - let selected_color_mode = this.selected_color_mode; - if( selected_profile && selected_color_mode ){ - if( selected_color_mode == 'rgb') { - return [ - { - name: 'R', - range: selected_profile.rgb.r - }, - { - name: 'G', - range: selected_profile.rgb.g - }, - { - name: 'B', - range: selected_profile.rgb.b - } - ] - } - if( selected_color_mode == 'hsl') { - return [ - { - name: 'H', - range: selected_profile.hsl.h - }, - { - name: 'S', - range: selected_profile.hsl.s - }, - { - name: 'V', - range: selected_profile.hsl.l - } - ] - } - if( selected_color_mode == 'hsv') { - return [ - { - name: 'H', - range: selected_profile.hsv.h - }, - { - name: 'S', - range: selected_profile.hsv.s - }, - { - name: 'V', - range: selected_profile.hsv.v - } - ] - } - } - return [] - } - }, - mounted: function () { - var self = this; - self.controls_ws = new_web_socket('/dashboard/ws'); - self.controls_ws.onmessage = function(msg) { - var data = JSON.parse(msg.data) - console.log(data); - if(data.hasOwnProperty('color_profiles')){ - self.color_profiles = data.color_profiles; - } - } - }, - methods: { - changeProfile: function(profile) { - this.selected_profile = profile; - ws_imagestream.send(JSON.stringify({camera_mode:this.selected_profile.camera_mode, - color_mode: this.selected_color_mode, - apply_mask: this.apply_mask})); - }, - changeApplyMask: function() { - ws_imagestream.send(JSON.stringify({camera_mode:this.selected_profile.camera_mode, - color_mode: this.selected_color_mode, - apply_mask: this.apply_mask})); - }, - changeColorMode: function(color_mode) { - this.selected_color_mode = color_mode - ws_imagestream.send(JSON.stringify({camera_mode:this.selected_profile.camera_mode, - color_mode: this.selected_color_mode, - apply_mask: this.apply_mask})); - }, - updateColors: function() { - var self = this; - console.log(self.color_profiles) - self.controls_ws.send(JSON.stringify({'color_profile': self.selected_profile})) - } - } -}); +var start_camera_stream = function( websocket_source, target) { -$(document).ready(function() { + var image = document.getElementById(target) + ws_imagestream = new_web_socket( websocket_source); time_0 = (new Date()).getTime(); counter = 0; @@ -132,9 +30,6 @@ $(document).ready(function() { } - - ws_imagestream = new_web_socket('/camera/ws'); - ws_imagestream.onmessage = function(e) { if (e.data instanceof Blob) { update_fps() @@ -143,17 +38,13 @@ $(document).ready(function() { URL.revokeObjectURL(image.src); } } - if (window.stream_mode == "get") { - setTimeout(function(){ws_imagestream.send('?')}, interval); - } } ws_imagestream.onopen = function() { console.log('connected ws_imagestream...'); - ws_imagestream.send('?'); + ws_imagestream.send("open feed") }; ws_imagestream.onclose = function() { - console.log('closed ws_imagestream'); + console.log('closed feed '); }; - ws_imagestream.send('?'); -}); +};