From bc2a1432eaece34c2a525c4b703003de7034945e Mon Sep 17 00:00:00 2001 From: nptweij Date: Sat, 11 Jan 2020 10:05:57 -0500 Subject: [PATCH 1/3] add RGB controls to web app --- .gitignore | 1 + controls.py | 11 ++-- network/__init__.py | 4 +- network/color_profile_listener.py | 28 ++++++++++ network/keys.py | 12 ++++- processing/color_calibrate.py | 33 ++++++++++++ processing/cvfilters.py | 89 +++++++++++++++++++++++++++++++ profiles/color_profile.py | 23 ++++++++ web/handlers.py | 28 +++++++--- web/www/app.js | 25 ++++++++- web/www/index.html | 53 ++++++++++++++++-- 11 files changed, 286 insertions(+), 21 deletions(-) create mode 100644 .gitignore create mode 100644 network/color_profile_listener.py create mode 100644 processing/color_calibrate.py create mode 100644 processing/cvfilters.py create mode 100644 profiles/color_profile.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/controls.py b/controls.py index 41571bd..bd6e01c 100644 --- a/controls.py +++ b/controls.py @@ -1,20 +1,21 @@ from network import controller_listener -CAMERA_MODE_RAW = 'R' -CAMERA_MODE_BALL = 'B' -CAMERA_MODE_HEXAGON = 'H' +CAMERA_MODE_RAW = 'RAW' +CAMERA_MODE_CALIBRATE = 'CALIBRATE' +CAMERA_MODE_BALL = 'BALL' +CAMERA_MODE_HEXAGON = 'HEXAGON' + class Controls(): def __init__(self): self.enable_camera = True self.enable_processing = False - self.camera_mode = CAMERA_MODE_RAW + self.camera_mode = CAMERA_MODE_CALIBRATE self.enable_feed = True self.turn_camera_off = False - def connect(self): controller_listener.connect(self) diff --git a/network/__init__.py b/network/__init__.py index 6ab0069..7e3aa0c 100644 --- a/network/__init__.py +++ b/network/__init__.py @@ -1,3 +1,5 @@ from network import keys from network.connection import init -from network.connection import get \ No newline at end of file +from network.connection import get + +from network import color_profile_listener \ No newline at end of file diff --git a/network/color_profile_listener.py b/network/color_profile_listener.py new file mode 100644 index 0000000..4e18374 --- /dev/null +++ b/network/color_profile_listener.py @@ -0,0 +1,28 @@ +import network as networktables +import json + +def connect(profile): + + dashboard = networktables.get() + + def on_profile_change(table, key, value, isNew): + print("ColorProfile Update: '%s' -> %s" % (key, value)) + + p = json.loads(value) + + profile.red.min = int(p['red']['min']) + profile.red.max = int(p['red']['max']) + + profile.blue.min = int(p['blue']['min']) + profile.blue.max = int(p['blue']['max']) + + profile.green.min = int(p['green']['min']) + profile.green.max = int(p['green']['max']) + + + dashboard.addEntryListener(on_profile_change, + immediateNotify=True, + key=networktables.keys.vision_color_profile) + + + diff --git a/network/keys.py b/network/keys.py index 2595d9b..87df70f 100644 --- a/network/keys.py +++ b/network/keys.py @@ -1,4 +1,14 @@ vision_initialized = "vision/initialized" vision_enable_camera = "vision/enable_camera" -vision_camera_mode = "vision/camera_mode" \ No newline at end of file +vision_enable_processing = 'vision/enable_processing' + +# R(aw), B(all), H(exagon) +vision_camera_mode = "vision/camera_mode" + + +# json object of color profile settings +vision_color_profile = "vision/color_profile" + +# json object of filter settings +vision_filters = 'vision/filters' \ No newline at end of file diff --git a/processing/color_calibrate.py b/processing/color_calibrate.py new file mode 100644 index 0000000..a7c6a8d --- /dev/null +++ b/processing/color_calibrate.py @@ -0,0 +1,33 @@ +""" + +Object Tracker based on a color profile + +uses contour lines and rough area calculations + +""" + +import cv2 +from processing import colors +from processing import cvfilters + +def process(img, + camera, + profile): + + FRAME_WIDTH = camera.FRAME_WIDTH + FRAME_HEIGHT = camera.FRAME_HEIGHT + # + # original_img = img + # + # img = cvfilters.resize(img, camera.FRAME_WIDTH, camera.FRAME_HEIGHT ) + + rgb_mask = cvfilters.rgb_threshold(img, profile) + # + img = cvfilters.apply_mask(img, rgb_mask) + # + # img = cvfilters.hsv_threshold(img, profile) + # + + return img + + diff --git a/processing/cvfilters.py b/processing/cvfilters.py new file mode 100644 index 0000000..658c4c7 --- /dev/null +++ b/processing/cvfilters.py @@ -0,0 +1,89 @@ +import numpy as np +import cv2 + + +def resize(input, width, height, interpolation=cv2.INTER_CUBIC): + return cv2.resize(input, ((int)(width), (int)(height)), 0, 0, interpolation) + + +def median_filter(input, blur=5): + return cv2.medianBlur(input,blur) + + +def hsl_threshold(input,profile): + hue = profile.hue + sat = profile.sat + lum = profile.lum + + out = cv2.cvtColor(input, cv2.COLOR_BGR2HLS) + return cv2.inRange(out, (hue.min, lum.min, sat.min), (hue.max, lum.max, sat.max)) + + +def hsv_threshold(input,profile): + hue = profile.hsv_hue + sat = profile.hsv_sat + val = profile.hsv_val + + out = cv2.cvtColor(input, cv2.COLOR_BGR2HSV) + return cv2.inRange(out, (hue.min, sat.min, val.min), (hue.max, sat.max, val.max)) + + +def rgb_threshold(input, profile): + rgb = cv2.cvtColor(input, cv2.COLOR_BGR2RGB) + return cv2.inRange(rgb, + (profile.red.min, profile.green.min, profile.blue.min), + (profile.red.max, profile.green.max, profile.blue.max)) + +def grayscale(input): + """ + convert to grayscale + """ + return cv2.cvtColor(input,cv2.COLOR_RGB2GRAY) + + +def noise_removal(input): + """ + Noise removal with iterative + bilateral filter(removes noise while preserving edges) + """ + return cv2.bilateralFilter(input, 9,75,75) + + +def detect_canny_edges(input, debug=False): + """ + Applying Canny Edge detection + """ + + canny_image = cv2.Canny(input,250,255) + if debug: + cv2.imshow("Image after applying Canny",canny_image) + # # Display Image + return cv2.convertScaleAbs(canny_image) + + +def dilate_edges(img): + # dilation to strengthen the edges + kernel = np.ones((3,3), np.uint8) + # Creating the kernel for dilation + dilated_image = cv2.dilate(img,kernel,iterations=1) + return dilated_image + + +def threshold_OTSU(img): + # Display Image + # Thresholding the image + _,thresh_image = cv2.threshold(img,0,255,cv2.THRESH_OTSU) + + return thresh_image + + + +def apply_mask(input, mask): + """Filter out an area of an image using a binary mask. + Args: + input: A three channel numpy.ndarray. + mask: A black and white numpy.ndarray. + Returns: + A three channel numpy.ndarray. + """ + return cv2.bitwise_and(input, input, mask=mask) \ No newline at end of file diff --git a/profiles/color_profile.py b/profiles/color_profile.py new file mode 100644 index 0000000..0f9bd0f --- /dev/null +++ b/profiles/color_profile.py @@ -0,0 +1,23 @@ + +class Range: + + def __init__(self, min, max): + self.min = min + self.max = max + + +class ColorProfile: + + def __init__(self): + + self.red = Range(0,255) + self.green = Range(0,255) + self.blue = Range(0,255) + + self.hsl_hue = Range(0,255) + self.hsl_sat = Range(0,255) + self.hsl_lum = Range(0,255) + + self.hsv_hue = Range(0,255) + self.hsv_sat = Range(0,255) + self.hsv_val = Range(0,255) diff --git a/web/handlers.py b/web/handlers.py index 21bfe08..b14e832 100644 --- a/web/handlers.py +++ b/web/handlers.py @@ -32,20 +32,32 @@ def check_origin(self, origin): def on_message(self, message): - controls = json.loads(message) - dashboard = networktables.get() - dashboard.putBoolean(networktables.keys.vision_enable_camera, controls['enable_camera']) + + inputs = json.loads(message) + + data = None + if 'controls' in inputs: + controls = inputs['controls'] + dashboard.putBoolean(networktables.keys.vision_enable_camera, controls['enable_camera']) + # + # data = dict(enable_camera=main_controller.enable_camera, + # camera_mode=main_controller.camera_mode, + # enable_processing=main_controller.enable_processing, + # ) + + elif 'rgb' in inputs: + rgb = inputs['rgb'] + dashboard.putValue(networktables.keys.vision_color_profile, json.dumps(rgb)) + + # print(json.dumps(data)) + # self.send_msg_threadsafe(json.dumps(data)) + # print(controls['enable_camera']) # main_controller.enable_camera = controls['enable_camera'] - data = dict(enable_camera=main_controller.enable_camera, - camera_mode=main_controller.camera_mode, - enable_processing=main_controller.enable_processing) - print(json.dumps(data)) - self.send_msg_threadsafe(json.dumps(data)) def send_msg(self, msg): try: diff --git a/web/www/app.js b/web/www/app.js index 4b2756e..85673ec 100644 --- a/web/www/app.js +++ b/web/www/app.js @@ -8,16 +8,37 @@ new Vue({ enable_camera: false, enable_processing: false, camera_mode: 'R' + }, + rgb: { + red: { + min: 0, + max: 255 + }, + green: { + min: 0, + max: 255 + }, + blue: { + min: 0, + max: 255 + } } }, mounted: function () { console.log('mounted'); }, methods: { + + updateColors: function() { + var self = this; + console.log(self.rgb) + Socket.send({ + 'rgb': self.rgb + }) + }, enableCamera: function () { var self = this; - self.controls.enable_camera = self.controls.enable_camera - Socket.send(self.controls) + Socket.send({'controls': self.controls}) } } }); \ No newline at end of file diff --git a/web/www/index.html b/web/www/index.html index f338552..b897e15 100644 --- a/web/www/index.html +++ b/web/www/index.html @@ -13,10 +13,10 @@ + @@ -92,13 +92,58 @@
-
+ + + RGB Red Threshod + {{ rgb.red.min }} - {{ rgb.red.max }} + Min: + + Max: + + + + + RGB Green Threshod + {{ rgb.green.min }} - {{ rgb.green.max }} + Min: + + Max: + + + + + RGB Blue Threshod + {{ rgb.blue.min }} - {{ rgb.blue.max }} + Min: + + Max: + + + From 5fd85731ce7524650f1873d5edd79a30f1eb9443 Mon Sep 17 00:00:00 2001 From: Weijian Zeng Date: Tue, 14 Jan 2020 11:55:42 -0500 Subject: [PATCH 2/3] add imagezmq for streaming --- .gitignore | 1 + example_imagezmq.py | 23 +++ imagezmq/__init__.py | 14 ++ imagezmq/__version__.py | 9 ++ imagezmq/imagezmq.py | 347 ++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 6 files changed, 395 insertions(+) create mode 100644 example_imagezmq.py create mode 100644 imagezmq/__init__.py create mode 100644 imagezmq/__version__.py create mode 100644 imagezmq/imagezmq.py diff --git a/.gitignore b/.gitignore index 485dee6..c10666e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .idea +*.pyc diff --git a/example_imagezmq.py b/example_imagezmq.py new file mode 100644 index 0000000..7ee8844 --- /dev/null +++ b/example_imagezmq.py @@ -0,0 +1,23 @@ +import cv2 +import config +import time + +from imagezmq import imagezmq +from processing import filters + +def main(): + + cap = cv2.VideoCapture(config.video_source_number) + + sender = imagezmq.ImageSender(connect_to="tcp://{}:5555".format('192.168.1.25')) + + while(True): + + _, frame = cap.read() + + frame = filters.resize(frame, 640, 480, interpolation=cv2.INTER_CUBIC) + + sender.send_image('testImage', frame) + +if __name__ == '__main__': + main() diff --git a/imagezmq/__init__.py b/imagezmq/__init__.py new file mode 100644 index 0000000..6b7def1 --- /dev/null +++ b/imagezmq/__init__.py @@ -0,0 +1,14 @@ +""" +imagezmq: transport OpenCV images via ZMQ. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +A pair of Python classes that transport OpenCV images from one +computer to another. For example, OpenCV images gathered by +a Raspberry Pi camera could be sent to another computer +for displaying the images using cv2.imshow() or for further image processing. +Copyright (c) 2017 by Jeff Bass. +License: MIT, see LICENSE for more details. +""" +# populate fields for >>>help(imagezmq) +from .__version__ import __title__, __description__, __url__, __version__ +from .__version__ import __author__, __author_email__, __license__ +from .__version__ import __copyright__ diff --git a/imagezmq/__version__.py b/imagezmq/__version__.py new file mode 100644 index 0000000..e1be35e --- /dev/null +++ b/imagezmq/__version__.py @@ -0,0 +1,9 @@ +# populates fields for >>>help(imagezmq) +__title__ = 'imagezmq' +__description__ = 'Transporting OpenCV images via ZMQ' +__url__ = 'https://github.com/jeffbass/imagezmq' +__version__ = '0.3.0' +__author__ = 'Jeff Bass' +__author_email__ = 'jeff@yin-yang-ranch.com' +__license__ = 'MIT 1.0' +__copyright__ = 'Copyright 2019 Jeff Bass' diff --git a/imagezmq/imagezmq.py b/imagezmq/imagezmq.py new file mode 100644 index 0000000..2698883 --- /dev/null +++ b/imagezmq/imagezmq.py @@ -0,0 +1,347 @@ +""" imagezmq: Transport OpenCV images via ZMQ. +Classes that transport OpenCV images from one computer to another. For example, +OpenCV images gathered by a Raspberry Pi camera could be sent to another +computer for displaying the images using cv2.imshow() or for further image +processing. See API and Usage Examples for details. +Copyright (c) 2019 by Jeff Bass. +License: MIT, see LICENSE for more details. +""" + +import zmq +import numpy as np +import cv2 + + +class ImageSender(): + """Opens a zmq socket and sends images + Opens a zmq (REQ or PUB) socket on the image sending computer, often a + Raspberry Pi, that will be sending OpenCV images and + related text messages to the hub computer. Provides methods to + send images or send jpg compressed images. + Two kinds of ZMQ message patterns are possible in imagezmq: + REQ/REP: an image is sent and the sender waits for a reply ("blocking"). + PUB/SUB: an images is sent and no reply is sent or expected ("non-blocking"). + There are advantabes and disadvantages for each message pattern. + See the documentation for a full description of REQ/REP and PUB/SUB. + The default is REQ/REP for the ImageSender class and the ImageHub class. + Arguments: + connect_to: the tcp address:port of the hub computer. + REQ_REP: (optional) if True (the default), a REQ socket will be created + if False, a PUB socket will be created + """ + + def __init__(self, connect_to='tcp://127.0.0.1:5555', REQ_REP = True): + """Initializes zmq socket for sending images to the hub. + Expects an appropriate ZMQ socket at the connect_to tcp:port address: + If REQ_REP is True (the default), then a REQ socket is created. It + must connect to a matching REP socket on the ImageHub(). + If REQ_REP = False, then a PUB socket is created. It must connect to + a matching SUB socket on the ImageHub(). + """ + + if REQ_REP == True: + # REQ/REP mode, this is a blocking scenario + self.init_reqrep(connect_to) + else: + #PUB/SUB mode, non-blocking scenario + self.init_pubsub(connect_to) + + def init_reqrep(self, address): + """ Creates and inits a socket in REQ/REP mode + """ + + socketType = zmq.REQ + self.zmq_context = SerializingContext() + self.zmq_socket = self.zmq_context.socket(socketType) + self.zmq_socket.connect(address) + + # Assign corresponding send methods for REQ/REP mode + self.send_image = self.send_image_reqrep + self.send_jpg = self.send_jpg_reqrep + + def init_pubsub(self, address): + """Creates and inits a socket in PUB/SUB mode + """ + + socketType = zmq.PUB + self.zmq_context = SerializingContext() + self.zmq_socket = self.zmq_context.socket(socketType) + self.zmq_socket.bind(address) + + # Assign corresponding send methods for PUB/SUB mode + self.send_image = self.send_image_pubsub + self.send_jpg = self.send_jpg_pubsub + + def send_image(self, msg, image): + """ This is a placeholder. This method will be set to either a REQ/REP + or PUB/SUB sending method, depending on REQ_REP option value. + Arguments: + msg: text message or image name. + image: OpenCV image to send to hub. + Returns: + A text reply from hub in REQ/REP mode or nothing in PUB/SUB mode. + """ + pass + + def send_image_reqrep(self, msg, image): + """Sends OpenCV image and msg to hub computer in REQ/REP mode + Arguments: + msg: text message or image name. + image: OpenCV image to send to hub. + Returns: + A text reply from hub. + """ + + if image.flags['C_CONTIGUOUS']: + # if image is already contiguous in memory just send it + self.zmq_socket.send_array(image, msg, copy=False) + else: + # else make it contiguous before sending + image = np.ascontiguousarray(image) + self.zmq_socket.send_array(image, msg, copy=False) + hub_reply = self.zmq_socket.recv() # receive the reply message + return hub_reply + + def send_image_pubsub(self, msg, image): + """Sends OpenCV image and msg hub computer in PUB/SUB mode. If + there is no hub computer subscribed to this socket, then image and msg + are discarded. + Arguments: + msg: text message or image name. + image: OpenCV image to send to hub. + Returns: + Nothing; there is no reply from hub computer in PUB/SUB mode + """ + + if image.flags['C_CONTIGUOUS']: + # if image is already contiguous in memory just send it + self.zmq_socket.send_array(image, msg, copy=False) + else: + # else make it contiguous before sending + image = np.ascontiguousarray(image) + self.zmq_socket.send_array(image, msg, copy=False) + + def send_jpg(self, msg, jpg_buffer): + """This is a placeholder. This method will be set to either a REQ/REP + or PUB/SUB sending method, depending on REQ_REP option value. + Arguments: + msg: image name or message text. + jpg_buffer: bytestring containing the jpg image to send to hub. + Returns: + A text reply from hub in REQ/REP mode or nothing in PUB/SUB mode. + """ + + def send_jpg_reqrep(self, msg, jpg_buffer): + """Sends msg text and jpg buffer to hub computer in REQ/REP mode. + Arguments: + msg: image name or message text. + jpg_buffer: bytestring containing the jpg image to send to hub. + Returns: + A text reply from hub. + """ + + self.zmq_socket.send_jpg(msg, jpg_buffer, copy=False) + hub_reply = self.zmq_socket.recv() # receive the reply message + return hub_reply + + def send_jpg_pubsub(self, msg, jpg_buffer): + """Sends msg text and jpg buffer to hub computer in PUB/SUB mode. If + there is no hub computer subscribed to this socket, then image and msg + are discarded. + Arguments: + msg: image name or message text. + jpg_buffer: bytestring containing the jpg image to send to hub. + Returns: + Nothing; there is no reply from the hub computer in PUB/SUB mode. + """ + + self.zmq_socket.send_jpg(msg, jpg_buffer, copy=False) + + +class ImageHub(): + """Opens a zmq socket and receives images + Opens a zmq (REP or SUB) socket on the hub computer, for example, + a Mac, that will be receiving and displaying or processing OpenCV images + and related text messages. Provides methods to receive images or receive + jpg compressed images. + Two kinds of ZMQ message patterns are possible in imagezmq: + REQ/REP: an image is sent and the sender waits for a reply ("blocking"). + PUB/SUB: an images is sent and no reply is sent or expected ("non-blocking"). + There are advantabes and disadvantages for each message pattern. + See the documentation for a full description of REQ/REP and PUB/SUB. + The default is REQ/REP for the ImageSender class and the ImageHub class. + Arguments: + open_port: (optional) the socket to open for receiving REQ requests or + socket to connect to for SUB requests. + REQ_REP: (optional) if True (the default), a REP socket will be created + if False, a SUB socket will be created + """ + + def __init__(self, open_port='tcp://*:5555', REQ_REP = True): + """Initializes zmq socket to receive images and text. + Expects an appropriate ZMQ socket at the senders tcp:port address: + If REQ_REP is True (the default), then a REP socket is created. It + must connect to a matching REQ socket on the ImageSender(). + If REQ_REP = False, then a SUB socket is created. It must connect to + a matching PUB socket on the ImageSender(). + """ + self.REQ_REP = REQ_REP + if REQ_REP ==True: + #Init REP socket for blocking mode + self.init_reqrep(open_port) + else: + #Connect to PUB socket for non-blocking mode + self.init_pubsub(open_port) + + def init_reqrep(self, address): + """ Initializes Hub in REQ/REP mode + """ + socketType = zmq.REP + self.zmq_context = SerializingContext() + self.zmq_socket = self.zmq_context.socket(socketType) + self.zmq_socket.bind(address) + + def init_pubsub(self, address): + """ Initialize Hub in PUB/SUB mode + """ + socketType = zmq.SUB + self.zmq_context = SerializingContext() + self.zmq_socket = self.zmq_context.socket(socketType) + self.zmq_socket.setsockopt(zmq.SUBSCRIBE, b'') + self.zmq_socket.connect(address) + + def connect(self, open_port): + """In PUB/SUB mode, the hub can connect to multiple senders at the same + time. + Use this method to connect (and subscribe) to additional senders. + Arguments: + open_port: the PUB socket to connect to. + """ + + if self.REQ_REP == False: + #This makes sense only in PUB/SUB mode + self.zmq_socket.setsockopt(zmq.SUBSCRIBE, b'') + self.zmq_socket.connect(open_port) + self.zmq_socket.subscribe(b'') + return + + def recv_image(self, copy=False): + """Receives OpenCV image and text msg. + Arguments: + copy: (optional) zmq copy flag. + Returns: + msg: text msg, often the image name. + image: OpenCV image. + """ + + msg, image = self.zmq_socket.recv_array(copy=False) + return msg, image + + def recv_jpg(self, copy=False): + """Receives text msg, jpg buffer. + Arguments: + copy: (optional) zmq copy flag + Returns: + msg: text message, often image name + jpg_buffer: bytestring jpg compressed image + """ + + msg, jpg_buffer = self.zmq_socket.recv_jpg(copy=False) + return msg, jpg_buffer + + def send_reply(self, reply_message=b'OK'): + """Sends the zmq REP reply message. + Arguments: + reply_message: reply message text, often just string 'OK' + """ + self.zmq_socket.send(reply_message) + + +class SerializingSocket(zmq.Socket): + """Numpy array serialization methods. + Modelled on PyZMQ serialization examples. + Used for sending / receiving OpenCV images, which are Numpy arrays. + Also used for sending / receiving jpg compressed OpenCV images. + """ + + def send_array(self, A, msg='NoName', flags=0, copy=True, track=False): + """Sends a numpy array with metadata and text message. + Sends a numpy array with the metadata necessary for reconstructing + the array (dtype,shape). Also sends a text msg, often the array or + image name. + Arguments: + A: numpy array or OpenCV image. + msg: (optional) array name, image name or text message. + flags: (optional) zmq flags. + copy: (optional) zmq copy flag. + track: (optional) zmq track flag. + """ + + md = dict( + msg=msg, + dtype=str(A.dtype), + shape=A.shape, + ) + self.send_json(md, flags | zmq.SNDMORE) + return self.send(A, flags, copy=copy, track=track) + + def send_jpg(self, + msg='NoName', + jpg_buffer=b'00', + flags=0, + copy=True, + track=False): + """Send a jpg buffer with a text message. + Sends a jpg bytestring of an OpenCV image. + Also sends text msg, often the image name. + Arguments: + msg: image name or text message. + jpg_buffer: jpg buffer of compressed image to be sent. + flags: (optional) zmq flags. + copy: (optional) zmq copy flag. + track: (optional) zmq track flag. + """ + + md = dict(msg=msg, ) + self.send_json(md, flags | zmq.SNDMORE) + return self.send(jpg_buffer, flags, copy=copy, track=track) + + def recv_array(self, flags=0, copy=True, track=False): + """Receives a numpy array with metadata and text message. + Receives a numpy array with the metadata necessary + for reconstructing the array (dtype,shape). + Returns the array and a text msg, often the array or image name. + Arguments: + flags: (optional) zmq flags. + copy: (optional) zmq copy flag. + track: (optional) zmq track flag. + Returns: + msg: image name or text message. + A: numpy array or OpenCV image reconstructed with dtype and shape. + """ + + md = self.recv_json(flags=flags) + msg = self.recv(flags=flags, copy=copy, track=track) + A = np.frombuffer(msg, dtype=md['dtype']) + return (md['msg'], A.reshape(md['shape'])) + + def recv_jpg(self, flags=0, copy=True, track=False): + """Receives a jpg buffer and a text msg. + Receives a jpg bytestring of an OpenCV image. + Also receives a text msg, often the image name. + Arguments: + flags: (optional) zmq flags. + copy: (optional) zmq copy flag. + track: (optional) zmq track flag. + Returns: + msg: image name or text message. + jpg_buffer: bytestring, containing jpg image. + """ + + md = self.recv_json(flags=flags) # metadata text + jpg_buffer = self.recv(flags=flags, copy=copy, track=track) + return (md['msg'], jpg_buffer) + + +class SerializingContext(zmq.Context): + _socket_class = SerializingSocket diff --git a/requirements.txt b/requirements.txt index 3994822..0c34ed1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ pynetworktables==2019.0.2 tornado==6.0.3 +zmq=0.0.0 From 768e66e3edf3720137b8247367280b2588124959 Mon Sep 17 00:00:00 2001 From: Weijian Zeng Date: Tue, 14 Jan 2020 14:26:06 -0500 Subject: [PATCH 3/3] add common used gst pipelines --- gst_utils.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 gst_utils.py diff --git a/gst_utils.py b/gst_utils.py new file mode 100644 index 0000000..ac6ca45 --- /dev/null +++ b/gst_utils.py @@ -0,0 +1,47 @@ +""" +collect all known GStreamer methods here +""" + +def get_udp_sender(host, port): + 'appsrc ! queue ! videoconvert ! video/x-raw ! x264enc tune=zerolatency ! h264parse ! rtph264pay ! udpsink host="192.168.1.10" port="5000" sync=false' + pipeline = ' ! '.join( [appsrc(), + queue(), + videoconvert(), + video_x_raw(), + x264enc(zerolatency=True), + h264parse(), + rtph264pay(), + udpsink(host,port)]) + + print(pipeline) + return pipeline + +## PIPELINES + +def appsrc(): + return 'appsrc' + +def queue(): + return 'queue' + +def videoconvert(): + return 'videoconvert' + +def video_x_raw(): + return 'video/x-raw' + +def x264enc(zerolatency=True): + if zerolatency: + return 'x264enc tune=zerolatency' + return 'x264enc' + +def h264parse(): + return 'h264parse' + +def rtph264pay(): + return 'rtph264pay' + +def udpsink(host, port, sync=False): + if sync is False: + return 'udpsink host="%s" port="%s" sync=false' % (host,port) + return 'udpsink host="%s" port="%s"' % (host,port)