diff --git a/turbo/api.py b/turbo/api.py new file mode 100644 index 0000000..0afe0ed --- /dev/null +++ b/turbo/api.py @@ -0,0 +1,133 @@ +#This is an example that uses the websockets api to know when a prompt execution is done +#Once the prompt execution is done it downloads the images using the /history endpoint +import os +import websocket #NOTE: websocket-client (https://github.com/websocket-client/websocket-client) +import uuid +import json +import urllib.request +import urllib.parse +from flask import Flask, request, jsonify, render_template, session, abort +from flask_socketio import SocketIO, join_room, leave_room,send, emit +import secrets + +from PIL import Image +import io +import base64 + +server_address = "127.0.0.1:8188" +client_id = str(uuid.uuid4()) + +def queue_prompt(prompt): + p = {"prompt": prompt, "client_id": client_id} + data = json.dumps(p).encode('utf-8') + req = urllib.request.Request("http://{}/prompt".format(server_address), data=data) + return json.loads(urllib.request.urlopen(req).read()) + +def get_image(filename, subfolder, folder_type): + data = {"filename": filename, "subfolder": subfolder, "type": folder_type} + url_values = urllib.parse.urlencode(data) + with urllib.request.urlopen("http://{}/view?{}".format(server_address, url_values)) as response: + return response.read() + +def get_history(prompt_id): + with urllib.request.urlopen("http://{}/history/{}".format(server_address, prompt_id)) as response: + return json.loads(response.read()) + +def get_images(ws, prompt): + prompt_id = queue_prompt(prompt)['prompt_id'] + output_images = {} + while True: + out = ws.recv() + if isinstance(out, str): + message = json.loads(out) + if message['type'] == 'executing': + data = message['data'] + if data['node'] is None and data['prompt_id'] == prompt_id: + break #Execution is done + else: + continue #previews are binary data + + history = get_history(prompt_id)[prompt_id] + for o in history['outputs']: + for node_id in history['outputs']: + node_output = history['outputs'][node_id] + if 'images' in node_output: + images_output = [] + for image in node_output['images']: + image_data = get_image(image['filename'], image['subfolder'], image['type']) + images_output.append(image_data) + output_images[node_id] = images_output + + return output_images + +prompt={} +with open('./workflow_api_motionctrl_turbo.json') as fr: + prompt = json.load(fr) + +ws = websocket.WebSocket() +ws.connect("ws://{}/ws?clientId={}".format(server_address, client_id)) + +#Commented out code to display the output images: + +# for node_id in images: +# for image_data in images[node_id]: +# from PIL import Image +# import io +# image = Image.open(io.BytesIO(image_data)) +# image.show() + +app = Flask(__name__, template_folder=os.path.abspath('.'), static_folder='assets') +app.secret_key = secrets.token_hex(16) + +socketio = SocketIO(app, cors_allowed_origins='*') +connected_sids = set() # 存放已连接的客户端 + +#后端程序 +lockroom='None' +@socketio.on('connect') +def on_connect(): + connected_sids.add(request.sid) + print(f'{request.sid} 已连接') + socketio.start_background_task(background_thread_heartbeat) + +@socketio.on('disconnect') +def on_disconnect(): + connected_sids.remove(request.sid) + print(f'{request.sid} 已断开') + +@socketio.on('message') +def handle_message(message): + """收消息""" + print(f'message:{request.sid} {message}') + json.loads(message) + +@socketio.on('camera_poses') +def handle_message(camera_poses): + print(f'camera_poses:{request.sid} {camera_poses}') + prompt["60"]["inputs"]["camera"] = camera_poses["camera_poses"] + images = get_images(ws, prompt) + for node_id in images: + for image_data in images[node_id]: + b64img=base64.b64encode(image_data).decode('utf-8') + socketio.emit('server_response',{'b64img':b64img}, to=camera_poses["roomid"]) + +@socketio.on('server_reconnect') +def server_reconnect(message): + print(f'server_reconnect:{request.sid} {message}') + join_room(message['roomid']) + +def background_thread_heartbeat(): + global lockroom + while True: + socketio.emit('server_response',{'lockroom':lockroom}) + socketio.sleep(5) + + +@app.route('/') +def index(): + session['user'] = None + return render_template('index.html') + +if __name__ == '__main__': + socketio.run(app, host='0.0.0.0', port=5017, debug=True) + #app.run(host='0.0.0.0', port=5017) \ No newline at end of file diff --git a/turbo/dist/index.html b/turbo/dist/index.html new file mode 100644 index 0000000..1867f75 --- /dev/null +++ b/turbo/dist/index.html @@ -0,0 +1,22 @@ + + + + + + CAMERA MOTION DESIGNER + + + + + +
+ +
+ +
+
+
+ + \ No newline at end of file diff --git a/turbo/index.html b/turbo/index.html new file mode 100644 index 0000000..03a1e5e --- /dev/null +++ b/turbo/index.html @@ -0,0 +1,22 @@ + + + + + + + CAMERA MOTION DESIGNER + + + +
+ +
+ +
+
+
+ + + \ No newline at end of file diff --git a/turbo/main.css b/turbo/main.css new file mode 100644 index 0000000..cf37d25 --- /dev/null +++ b/turbo/main.css @@ -0,0 +1,91 @@ +body { + margin: 0; + background-color: #000; + color: #fff; + font-family: Monospace; + font-size: 13px; + line-height: 24px; + overscroll-behavior: none; +} + +a { + color: #ff0; + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +button { + cursor: pointer; + text-transform: uppercase; +} + +#info { + position: absolute; + top: 0px; + left: 0px; + padding: 10px; + box-sizing: border-box; + text-align: center; + /*-moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; + user-select: none; + pointer-events: none;*/ + z-index: 1; /* TODO Solve this in HTML */ +} + +a, button, input, select { + pointer-events: auto; +} + +.lil-gui { + z-index: 2 !important; /* TODO Solve this in HTML */ +} + +@media all and ( max-width: 640px ) { + .lil-gui.root { + right: auto; + top: auto; + max-height: 50%; + max-width: 80%; + bottom: 0; + left: 0; + } +} + +#overlay { + position: absolute; + font-size: 16px; + z-index: 2; + top: 0; + left: 0; + width: 100%; + height: 100%; + display: flex; + align-items: center; + justify-content: center; + flex-direction: column; + background: rgba(0,0,0,0.7); +} + + #overlay button { + background: transparent; + border: 0; + border: 1px solid rgb(255, 255, 255); + border-radius: 4px; + color: #ffffff; + padding: 12px 18px; + text-transform: uppercase; + cursor: pointer; + } + +#notSupported { + width: 50%; + margin: auto; + background-color: #f00; + margin-top: 20px; + padding: 10px; +} diff --git a/turbo/main.js b/turbo/main.js new file mode 100644 index 0000000..2f8838d --- /dev/null +++ b/turbo/main.js @@ -0,0 +1,270 @@ +import * as THREE from 'three'; + +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js'; +import { TransformControls } from 'three/examples/jsm/controls/TransformControls.js'; +import { io } from "https://cdn.socket.io/4.7.2/socket.io.esm.min.js"; + +let cameraPersp, cameraPerspTransform, currentCamera, cameraPerspTransformHelper; +let scene, renderer, control, orbit; + +let roomid=new Date().getTime(); + +let cposes=[]; +let cmatrix=[]; + +var socket = io(); +socket.on('connect', function() { + socket.emit('server_reconnect', {roomid: roomid}); +}); + + +socket.on("server_response", function (msg,ack) { + console.log(msg); + //接收到后端发送过来的消息 + var b64img = msg.b64img; + if(!b64img)return; + const imageContainer = document.querySelector('.imageContainer'); + + var image = new Image(); + image.src = 'data:image/jpeg;base64,' + b64img; + + while (imageContainer.firstChild) { + imageContainer.removeChild(imageContainer.firstChild); + } + + imageContainer.prepend(image); + }); + +init(); +render(); + + + +function detect_change(){ + var matrix=cameraPerspTransform.matrix.elements; + matrix[3]=cameraPerspTransform.position.x; + matrix[7]=cameraPerspTransform.position.y; + matrix[11]=cameraPerspTransform.position.z; + matrix=matrix.slice(0, 12); + + if(JSON.stringify(cmatrix)!=JSON.stringify(matrix)){ + cposes.push(matrix); + cmatrix=matrix; + }else{ + if(cposes.length){ + console.log(cposes); + socket.emit('camera_poses', {roomid:roomid,camera_poses:JSON.stringify(cposes)}); + } + + cposes=[]; + } + + setTimeout(function(){ + detect_change(); + },100); +} + +function init() { + + renderer = new THREE.WebGLRenderer( { antialias: true } ); + renderer.setPixelRatio( window.devicePixelRatio ); + renderer.setSize( window.innerWidth, window.innerHeight ); + document.body.appendChild( renderer.domElement ); + + const aspect = window.innerWidth / window.innerHeight; + + cameraPersp = new THREE.PerspectiveCamera( 50, aspect, 0.01, 30000 ); + cameraPerspTransform = new THREE.PerspectiveCamera( 50, aspect, 0.01, 30000 ); + + cameraPerspTransformHelper = new THREE.CameraHelper( cameraPerspTransform ); + + currentCamera = cameraPersp; + + currentCamera.position.set( 5, 2.5, 5 ); + + scene = new THREE.Scene(); + scene.add( new THREE.GridHelper( 5, 10, 0x888888, 0x444444 ) ); + + const ambientLight = new THREE.AmbientLight( 0xffffff ); + scene.add( ambientLight ); + + const light = new THREE.DirectionalLight( 0xffffff, 4 ); + light.position.set( 1, 1, 1 ); + scene.add( light ); + + const texture = new THREE.TextureLoader().load( 'textures/crate.gif', render ); + texture.colorSpace = THREE.SRGBColorSpace; + texture.anisotropy = renderer.capabilities.getMaxAnisotropy(); + + const geometry = new THREE.BoxGeometry(); + const material = new THREE.MeshLambertMaterial( { map: texture } ); + + orbit = new OrbitControls( currentCamera, renderer.domElement ); + orbit.update(); + orbit.addEventListener( 'change', render ); + + control = new TransformControls( currentCamera, renderer.domElement ); + control.addEventListener( 'change', render ); + + control.addEventListener( 'dragging-changed', function ( event ) { + + orbit.enabled = ! event.value; + + } ); + + const mesh = new THREE.Mesh( geometry, material ); + scene.add( cameraPerspTransform ); + + control.attach( cameraPerspTransform ); + scene.add( control ); + scene.add( cameraPerspTransformHelper ); + + document.getElementById('btn_translate').addEventListener('click',function(){ + control.setMode( 'translate' ); + }); + + document.getElementById('btn_rotate').addEventListener('click',function(){ + control.setMode( 'rotate' ); + }); + + document.getElementById('btn_addpoint').addEventListener('click',function(){ + var ret=JSON.parse(document.getElementById('tb_result').value); + var matrix=cameraPerspTransform.matrix.elements; + matrix[3]=cameraPerspTransform.position.x; + matrix[7]=cameraPerspTransform.position.y; + matrix[11]=cameraPerspTransform.position.z; + matrix=matrix.slice(0, 12); + ret.push(matrix); + document.getElementById('tb_result').value=JSON.stringify(ret); + }); + + document.getElementById('btn_startrt').addEventListener('click',function(){ + detect_change(); + }); + + window.addEventListener( 'resize', onWindowResize ); + + window.addEventListener( 'keydown', function ( event ) { + + switch ( event.keyCode ) { + + case 81: // Q + control.setSpace( control.space === 'local' ? 'world' : 'local' ); + break; + + case 16: // Shift + control.setTranslationSnap( 100 ); + control.setRotationSnap( THREE.MathUtils.degToRad( 15 ) ); + control.setScaleSnap( 0.25 ); + break; + + case 87: // W + control.setMode( 'translate' ); + break; + + case 69: // E + control.setMode( 'rotate' ); + break; + + case 82: // R + control.setMode( 'scale' ); + break; + + case 67: // C + const position = currentCamera.position.clone(); + + currentCamera = currentCamera.isPerspectiveCamera ? cameraPerspTransform : cameraPersp; + currentCamera.position.copy( position ); + + orbit.object = currentCamera; + control.camera = currentCamera; + + currentCamera.lookAt( orbit.target.x, orbit.target.y, orbit.target.z ); + onWindowResize(); + break; + + case 86: // V + const randomFoV = Math.random() + 0.1; + const randomZoom = Math.random() + 0.1; + + cameraPersp.fov = randomFoV * 160; + cameraPerspTransform.bottom = - randomFoV * 500; + cameraPerspTransform.top = randomFoV * 500; + + cameraPersp.zoom = randomZoom * 5; + cameraPerspTransform.zoom = randomZoom * 5; + onWindowResize(); + break; + + case 187: + case 107: // +, =, num+ + control.setSize( control.size + 0.1 ); + break; + + case 189: + case 109: // -, _, num- + control.setSize( Math.max( control.size - 0.1, 0.1 ) ); + break; + + case 88: // X + control.showX = ! control.showX; + break; + + case 89: // Y + control.showY = ! control.showY; + break; + + case 90: // Z + control.showZ = ! control.showZ; + break; + + case 32: // Spacebar + control.enabled = ! control.enabled; + break; + + case 27: // Esc + control.reset(); + break; + + } + + } ); + + window.addEventListener( 'keyup', function ( event ) { + + switch ( event.keyCode ) { + + case 16: // Shift + control.setTranslationSnap( null ); + control.setRotationSnap( null ); + control.setScaleSnap( null ); + break; + + } + + } ); + +} + +function onWindowResize() { + + const aspect = window.innerWidth / window.innerHeight; + + cameraPersp.aspect = aspect; + cameraPersp.updateProjectionMatrix(); + + cameraPerspTransform.left = cameraPerspTransform.bottom * aspect; + cameraPerspTransform.right = cameraPerspTransform.top * aspect; + cameraPerspTransform.updateProjectionMatrix(); + + renderer.setSize( window.innerWidth, window.innerHeight ); + + render(); + +} + +function render() { + + renderer.render( scene, currentCamera ); + +} diff --git a/turbo/package.json b/turbo/package.json new file mode 100644 index 0000000..70269be --- /dev/null +++ b/turbo/package.json @@ -0,0 +1,8 @@ +{ + "dependencies": { + "three": "^0.160.0" + }, + "devDependencies": { + "vite": "^5.0.11" + } +} diff --git a/turbo/workflow_api_motionctrl_turbo.json b/turbo/workflow_api_motionctrl_turbo.json new file mode 100644 index 0000000..43d2d6a --- /dev/null +++ b/turbo/workflow_api_motionctrl_turbo.json @@ -0,0 +1,77 @@ +{ + "56": { + "inputs": { + "ckpt_name": "motionctrl.pth", + "frame_length": 16 + }, + "class_type": "Load Motionctrl Checkpoint" + }, + "57": { + "inputs": { + "steps": 12, + "seed": 1860, + "traj_tool": "https://chaojie.github.io/ComfyUI-MotionCtrl/tools/draw.html", + "draw_traj_dot": false, + "draw_camera_dot": false, + "model": [ + "56", + 0 + ], + "clip": [ + "56", + 1 + ], + "vae": [ + "56", + 2 + ], + "ddim_sampler": [ + "56", + 3 + ], + "positive": [ + "60", + 0 + ], + "negative": [ + "60", + 1 + ], + "traj": [ + "60", + 2 + ], + "rt": [ + "60", + 3 + ], + "noise_shape": [ + "60", + 4 + ] + }, + "class_type": "Motionctrl Sample Simple" + }, + "59": { + "inputs": { + "filename_prefix": "motionctrl", + "images": [ + "57", + 0 + ] + }, + "class_type": "SaveImage" + }, + "60": { + "inputs": { + "prompt": "a rose swaying in the wind", + "camera": "[[1,0,0,0,0,1,0,0,0,0,1,0.2]]", + "traj": "[[117, 102]]", + "model": [ + "56", + 0 + ] + }, + "class_type": "Motionctrl Cond" + } +} \ No newline at end of file