From 1fd1cba2e74a80736475acb623cbd3158c928a77 Mon Sep 17 00:00:00 2001 From: froxcey Date: Fri, 1 Nov 2024 16:21:14 +0000 Subject: [PATCH] Add more movement supports --- README.md | 11 ++- src/{views => display}/landmark.zig | 40 +++++--- src/{views => display}/main.zig | 10 +- src/display/shader.wgsl | 136 ++++++++++++++++++++++++++++ src/main.zig | 2 +- src/views/shader.wgsl | 83 ----------------- 6 files changed, 177 insertions(+), 105 deletions(-) rename src/{views => display}/landmark.zig (73%) rename src/{views => display}/main.zig (97%) create mode 100644 src/display/shader.wgsl delete mode 100644 src/views/shader.wgsl diff --git a/README.md b/README.md index 91c03c8..7182b78 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # V2D Studio -Cool wip stuff, build it yourself to see what this does. +Animate an image into a virtual avatar using your webcam. This is especially useful for those who don't have a 3D avatar but wants to start VTubing. + +V2D Studio is written in Zig using [Mach](https://machengine.org/) and [Mediapipe](https://ai.google.dev/edge/mediapipe). ## Installation @@ -39,9 +41,12 @@ If you have any general question regarding the usage of v2d, feel free to contac You're welcome to add projects here, as long as it is primarily written in a [tier B language or above](https://github.com/Froxcey/Froxcey/blob/main/lang_tier.md) and source available. We encourage users to check out alternatives and choose what is the best for their own use case. -## Future plan +## Future plans -It would be cool to add WebRTP video streaming support +1. WebRTP remote support +2. Parallax +3. Special effects and shaders +4. Plug-in system (longer term) ## Copyright & Usage Guideline diff --git a/src/views/landmark.zig b/src/display/landmark.zig similarity index 73% rename from src/views/landmark.zig rename to src/display/landmark.zig index a7061d3..a02e592 100644 --- a/src/views/landmark.zig +++ b/src/display/landmark.zig @@ -8,8 +8,7 @@ const resource_dir = "/opt/homebrew/opt/libmediapipe/lib/data"; const Self = @This(); const InitConfig = struct { - camera_id: u8 = 0, - use_gpu: bool = true, + id: u8 = 0, }; webcam: cv.VideoCapture, @@ -20,7 +19,7 @@ face_landmarks_poller: *mediapipe.mp_poller, pub fn init(config: InitConfig) !Self { var webcam = try cv.VideoCapture.init(); - try webcam.openDevice(config.camera_id); + try webcam.openDevice(config.id); mediapipe.mp_set_resource_dir(resource_dir); const builder = mediapipe.mp_create_instance_builder(resource_dir ++ "/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.binarypb", "image"); @@ -58,11 +57,18 @@ pub fn deinit(self: *Self) void { } pub const Result = struct { - head_tilt: f32, + head_x: f32 = 0, + head_y: f32 = 0, + head_z: f32 = 0, + body_pos_x: f32 = 0, + body_pos_y: f32 = 0, + body_rot_x: f32 = 0, + body_rot_z: f32 = 0, }; + pub fn poll(self: *Self) !Result { var frame = self.frame; - var head_tilt: f32 = 0; + var result = Result{}; self.webcam.read(&frame) catch { std.debug.print("capture failed", .{}); @@ -81,8 +87,8 @@ pub fn poll(self: *Self) !Result { .format = mediapipe.mp_image_format_srgba, }; - const p = mediapipe.mp_create_packet_image(image); - checkBool(mediapipe.mp_process(self.instance, p)); + const image_packet = mediapipe.mp_create_packet_image(image); + checkBool(mediapipe.mp_process(self.instance, image_packet)); checkBool(mediapipe.mp_wait_until_idle(self.instance)); if (mediapipe.mp_get_queue_size(self.pose_landmarks_poller) > 0) { @@ -91,6 +97,12 @@ pub fn poll(self: *Self) !Result { const landmarks = mediapipe.mp_get_norm_landmarks(packet); defer mediapipe.mp_destroy_landmarks(landmarks); // do pose landmark logic + const left_shulder = landmarks.*.elements[11]; + const right_shulder = landmarks.*.elements[12]; + result.body_pos_x = ((left_shulder.x + right_shulder.x) / 2 - 0.5) * 2; + result.body_pos_y = ((left_shulder.y + right_shulder.y) / 2 - 0.8) * 2; + result.body_rot_x = -std.math.atan2(left_shulder.y - right_shulder.y, left_shulder.x - right_shulder.x); + //result.body_rot_z = std.math.atan2(left_shulder.x - right_shulder.x, left_shulder.z - right_shulder.z) - 1.570795; } if (mediapipe.mp_get_queue_size(self.face_landmarks_poller) > 0) { @@ -100,15 +112,15 @@ pub fn poll(self: *Self) !Result { defer mediapipe.mp_destroy_landmarks(landmarks); const nose = landmarks.*.elements[116]; const cheek = landmarks.*.elements[345]; - head_tilt = std.math.atan2(cheek.y - nose.y, cheek.x - nose.x); - - // do face landmark logic - //drawLandmarks(&frame, landmarks); + const forehead = landmarks.*.elements[8]; + result.head_x = std.math.atan2(cheek.y - nose.y, cheek.x - nose.x); + result.head_y = std.math.atan2(forehead.z - nose.z, forehead.y - nose.y) + 2.513272; + result.head_y = @min(1, result.head_y); + result.head_y = @max(-1, result.head_y); + result.head_z = std.math.atan2(cheek.x - nose.x, cheek.z - nose.z) - 1.5707963; } - return Result{ - .head_tilt = head_tilt, - }; + return result; } fn checkNull(result: anytype) void { diff --git a/src/views/main.zig b/src/display/main.zig similarity index 97% rename from src/views/main.zig rename to src/display/main.zig index aa2b31f..0db5ee4 100644 --- a/src/views/main.zig +++ b/src/display/main.zig @@ -36,6 +36,7 @@ fn init(game: *Mod, core: *mach.Core.Mod) !void { const Config = struct { seg1: f32, + shoulder: f32, }; fn afterInit(game: *Mod, core: *mach.Core.Mod) !void { @@ -116,7 +117,8 @@ fn afterInit(game: *Mod, core: *mach.Core.Mod) !void { defer sampler.release(); const config = Config{ - .seg1 = 0.21875, + .seg1 = 0.15, + .shoulder = 0.06, }; const config_buffer = device.createBuffer(&.{ .label = "config", @@ -140,7 +142,7 @@ fn afterInit(game: *Mod, core: *mach.Core.Mod) !void { game.init(.{ .title_timer = try mach.Timer.start(), .pipeline = pipeline, - .landmarker = try landmark.init(.{ .camera_id = 2 }), + .landmarker = try landmark.init(.{ .id = 2 }), }); try updateWindowTitle(core); @@ -201,7 +203,7 @@ fn tick(core: *mach.Core.Mod, game: *Mod) !void { defer encoder.release(); // Begin render pass - const sky_blue_background = gpu.Color{ .r = 0.776, .g = 0.988, .b = 1, .a = 1 }; + const sky_blue_background = gpu.Color{ .r = 0, .g = 1, .b = 0, .a = 1 }; const color_attachments = [_]gpu.RenderPassColorAttachment{.{ .view = back_buffer_view, .clear_value = sky_blue_background, @@ -218,7 +220,7 @@ fn tick(core: *mach.Core.Mod, game: *Mod) !void { render_pass.setPipeline(game.state().pipeline); render_pass.setBindGroup(0, binding0, null); render_pass.setBindGroup(1, binding1, null); - render_pass.draw(12, 1, 0, 0); + render_pass.draw(6, 1, 0, 0); // Finish render pass render_pass.end(); diff --git a/src/display/shader.wgsl b/src/display/shader.wgsl new file mode 100644 index 0000000..1951984 --- /dev/null +++ b/src/display/shader.wgsl @@ -0,0 +1,136 @@ +struct Config { + seg1: f32, + shoulder: f32, +}; + +@group(0) @binding(0) var mainSampler: sampler; +@group(0) @binding(1) var mainTexture: texture_2d; +@group(0) @binding(2) var config: Config; + +struct Result { + head_x: f32, + head_y: f32, + head_z: f32, + body_pos_x: f32, + body_pos_y: f32, + body_rot_x: f32, + body_rot_z: f32, +}; + +@group(1) @binding(0) var result: Result; + +struct VertexOutput { + @builtin(position) position: vec4, + @location(0) texcoord: vec2, + rotate: bool, +}; + +@vertex fn vertex_main( + @builtin(vertex_index) VertexIndex : u32 +) -> VertexOutput { + var uvSeg1 = config.seg1 * 2 - 1; + + var pos = array, 6>( + // Seg 1 + vec2(-1.0, -1.0), // ↙ + vec2(1.0, -1.0), // ↘ + //vec2(-1.0, uvSeg1), // ↖ + //vec2(-1.0, uvSeg1), // ↖ + //vec2(1.0, -1.0), // ↘ + //vec2(1.0, uvSeg1), // ↗ + vec2(-1.0, 1), // ↖ + + // Seg 2 + //vec2(-1.0, uvSeg1), // ↙ + //vec2(1.0, uvSeg1), // ↘ + vec2(-1.0, 1), // ↖ + //vec2(-1.0, 1), // ↖ + //vec2(1.0, uvSeg1), // ↘ + vec2(1.0, 1), // ↗ + vec2(1.0, -1.0), // ↘ + ); + + var vsOutput: VertexOutput; + let xy = pos[VertexIndex]; + + vsOutput.texcoord = xy; + vsOutput.position = vec4(xy, 0.0, 1.0); + vsOutput.rotate = VertexIndex > 5; + return vsOutput; +} + +fn stage1(p: vec3) -> vec3{ + var y = (config.seg1) * 2 - 1; + if (p.y < y) { + return p; + } + var r = result.head_x + result.body_rot_x; + + var a = atan2(p.y - y, p.x); + var l = length(vec2(p.x, p.y - y)); + var t = a - r * (1 - pow(2.718281828459045, - (p.y - y) * 4)) * 0.4; + return vec3(l * cos(t), l * sin(t) + y, p.z); +} + +fn stage2(p: vec3) -> vec3{ + var y = (config.seg1) * 2 - 1; + var r = result.head_y; + + var t = 1.5 * r / (1 + pow(2.718281828459045, -18 * (p.y - y))); //(1 - pow(2.718281828459045, - (p.y - y) * 12)); + //return vec3(p.x, p.y, 0.3 * sin(t)); + return vec3(p.x, (p.y - y) / cos(t) + y, sin(t)); +} + +fn stage3(p: vec3) -> vec3{ + var y = (config.seg1) * 2 - 1; + var r = result.head_z;// + result.body_rot_z; + + var t = r * 0.8 / (1 + pow(2.718281828459045, -18 * (p.y - y)));//(1 - pow(2.718281828459045, - (p.y - y) * 12)) * r; + return vec3(p.x / cos(t), p.y, p.z * sin(t)); +} + +fn stage4(p: vec3) -> vec3{ + var r = result.body_rot_z; + return vec3(p.x / cos(r) + p.z * sin(r), p.y, 0); +} + +fn stage5(p: vec3) -> vec3{ + var r = result.body_rot_x; + var shoulderUV = config.shoulder * 2 - 1; + return vec3(p.x * cos(r) - (p.y - shoulderUV) * sin(r), p.x * sin(r) + (p.y - shoulderUV) * cos(r) + shoulderUV, 0); +} + +fn toTexCoord(xy: vec2) -> vec2 { + return xy * vec2(0.5, -0.5) + 0.5; +} + +@fragment fn frag_main(fsInput: VertexOutput) -> @location(0) vec4 { + var coord = vec3(fsInput.texcoord, 0); + + // Stage 1: head x + coord = stage1(coord); + + // Stage 2: head y + coord = stage2(coord); + + // Stage 3: head z + coord = stage3(coord); + + // Stage 4: body rotate z + coord = stage4(coord); + + // Stage 5: body rotate x + coord = stage5(coord); + + // Stage 6: transform + coord = coord + vec3(result.body_pos_x, result.body_pos_y, 0); + + // Squash to 2d + var texcoord = vec2(coord.x, coord.y); + + // convert the coordinate + texcoord = toTexCoord(texcoord); + + // Render output + return textureSample(mainTexture, mainSampler, texcoord); +} diff --git a/src/main.zig b/src/main.zig index 020d308..b5c9ea0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2,7 +2,7 @@ const mach = @import("mach"); pub const modules = .{ mach.Core, - @import("views/main.zig"), + @import("display/main.zig"), }; pub fn main() !void { diff --git a/src/views/shader.wgsl b/src/views/shader.wgsl deleted file mode 100644 index 685b5bf..0000000 --- a/src/views/shader.wgsl +++ /dev/null @@ -1,83 +0,0 @@ -struct Config { - seg1: f32, -}; - -@group(0) @binding(0) var mainSampler: sampler; -@group(0) @binding(1) var mainTexture: texture_2d; -@group(0) @binding(2) var config: Config; - -struct Result { - head_tilt: f32, -}; - -@group(1) @binding(0) var result: Result; - -struct VertexOutput { - @builtin(position) position: vec4, - @location(0) texcoord: vec2, - rotate: bool, -}; - -@vertex fn vertex_main( - @builtin(vertex_index) VertexIndex : u32 -) -> VertexOutput { - var uvSeg1 = config.seg1 * 2 - 1; - - var pos = array, 18>( - // Seg 1 - vec2(-1.0, -1.0), // ↙ - vec2(1.0, -1.0), // ↘ - vec2(-1.0, uvSeg1), // ↖ - vec2(-1.0, uvSeg1), // ↖ - vec2(1.0, -1.0), // ↘ - vec2(1.0, uvSeg1), // ↗ - - // Seg 2 - vec2(-1.0, uvSeg1), // ↙ - vec2(1.0, uvSeg1), // ↘ - vec2(-1.0, 1), // ↖ - vec2(-1.0, 1), // ↖ - vec2(1.0, uvSeg1), // ↘ - vec2(1.0, 1), // ↗ - ); - - var vsOutput: VertexOutput; - let xy = pos[VertexIndex]; - - vsOutput.texcoord = xy;//flipY(xy); - vsOutput.position = vec4(xy, 0.0, 1.0); - vsOutput.rotate = VertexIndex > 5; - return vsOutput; -} - -fn rotate(xy: vec2, y: f32, r: f32) -> vec2{ - var a = atan2(xy.y - y, xy.x); - var l = length(vec2(xy.x, xy.y - y)); - var t = a - r * (1 - pow(2.718281828459045, - (xy.y - y) * 2.5)); - return vec2(l * cos(t), l * sin(t) + y); -} - -fn flipY(xy: vec2) -> vec2 { - return xy * vec2(0.5, -0.5) + 0.5; -} - -// fn mapping(pos: vec2) -> vec2 { -// let modulus: f32 = length(pos); -// let rotation: f32 = atan2(pos.y, pos.x) + pow(modulus / 1.414 - 1, 3) * 2; -// return vec2(cos(rotation), sin(rotation)) * modulus; -// } -fn mapping(pos: vec2) -> vec2 { - let modulus: f32 = length(pos); - let rotation: f32 = atan2(pos.y, pos.x) + pow(modulus / 1.414 - 1, 3) * 2; - return vec2(cos(rotation), sin(rotation)) * modulus; -} - -@fragment fn frag_main(fsInput: VertexOutput) -> @location(0) vec4 { - var uvSeg1 = config.seg1 * 2 - 1; - var head_tilt = result.head_tilt; - var texcoord = fsInput.texcoord; - if (fsInput.rotate) { - texcoord = rotate(texcoord, uvSeg1, head_tilt); - } - return textureSample(mainTexture, mainSampler, flipY(texcoord)); -}