Skip to content

Commit

Permalink
Add more movement supports
Browse files Browse the repository at this point in the history
  • Loading branch information
Froxcey committed Nov 1, 2024
1 parent 7a4d1c1 commit 1fd1cba
Show file tree
Hide file tree
Showing 6 changed files with 177 additions and 105 deletions.
11 changes: 8 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# V2D Studio

Cool wip stuff, build it yourself to see what this does.
Animate an image into a virtual avatar using your webcam. This is especially useful for those who don't have a 3D avatar but wants to start VTubing.

V2D Studio is written in Zig using [Mach](https://machengine.org/) and [Mediapipe](https://ai.google.dev/edge/mediapipe).

## Installation

Expand Down Expand Up @@ -39,9 +41,12 @@ If you have any general question regarding the usage of v2d, feel free to contac

You're welcome to add projects here, as long as it is primarily written in a [tier B language or above](https://github.com/Froxcey/Froxcey/blob/main/lang_tier.md) and source available. We encourage users to check out alternatives and choose what is the best for their own use case.

## Future plan
## Future plans

It would be cool to add WebRTP video streaming support
1. WebRTP remote support
2. Parallax
3. Special effects and shaders
4. Plug-in system (longer term)

## Copyright & Usage Guideline

Expand Down
40 changes: 26 additions & 14 deletions src/views/landmark.zig → src/display/landmark.zig
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@ const resource_dir = "/opt/homebrew/opt/libmediapipe/lib/data";

const Self = @This();
const InitConfig = struct {
camera_id: u8 = 0,
use_gpu: bool = true,
id: u8 = 0,
};

webcam: cv.VideoCapture,
Expand All @@ -20,7 +19,7 @@ face_landmarks_poller: *mediapipe.mp_poller,

pub fn init(config: InitConfig) !Self {
var webcam = try cv.VideoCapture.init();
try webcam.openDevice(config.camera_id);
try webcam.openDevice(config.id);

mediapipe.mp_set_resource_dir(resource_dir);
const builder = mediapipe.mp_create_instance_builder(resource_dir ++ "/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.binarypb", "image");
Expand Down Expand Up @@ -58,11 +57,18 @@ pub fn deinit(self: *Self) void {
}

pub const Result = struct {
head_tilt: f32,
head_x: f32 = 0,
head_y: f32 = 0,
head_z: f32 = 0,
body_pos_x: f32 = 0,
body_pos_y: f32 = 0,
body_rot_x: f32 = 0,
body_rot_z: f32 = 0,
};

pub fn poll(self: *Self) !Result {
var frame = self.frame;
var head_tilt: f32 = 0;
var result = Result{};

self.webcam.read(&frame) catch {
std.debug.print("capture failed", .{});
Expand All @@ -81,8 +87,8 @@ pub fn poll(self: *Self) !Result {
.format = mediapipe.mp_image_format_srgba,
};

const p = mediapipe.mp_create_packet_image(image);
checkBool(mediapipe.mp_process(self.instance, p));
const image_packet = mediapipe.mp_create_packet_image(image);
checkBool(mediapipe.mp_process(self.instance, image_packet));
checkBool(mediapipe.mp_wait_until_idle(self.instance));

if (mediapipe.mp_get_queue_size(self.pose_landmarks_poller) > 0) {
Expand All @@ -91,6 +97,12 @@ pub fn poll(self: *Self) !Result {
const landmarks = mediapipe.mp_get_norm_landmarks(packet);
defer mediapipe.mp_destroy_landmarks(landmarks);
// do pose landmark logic
const left_shulder = landmarks.*.elements[11];
const right_shulder = landmarks.*.elements[12];
result.body_pos_x = ((left_shulder.x + right_shulder.x) / 2 - 0.5) * 2;
result.body_pos_y = ((left_shulder.y + right_shulder.y) / 2 - 0.8) * 2;
result.body_rot_x = -std.math.atan2(left_shulder.y - right_shulder.y, left_shulder.x - right_shulder.x);
//result.body_rot_z = std.math.atan2(left_shulder.x - right_shulder.x, left_shulder.z - right_shulder.z) - 1.570795;
}

if (mediapipe.mp_get_queue_size(self.face_landmarks_poller) > 0) {
Expand All @@ -100,15 +112,15 @@ pub fn poll(self: *Self) !Result {
defer mediapipe.mp_destroy_landmarks(landmarks);
const nose = landmarks.*.elements[116];
const cheek = landmarks.*.elements[345];
head_tilt = std.math.atan2(cheek.y - nose.y, cheek.x - nose.x);

// do face landmark logic
//drawLandmarks(&frame, landmarks);
const forehead = landmarks.*.elements[8];
result.head_x = std.math.atan2(cheek.y - nose.y, cheek.x - nose.x);
result.head_y = std.math.atan2(forehead.z - nose.z, forehead.y - nose.y) + 2.513272;
result.head_y = @min(1, result.head_y);
result.head_y = @max(-1, result.head_y);
result.head_z = std.math.atan2(cheek.x - nose.x, cheek.z - nose.z) - 1.5707963;
}

return Result{
.head_tilt = head_tilt,
};
return result;
}

fn checkNull(result: anytype) void {
Expand Down
10 changes: 6 additions & 4 deletions src/views/main.zig → src/display/main.zig
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ fn init(game: *Mod, core: *mach.Core.Mod) !void {

const Config = struct {
seg1: f32,
shoulder: f32,
};

fn afterInit(game: *Mod, core: *mach.Core.Mod) !void {
Expand Down Expand Up @@ -116,7 +117,8 @@ fn afterInit(game: *Mod, core: *mach.Core.Mod) !void {
defer sampler.release();

const config = Config{
.seg1 = 0.21875,
.seg1 = 0.15,
.shoulder = 0.06,
};
const config_buffer = device.createBuffer(&.{
.label = "config",
Expand All @@ -140,7 +142,7 @@ fn afterInit(game: *Mod, core: *mach.Core.Mod) !void {
game.init(.{
.title_timer = try mach.Timer.start(),
.pipeline = pipeline,
.landmarker = try landmark.init(.{ .camera_id = 2 }),
.landmarker = try landmark.init(.{ .id = 2 }),
});
try updateWindowTitle(core);

Expand Down Expand Up @@ -201,7 +203,7 @@ fn tick(core: *mach.Core.Mod, game: *Mod) !void {
defer encoder.release();

// Begin render pass
const sky_blue_background = gpu.Color{ .r = 0.776, .g = 0.988, .b = 1, .a = 1 };
const sky_blue_background = gpu.Color{ .r = 0, .g = 1, .b = 0, .a = 1 };
const color_attachments = [_]gpu.RenderPassColorAttachment{.{
.view = back_buffer_view,
.clear_value = sky_blue_background,
Expand All @@ -218,7 +220,7 @@ fn tick(core: *mach.Core.Mod, game: *Mod) !void {
render_pass.setPipeline(game.state().pipeline);
render_pass.setBindGroup(0, binding0, null);
render_pass.setBindGroup(1, binding1, null);
render_pass.draw(12, 1, 0, 0);
render_pass.draw(6, 1, 0, 0);

// Finish render pass
render_pass.end();
Expand Down
136 changes: 136 additions & 0 deletions src/display/shader.wgsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
struct Config {
seg1: f32,
shoulder: f32,
};

@group(0) @binding(0) var mainSampler: sampler;
@group(0) @binding(1) var mainTexture: texture_2d<f32>;
@group(0) @binding(2) var<uniform> config: Config;

struct Result {
head_x: f32,
head_y: f32,
head_z: f32,
body_pos_x: f32,
body_pos_y: f32,
body_rot_x: f32,
body_rot_z: f32,
};

@group(1) @binding(0) var<uniform> result: Result;

struct VertexOutput {
@builtin(position) position: vec4<f32>,
@location(0) texcoord: vec2<f32>,
rotate: bool,
};

@vertex fn vertex_main(
@builtin(vertex_index) VertexIndex : u32
) -> VertexOutput {
var uvSeg1 = config.seg1 * 2 - 1;

var pos = array<vec2<f32>, 6>(
// Seg 1
vec2<f32>(-1.0, -1.0), // ↙
vec2<f32>(1.0, -1.0), // ↘
//vec2<f32>(-1.0, uvSeg1), // ↖
//vec2<f32>(-1.0, uvSeg1), // ↖
//vec2<f32>(1.0, -1.0), // ↘
//vec2<f32>(1.0, uvSeg1), // ↗
vec2<f32>(-1.0, 1), // ↖

// Seg 2
//vec2<f32>(-1.0, uvSeg1), // ↙
//vec2<f32>(1.0, uvSeg1), // ↘
vec2<f32>(-1.0, 1), // ↖
//vec2<f32>(-1.0, 1), // ↖
//vec2<f32>(1.0, uvSeg1), // ↘
vec2<f32>(1.0, 1), // ↗
vec2<f32>(1.0, -1.0), // ↘
);

var vsOutput: VertexOutput;
let xy = pos[VertexIndex];

vsOutput.texcoord = xy;
vsOutput.position = vec4<f32>(xy, 0.0, 1.0);
vsOutput.rotate = VertexIndex > 5;
return vsOutput;
}

fn stage1(p: vec3<f32>) -> vec3<f32>{
var y = (config.seg1) * 2 - 1;
if (p.y < y) {
return p;
}
var r = result.head_x + result.body_rot_x;

var a = atan2(p.y - y, p.x);
var l = length(vec2<f32>(p.x, p.y - y));
var t = a - r * (1 - pow(2.718281828459045, - (p.y - y) * 4)) * 0.4;
return vec3<f32>(l * cos(t), l * sin(t) + y, p.z);
}

fn stage2(p: vec3<f32>) -> vec3<f32>{
var y = (config.seg1) * 2 - 1;
var r = result.head_y;

var t = 1.5 * r / (1 + pow(2.718281828459045, -18 * (p.y - y))); //(1 - pow(2.718281828459045, - (p.y - y) * 12));
//return vec3<f32>(p.x, p.y, 0.3 * sin(t));
return vec3<f32>(p.x, (p.y - y) / cos(t) + y, sin(t));
}

fn stage3(p: vec3<f32>) -> vec3<f32>{
var y = (config.seg1) * 2 - 1;
var r = result.head_z;// + result.body_rot_z;

var t = r * 0.8 / (1 + pow(2.718281828459045, -18 * (p.y - y)));//(1 - pow(2.718281828459045, - (p.y - y) * 12)) * r;
return vec3<f32>(p.x / cos(t), p.y, p.z * sin(t));
}

fn stage4(p: vec3<f32>) -> vec3<f32>{
var r = result.body_rot_z;
return vec3<f32>(p.x / cos(r) + p.z * sin(r), p.y, 0);
}

fn stage5(p: vec3<f32>) -> vec3<f32>{
var r = result.body_rot_x;
var shoulderUV = config.shoulder * 2 - 1;
return vec3<f32>(p.x * cos(r) - (p.y - shoulderUV) * sin(r), p.x * sin(r) + (p.y - shoulderUV) * cos(r) + shoulderUV, 0);
}

fn toTexCoord(xy: vec2<f32>) -> vec2<f32> {
return xy * vec2<f32>(0.5, -0.5) + 0.5;
}

@fragment fn frag_main(fsInput: VertexOutput) -> @location(0) vec4<f32> {
var coord = vec3<f32>(fsInput.texcoord, 0);

// Stage 1: head x
coord = stage1(coord);

// Stage 2: head y
coord = stage2(coord);

// Stage 3: head z
coord = stage3(coord);

// Stage 4: body rotate z
coord = stage4(coord);

// Stage 5: body rotate x
coord = stage5(coord);

// Stage 6: transform
coord = coord + vec3<f32>(result.body_pos_x, result.body_pos_y, 0);

// Squash to 2d
var texcoord = vec2<f32>(coord.x, coord.y);

// convert the coordinate
texcoord = toTexCoord(texcoord);

// Render output
return textureSample(mainTexture, mainSampler, texcoord);
}
2 changes: 1 addition & 1 deletion src/main.zig
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ const mach = @import("mach");

pub const modules = .{
mach.Core,
@import("views/main.zig"),
@import("display/main.zig"),
};

pub fn main() !void {
Expand Down
83 changes: 0 additions & 83 deletions src/views/shader.wgsl

This file was deleted.

0 comments on commit 1fd1cba

Please sign in to comment.