diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 12e8bac..e8cb09a 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -590,6 +590,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c132eebf10f5cad5289222520a4a058514204aed6d791f1cf4fe8088b82d15f" +dependencies = [ + "objc2", +] + [[package]] name = "blocking" version = "1.5.1" @@ -893,6 +902,15 @@ dependencies = [ "vob", ] +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + [[package]] name = "chrono" version = "0.4.38" @@ -1103,6 +1121,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "core-audio-types-rs" +version = "0.1.0" +source = "git+https://github.com/doom-fish/core-frameworks.git#da192ff073a9157c52fb078ffb66a7edb91ca085" +dependencies = [ + "core-foundation 0.10.0", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1190,6 +1216,39 @@ dependencies = [ "libc", ] +[[package]] +name = "core-media-rs" +version = "0.1.0" +source = "git+https://github.com/doom-fish/core-frameworks.git#da192ff073a9157c52fb078ffb66a7edb91ca085" +dependencies = [ + "core-audio-types-rs", + "core-foundation 0.10.0", + "core-utils-rs", + "core-video-rs", + "thiserror", +] + +[[package]] +name = "core-utils-rs" +version = "0.1.0" +source = "git+https://github.com/doom-fish/core-frameworks.git#da192ff073a9157c52fb078ffb66a7edb91ca085" +dependencies = [ + "core-foundation 0.10.0", + "four-char-code", +] + +[[package]] +name = "core-video-rs" +version = "0.1.0" +source = "git+https://github.com/doom-fish/core-frameworks.git#da192ff073a9157c52fb078ffb66a7edb91ca085" +dependencies = [ + "core-foundation 0.10.0", + "core-graphics 0.24.0", + "core-utils-rs", + "io-surface", + "thiserror", +] + [[package]] name = "coreaudio-rs" version = "0.10.0" @@ -2392,6 +2451,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "four-char-code" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c661315fd366b2a1f970df7b7cb1a28d2678d49ef4872f7dcc19b4a83150f20b" + [[package]] name = "fs_extra" version = "1.3.0" @@ -3561,6 +3626,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "io-surface" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8283575d5f0b2e7447ec0840363879d71c0fa325d4c699d5b45208ea4a51f45e" +dependencies = [ + "cgl", + "core-foundation 0.10.0", + "core-foundation-sys", + "leaky-cow", + "libc", +] + [[package]] name = "ipnet" version = "2.9.0" @@ -3883,6 +3961,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leak" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd100e01f1154f2908dfa7d02219aeab25d0b9c7fa955164192e3245255a0c73" + +[[package]] +name = "leaky-cow" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a8225d44241fd324a8af2806ba635fc7c8a7e9a7de4d5cf3ef54e71f5926fc" +dependencies = [ + "leak", +] + [[package]] name = "lebe" version = "0.5.2" @@ -4191,6 +4284,7 @@ version = "0.9.23" dependencies = [ "chrono", "core-graphics 0.24.0", + "core-media-rs", "cpal", "crossbeam-channel", "ct2rs", @@ -5015,6 +5109,28 @@ dependencies = [ "objc_id", ] +[[package]] +name = "objc-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb91bdd390c7ce1a8607f35f3ca7151b65afc0ff5ff3b34fa350f7d7c7e4310" + +[[package]] +name = "objc2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a785d4eeff09c14c487497c162e92766fbb3e4059a71840cecc03d9a50b804" +dependencies = [ + "objc-sys", + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7891e71393cd1f227313c9379a26a584ff3d7e6e7159e988851f0934c993f0f8" + [[package]] name = "objc_exception" version = "0.1.2" @@ -6463,23 +6579,16 @@ checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" [[package]] name = "screencapturekit" -version = "0.2.7" -source = "git+https://github.com/solaoi/screencapturekit-rs#e4f37692ce04d378262e19d26b56c5cc81af2330" -dependencies = [ - "screencapturekit-sys", -] - -[[package]] -name = "screencapturekit-sys" -version = "0.2.7" -source = "git+https://github.com/solaoi/screencapturekit-rs#e4f37692ce04d378262e19d26b56c5cc81af2330" +version = "0.3.0" +source = "git+https://github.com/doom-fish/screencapturekit-rs?rev=3582a58#3582a58cebf3e49c6ce299288f6571c83dc9bb6c" dependencies = [ - "block", + "block2", + "core-foundation 0.10.0", + "core-graphics 0.24.0", + "core-media-rs", + "core-utils-rs", "dispatch", "objc", - "objc-foundation", - "objc_id", - "once_cell", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 26b8337..9ee64be 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -34,9 +34,9 @@ futures-util = "0.3.30" # serve audio urlencoding = "2.1.3" # screencapturekit = "0.2.8" -# fix thread crash -# screencapturekit = { git = "https://github.com/doom-fish/screencapturekit-rs", rev = "8829f70"} -screencapturekit = { git = "https://github.com/solaoi/screencapturekit-rs" } +screencapturekit = { git = "https://github.com/doom-fish/screencapturekit-rs", rev = "3582a58"} +core-media-rs = { git = "https://github.com/doom-fish/core-frameworks.git" } + xcap = "0.0.14" # permission macos-accessibility-client = "0.0.1" diff --git a/src-tauri/src/module/record_desktop.rs b/src-tauri/src/module/record_desktop.rs index 8f3eb63..d112251 100644 --- a/src-tauri/src/module/record_desktop.rs +++ b/src-tauri/src/module/record_desktop.rs @@ -25,33 +25,27 @@ use crossbeam_channel::{unbounded, Receiver, Sender}; use hound::{WavSpec, WavWriter}; use tauri::{api::path::data_dir, AppHandle, Manager}; +use core_media_rs::cm_sample_buffer::CMSampleBuffer; use screencapturekit::{ - cm_sample_buffer::CMSampleBuffer, - sc_content_filter::{InitParams, SCContentFilter}, - sc_error_handler::StreamErrorHandler, - sc_output_handler::{SCStreamOutputType, StreamOutput}, - sc_shareable_content::SCShareableContent, - sc_stream::SCStream, - sc_stream_configuration::SCStreamConfiguration, + shareable_content::SCShareableContent, + stream::{ + configuration::SCStreamConfiguration, content_filter::SCContentFilter, + output_trait::SCStreamOutputTrait, output_type::SCStreamOutputType, SCStream, + }, }; use vosk::Recognizer; use super::{ - chat_online, recognizer::MyRecognizer, sqlite::Sqlite, transcription, transcription_amivoice, transcription_ja, transcription_online, translation_en, translation_ja, translation_ja_high, writer::Writer + chat_online, recognizer::MyRecognizer, sqlite::Sqlite, transcription, transcription_amivoice, + transcription_ja, transcription_online, translation_en, translation_ja, translation_ja_high, + writer::Writer, }; pub struct RecordDesktop { app_handle: AppHandle, } -struct ErrorHandler; -impl StreamErrorHandler for ErrorHandler { - fn on_error(&self) { - println!("Error!"); - } -} - struct StoreAudioHandler { app_handle: AppHandle, recognizer: Weak>, @@ -60,32 +54,34 @@ struct StoreAudioHandler { channels: u16, } -impl StreamOutput for StoreAudioHandler { +impl SCStreamOutputTrait for StoreAudioHandler { fn did_output_sample_buffer(&self, sample: CMSampleBuffer, _of_type: SCStreamOutputType) { - let audio_buffers = sample.sys_ref.get_av_audio_buffer_list(); + let audio_buffers = sample.get_audio_buffer_list(); if audio_buffers.is_err() { println!("Error getting audio buffer list"); } - let bytes = &audio_buffers.unwrap()[0].data; // 最初のチャンネルのデータを選択 + if let Some(buffer) = audio_buffers.unwrap().get(0) { + let bytes = &buffer.data(); - // バッファのデータを直接処理 - let samples: Vec = bytes - .chunks_exact(4) - .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap())) - .collect(); + // バッファのデータを f32 に変換 + let samples: Vec = bytes + .chunks_exact(4) + .map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap())) + .collect(); - if let Some(recognizer) = self.recognizer.upgrade() { - MyRecognizer::recognize( - self.app_handle.clone(), - &mut recognizer.lock().unwrap(), - &samples, - self.channels, - self.notify_decoding_state_is_finalized_tx.clone(), - true, - ); - } + if let Some(recognizer) = self.recognizer.upgrade() { + MyRecognizer::recognize( + self.app_handle.clone(), + &mut recognizer.lock().unwrap(), + &samples, + self.channels, + self.notify_decoding_state_is_finalized_tx.clone(), + true, + ); + } - Writer::write_input_data::(&samples, &self.writer_clone); + Writer::write_input_data::(&samples, &self.writer_clone); + } } } @@ -102,32 +98,29 @@ impl RecordDesktop { stop_record_rx: Receiver<()>, stop_record_clone_tx: Option>, ) { - let mut current = SCShareableContent::current(); - let display = current.displays.pop().unwrap(); + let display = SCShareableContent::get().unwrap().displays().remove(0); - let channels = 1 as u16; + let channels = 1 as u8; + // the system uses a default sample rate of 48 kHz. + // https://developer.apple.com/documentation/screencapturekit/scstreamconfiguration/3931903-samplerate let sample_rate = 48000; let bits_per_sample = 32; - let config = SCStreamConfiguration { - width: 100, - height: 100, - captures_audio: true, - sample_rate, - channel_count: channels as u32, - excludes_current_process_audio: true, - ..Default::default() - }; - + let config = SCStreamConfiguration::new() + .set_captures_audio(true) + .unwrap() + .set_channel_count(channels) + .unwrap(); + let recognizer = MyRecognizer::build( self.app_handle.clone(), speaker_language.clone(), - config.sample_rate as f32, + sample_rate as f32, ); let recognizer_arc = Arc::new(Mutex::new(recognizer)); let recognizer_weak = Arc::downgrade(&recognizer_arc); let spec = WavSpec { - channels, + channels: channels as u16, sample_rate, bits_per_sample, sample_format: hound::SampleFormat::Float, @@ -145,15 +138,15 @@ impl RecordDesktop { let (notify_decoding_state_is_finalized_tx, notify_decoding_state_is_finalized_rx) = sync_channel(1); let app_handle = self.app_handle.clone(); - let filter = SCContentFilter::new(InitParams::Display(display)); - let mut stream = SCStream::new(filter, config, ErrorHandler); - stream.add_output( + let filter = SCContentFilter::new().with_display_excluding_windows(&display, &[]); + let mut stream = SCStream::new(&filter, &config); + stream.add_output_handler( StoreAudioHandler { app_handle, recognizer: recognizer_weak, notify_decoding_state_is_finalized_tx, writer_clone, - channels, + channels: channels as u16, }, SCStreamOutputType::Audio, );