diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitignore b/.gitignore index 1f36757..cbc0c7f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target models +.direnv diff --git a/Cargo.lock b/Cargo.lock index 9009523..020b8cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,41 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "aho-corasick" version = "1.1.2" @@ -301,6 +336,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.7.0" @@ -461,6 +506,15 @@ dependencies = [ "windows", ] +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.4.0" @@ -510,6 +564,26 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "rand_core", + "typenum", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "darling" version = "0.14.4" @@ -666,6 +740,29 @@ dependencies = [ "syn 2.0.49", ] +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -912,6 +1009,16 @@ dependencies = [ "seq-macro", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.12" @@ -923,6 +1030,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gimli" version = "0.28.1" @@ -992,6 +1109,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hf-hub" version = "0.3.2" @@ -1052,6 +1175,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.28" @@ -1115,6 +1244,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -1275,9 +1413,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "louds-rs" @@ -1627,6 +1765,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl" version = "0.10.64" @@ -1730,6 +1874,18 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.6.0" @@ -2202,9 +2358,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.32" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ "indexmap", "itoa", @@ -2373,15 +2529,17 @@ dependencies = [ ] [[package]] -name = "tempest" +name = "tempest-client" version = "0.2.2" dependencies = [ + "aes-gcm", "anyhow", "aprilasr-sys", "candle-core", "candle-nn", "candle-transformers", "cpal", + "hex", "hf-hub", "log", "mouse-keyboard-input", @@ -2394,6 +2552,20 @@ dependencies = [ "trie-rs", ] +[[package]] +name = "tempest-daemon" +version = "0.2.2" +dependencies = [ + "aes-gcm", + "anyhow", + "env_logger", + "hex", + "log", + "mouse-keyboard-input", + "serde", + "serde_yaml", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -2607,6 +2779,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -2655,11 +2833,21 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common", + "subtle", +] + [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -2710,6 +2898,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "walkdir" version = "2.4.0" diff --git a/Cargo.toml b/Cargo.toml index e0afb99..2ad1c2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,2 @@ -[package] -name = "tempest" -version = "0.2.2" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = "1.0.80" -aprilasr-sys = { path = "src/aprilasr-sys" } -candle-core = "0.4.1" -candle-nn = "0.4.1" -candle-transformers = "0.4.1" -cpal = "0.15.2" -hf-hub = "0.3.2" -log = "0.4.20" -mouse-keyboard-input = "0.4.1" -reqwest = { version = "0.11.24", default-features = false, features = ["json", "blocking", "rustls"] } -serde = { version = "1.0.197", features = ["derive"] } -serde_json = "1.0.114" -serde_yaml = "0.9.32" -simple_logger = "4.3.3" -tokenizers = "0.15.2" -trie-rs = "0.2.0" +[workspace] +members = ["client", "daemon"] diff --git a/README.md b/README.md index a792dbf..5dc3bf2 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,6 @@ Prerequisites: Luckily, if you use NixOS with flakes, you can run `nix develop` in the project directory to get a dev shell with all the dependencies installed. -Note: You might have to add your user to the `uinput` group. - Once you have installed the necessary tools, clone this repo. Run the `download_models.sh` script to download the models needed for speech recongnition and textual inference. ``` @@ -34,18 +32,46 @@ cd tempest ./download_models.sh ``` -You may change the keybindings in the config file in case you are not using GNOME+PaperWM. +Change the bindings in the config file to suit your needs. + +Run the following to build the daemon and the client: + +```sh +cargo build --workspace --release +``` + +#### Daemon + +The daemon is optional and is only needed if you want phrases in your bindings to perform keyboard shortcuts. +Since performing keystrokes is a privileged action, you must run the daemon as root. + +```sh +sudo ./target/release/tempest-daemon +``` + +This will give a token to authenticate with the daemon. + +#### Client + +If you have the daemon running in the background, in a different terminal tab, run + +```sh +./target/release/tempest-client the_token_from_the_daemon +``` + +where `the_token_from_the_daemon` is the token provided by the daemon. -Finally, run the following: +If you wish to opt out of using the daemon, you can run the client standalone. However, config bindings with keyboard shortcuts will not work. ```sh -cargo run +./target/release/tempest-client ``` #### Note -The april model referenced in the download script might not work properly if you +The april model in the download script might not work properly if you have a terrible microphone like mine. In that case, you may download an older model from [here](https://april.sapples.net/aprilv0_en-us.april) and save it as `model.april` in the project directory. +This older model is less accurate in speech recognition but can work with more noisy data. ### Acknowledgements diff --git a/client/Cargo.toml b/client/Cargo.toml new file mode 100644 index 0000000..16ad333 --- /dev/null +++ b/client/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "tempest-client" +version = "0.2.2" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +aes-gcm = "0.10.3" +anyhow = "1.0.80" +aprilasr-sys = { path = "src/aprilasr-sys" } +candle-core = "0.4.1" +candle-nn = "0.4.1" +candle-transformers = "0.4.1" +cpal = "0.15.2" +hex = "0.4.3" +hf-hub = "0.3.2" +log = "0.4.20" +mouse-keyboard-input = "0.4.1" +reqwest = { version = "0.11.24", default-features = false, features = ["json", "blocking", "rustls"] } +serde = { version = "1.0.197", features = ["derive"] } +serde_json = "1.0.114" +serde_yaml = "0.9.32" +simple_logger = "4.3.3" +tokenizers = "0.15.2" +trie-rs = "0.2.0" diff --git a/src/aprilasr-sys/.cargo_vcs_info.json b/client/src/aprilasr-sys/.cargo_vcs_info.json similarity index 100% rename from src/aprilasr-sys/.cargo_vcs_info.json rename to client/src/aprilasr-sys/.cargo_vcs_info.json diff --git a/src/aprilasr-sys/COPYING b/client/src/aprilasr-sys/COPYING similarity index 100% rename from src/aprilasr-sys/COPYING rename to client/src/aprilasr-sys/COPYING diff --git a/src/aprilasr-sys/Cargo.lock b/client/src/aprilasr-sys/Cargo.lock similarity index 100% rename from src/aprilasr-sys/Cargo.lock rename to client/src/aprilasr-sys/Cargo.lock diff --git a/src/aprilasr-sys/Cargo.toml b/client/src/aprilasr-sys/Cargo.toml similarity index 100% rename from src/aprilasr-sys/Cargo.toml rename to client/src/aprilasr-sys/Cargo.toml diff --git a/src/aprilasr-sys/Cargo.toml.orig b/client/src/aprilasr-sys/Cargo.toml.orig similarity index 100% rename from src/aprilasr-sys/Cargo.toml.orig rename to client/src/aprilasr-sys/Cargo.toml.orig diff --git a/src/aprilasr-sys/README.md b/client/src/aprilasr-sys/README.md similarity index 100% rename from src/aprilasr-sys/README.md rename to client/src/aprilasr-sys/README.md diff --git a/src/aprilasr-sys/build.rs b/client/src/aprilasr-sys/build.rs similarity index 100% rename from src/aprilasr-sys/build.rs rename to client/src/aprilasr-sys/build.rs diff --git a/src/aprilasr-sys/src/lib.rs b/client/src/aprilasr-sys/src/lib.rs similarity index 100% rename from src/aprilasr-sys/src/lib.rs rename to client/src/aprilasr-sys/src/lib.rs diff --git a/src/aprilasr-sys/vendor/april-asr/CMakeLists.txt b/client/src/aprilasr-sys/vendor/april-asr/CMakeLists.txt similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/CMakeLists.txt rename to client/src/aprilasr-sys/vendor/april-asr/CMakeLists.txt diff --git a/src/aprilasr-sys/vendor/april-asr/COPYING b/client/src/aprilasr-sys/vendor/april-asr/COPYING similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/COPYING rename to client/src/aprilasr-sys/vendor/april-asr/COPYING diff --git a/src/aprilasr-sys/vendor/april-asr/README.md b/client/src/aprilasr-sys/vendor/april-asr/README.md similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/README.md rename to client/src/aprilasr-sys/vendor/april-asr/README.md diff --git a/src/aprilasr-sys/vendor/april-asr/april-asr.pc.in b/client/src/aprilasr-sys/vendor/april-asr/april-asr.pc.in similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/april-asr.pc.in rename to client/src/aprilasr-sys/vendor/april-asr/april-asr.pc.in diff --git a/src/aprilasr-sys/vendor/april-asr/april_api.h b/client/src/aprilasr-sys/vendor/april-asr/april_api.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/april_api.h rename to client/src/aprilasr-sys/vendor/april-asr/april_api.h diff --git a/src/aprilasr-sys/vendor/april-asr/cmake/FindONNXRuntime.cmake b/client/src/aprilasr-sys/vendor/april-asr/cmake/FindONNXRuntime.cmake similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/cmake/FindONNXRuntime.cmake rename to client/src/aprilasr-sys/vendor/april-asr/cmake/FindONNXRuntime.cmake diff --git a/src/aprilasr-sys/vendor/april-asr/src/april_model.c b/client/src/aprilasr-sys/vendor/april-asr/src/april_model.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/april_model.c rename to client/src/aprilasr-sys/vendor/april-asr/src/april_model.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/april_model.h b/client/src/aprilasr-sys/vendor/april-asr/src/april_model.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/april_model.h rename to client/src/aprilasr-sys/vendor/april-asr/src/april_model.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/april_session.c b/client/src/aprilasr-sys/vendor/april-asr/src/april_session.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/april_session.c rename to client/src/aprilasr-sys/vendor/april-asr/src/april_session.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/april_session.h b/client/src/aprilasr-sys/vendor/april-asr/src/april_session.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/april_session.h rename to client/src/aprilasr-sys/vendor/april-asr/src/april_session.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/audio_provider.c b/client/src/aprilasr-sys/vendor/april-asr/src/audio_provider.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/audio_provider.c rename to client/src/aprilasr-sys/vendor/april-asr/src/audio_provider.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/audio_provider.h b/client/src/aprilasr-sys/vendor/april-asr/src/audio_provider.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/audio_provider.h rename to client/src/aprilasr-sys/vendor/april-asr/src/audio_provider.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/common.h b/client/src/aprilasr-sys/vendor/april-asr/src/common.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/common.h rename to client/src/aprilasr-sys/vendor/april-asr/src/common.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/fbank.c b/client/src/aprilasr-sys/vendor/april-asr/src/fbank.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/fbank.c rename to client/src/aprilasr-sys/vendor/april-asr/src/fbank.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/fbank.h b/client/src/aprilasr-sys/vendor/april-asr/src/fbank.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/fbank.h rename to client/src/aprilasr-sys/vendor/april-asr/src/fbank.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.c b/client/src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.c rename to client/src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.h b/client/src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.h rename to client/src/aprilasr-sys/vendor/april-asr/src/fft/pocketfft.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/file/model_file.c b/client/src/aprilasr-sys/vendor/april-asr/src/file/model_file.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/file/model_file.c rename to client/src/aprilasr-sys/vendor/april-asr/src/file/model_file.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/file/model_file.h b/client/src/aprilasr-sys/vendor/april-asr/src/file/model_file.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/file/model_file.h rename to client/src/aprilasr-sys/vendor/april-asr/src/file/model_file.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/file/util.h b/client/src/aprilasr-sys/vendor/april-asr/src/file/util.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/file/util.h rename to client/src/aprilasr-sys/vendor/april-asr/src/file/util.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/init.c b/client/src/aprilasr-sys/vendor/april-asr/src/init.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/init.c rename to client/src/aprilasr-sys/vendor/april-asr/src/init.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/log.h b/client/src/aprilasr-sys/vendor/april-asr/src/log.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/log.h rename to client/src/aprilasr-sys/vendor/april-asr/src/log.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/ort_util.c b/client/src/aprilasr-sys/vendor/april-asr/src/ort_util.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/ort_util.c rename to client/src/aprilasr-sys/vendor/april-asr/src/ort_util.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/ort_util.h b/client/src/aprilasr-sys/vendor/april-asr/src/ort_util.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/ort_util.h rename to client/src/aprilasr-sys/vendor/april-asr/src/ort_util.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/params.c b/client/src/aprilasr-sys/vendor/april-asr/src/params.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/params.c rename to client/src/aprilasr-sys/vendor/april-asr/src/params.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/params.h b/client/src/aprilasr-sys/vendor/april-asr/src/params.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/params.h rename to client/src/aprilasr-sys/vendor/april-asr/src/params.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/proc_thread.c b/client/src/aprilasr-sys/vendor/april-asr/src/proc_thread.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/proc_thread.c rename to client/src/aprilasr-sys/vendor/april-asr/src/proc_thread.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/proc_thread.h b/client/src/aprilasr-sys/vendor/april-asr/src/proc_thread.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/proc_thread.h rename to client/src/aprilasr-sys/vendor/april-asr/src/proc_thread.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.c b/client/src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.c rename to client/src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.h b/client/src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.h rename to client/src/aprilasr-sys/vendor/april-asr/src/sonic/sonic.h diff --git a/src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.c b/client/src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.c similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.c rename to client/src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.c diff --git a/src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.h b/client/src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.h similarity index 100% rename from src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.h rename to client/src/aprilasr-sys/vendor/april-asr/src/tinycthread/tinycthread.h diff --git a/src/aprilasr-sys/wrapper.h b/client/src/aprilasr-sys/wrapper.h similarity index 100% rename from src/aprilasr-sys/wrapper.h rename to client/src/aprilasr-sys/wrapper.h diff --git a/client/src/config.rs b/client/src/config.rs new file mode 100644 index 0000000..c7f8d69 --- /dev/null +++ b/client/src/config.rs @@ -0,0 +1,109 @@ +use serde::Deserialize; +use std::collections::BTreeMap; +use trie_rs::TrieBuilder; + +#[derive(Deserialize)] +pub struct RawConfig { + pub model_path: String, + pub wake_phrase: String, + pub rest_phrase: String, + pub infer_phrase: String, + pub actions: Vec, + pub ollama_model: String, + pub ollama_endpoint: String, +} + +#[derive(Clone, Debug)] +pub enum Action { + Keys(String), + Command(Vec), +} + +#[derive(Deserialize)] +#[serde(rename_all = "lowercase")] +enum RawAction { + Keys(Vec), + Command(Vec), +} + +#[derive(Deserialize)] +pub struct RawBinding { + phrase: String, + + #[serde(flatten)] + action: RawAction, +} + +pub struct Config { + pub model_path: String, + pub actions: BTreeMap, + pub word_trie: trie_rs::Trie, + pub keys: Vec, + pub abstract_triggers: trie_rs::Trie, + pub modes: BTreeMap, + pub ollama_model: String, + pub ollama_endpoint: String, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Mode { + Wake, + Rest, + Infer, + Custom(usize), +} + +impl From for Config { + fn from(value: RawConfig) -> Self { + let wake_phrase = value.wake_phrase.to_lowercase(); + let rest_phrase = value.rest_phrase.to_lowercase(); + let infer_phrase = value.infer_phrase.to_lowercase(); + let mut trie_builder = TrieBuilder::new(); + for phrase in value.actions.iter().map(|b| b.phrase.to_lowercase()) { + trie_builder.push(phrase); + } + let word_trie = trie_builder.build(); + let keys = value + .actions + .iter() + .map(|b| b.phrase.to_lowercase()) + .collect(); + let actions = value + .actions + .into_iter() + .map(|b| { + let action = match b.action { + RawAction::Command(v) => Action::Command(v), + RawAction::Keys(_) => Action::Keys(b.phrase.to_lowercase()), + }; + + (b.phrase.to_lowercase(), action) + }) + .collect(); + + let mut trie_builder = TrieBuilder::new(); + trie_builder.push(wake_phrase.clone()); + trie_builder.push(rest_phrase.clone()); + trie_builder.push(infer_phrase.clone()); + + let abstract_triggers = trie_builder.build(); + let modes = [ + (wake_phrase, Mode::Wake), + (rest_phrase, Mode::Rest), + (infer_phrase, Mode::Infer), + ] + .into_iter() + .collect(); + + Self { + model_path: value.model_path, + abstract_triggers, + modes, + keys, + actions, + word_trie, + ollama_model: value.ollama_model, + ollama_endpoint: value.ollama_endpoint, + } + } +} diff --git a/src/lib.rs b/client/src/lib.rs similarity index 100% rename from src/lib.rs rename to client/src/lib.rs diff --git a/src/llm.rs b/client/src/llm.rs similarity index 100% rename from src/llm.rs rename to client/src/llm.rs diff --git a/src/main.rs b/client/src/main.rs similarity index 84% rename from src/main.rs rename to client/src/main.rs index ffc0526..a23f15f 100644 --- a/src/main.rs +++ b/client/src/main.rs @@ -1,3 +1,7 @@ +use aes_gcm::{ + aead::{Aead, OsRng}, + AeadCore, Aes256Gcm, Key, KeyInit, +}; use anyhow::{anyhow, bail, Context, Result}; use config::{Action, Mode}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; @@ -5,11 +9,14 @@ use cpal::StreamConfig; use log::error; use state::State; use std::collections::BTreeMap; +use std::env::args; use std::fs::File; +use std::io::Write; +use std::os::unix::net::UnixStream; use std::process::Command; use std::sync::mpsc::{channel, Receiver}; use std::thread; -use tempest::{init_april_api, Model, ResultType, Session, Token}; +use tempest_client::{init_april_api, Model, ResultType, Session, Token}; use trie_rs::Trie; use candle_transformers::models::bert::{BertModel, Config, HiddenAct, DTYPE}; @@ -20,34 +27,20 @@ use candle_nn::VarBuilder; use hf_hub::{api::sync::Api, Repo, RepoType}; use tokenizers::{PaddingParams, Tokenizer}; -use mouse_keyboard_input::VirtualDevice; - mod config; mod llm; mod state; +pub struct AuthenticatedUnixStream { + key: Key, + stream: UnixStream, +} + fn tokens_to_string(tokens: Vec) -> String { let tokens_str: Vec = tokens.iter().map(|t| t.token()).collect(); tokens_str.join("") } -pub struct VirtualInput(VirtualDevice); - -impl VirtualInput { - fn key_chord(&mut self, keys: &[u16]) { - for &key in keys { - if let Err(e) = self.0.press(key) { - error!("failed to press key {key}: {e}"); - } - } - for &key in keys.iter().rev() { - if let Err(e) = self.0.release(key) { - error!("failed to release key {key}: {e}"); - } - } - } -} - pub struct TrieMatchBookkeeper { pub actions_consumed_upto: usize, pub trie: Trie, @@ -59,7 +52,11 @@ pub struct TrieMatchBookkeeper { } impl TrieMatchBookkeeper { - fn word_to_action(&mut self, phrase: &str, vd: &mut VirtualInput) -> bool { + fn word_to_action( + &mut self, + phrase: &str, + stream: &mut Option, + ) -> bool { if phrase.is_empty() { return false; } @@ -72,7 +69,7 @@ impl TrieMatchBookkeeper { 0 => start = i - 1, 1 if self.trie.exact_match(search) => { self.current_action = self.actions.get(search).cloned(); - self.do_action(vd); + self.do_action(stream); self.actions_consumed_upto = i; } _ => {} @@ -108,12 +105,33 @@ impl TrieMatchBookkeeper { self.current_action = None; self.actions_consumed_upto = 0; } - fn do_action(&self, vd: &mut VirtualInput) { + fn do_action(&self, stream: &mut Option) { if self.current_action.is_none() { return; } match self.current_action.clone().unwrap() { - Action::Keys(keys) => vd.key_chord(&keys), + Action::Keys(keys) => { + let Some(stream) = stream else { + error!("recognized keybinding will not be executed since the daemon is not running"); + error!("please make sure the daemon is running as a member of the input / uinput group"); + return; + }; + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let cipher = Aes256Gcm::new(&stream.key); + let ciphertext = cipher.encrypt(&nonce, keys.as_bytes()).unwrap(); + let nonce_cat_ciphertext_cat_newline = + format!("{}{}\n", hex::encode(nonce), hex::encode(ciphertext)); + if let Err(e) = stream + .stream + .write_all(nonce_cat_ciphertext_cat_newline.as_bytes()) + { + error!( + "failed to send keyboard shortcut `{}` to daemon socket: {}", + keys, e + ); + error!("please make sure the daemon is running as a member of the input / uinput group"); + } + } Action::Command(command) => { let res = if let Some((command, args)) = command.split_first() { Command::new(command).args(args).spawn() @@ -131,7 +149,7 @@ impl TrieMatchBookkeeper { } fn inference_loop( - mut device: VirtualInput, + mut stream: Option, mut state: State, mut bookkeeper: TrieMatchBookkeeper, session_rx: Receiver, @@ -151,7 +169,7 @@ fn inference_loop( log::info!("{sentence:#?} is inferred as: {:#?}", action_str); if let Some(action) = bookkeeper.actions.get(action_str) { bookkeeper.current_action = Some(action.clone()); - bookkeeper.do_action(&mut device); + bookkeeper.do_action(&mut stream); } } _ => {} @@ -179,7 +197,7 @@ fn inference_loop( // a bunch of indicators for sanity check let mode = if state.infer { "infer" } else { "eager" }; let listening_indicator = if state.listening { "" } else { "not " }; - log::info!("[{}] [{}listening] {}", mode, listening_indicator, sentence); + log::info!("[{}] [{}listening] {}", mode, listening_indicator, sentence,); if !state.listening && bookkeeper.word_to_trigger(&sentence) == Some(Mode::Wake) { state.listening = true; @@ -196,7 +214,7 @@ fn inference_loop( continue; } - if bookkeeper.word_to_action(&sentence, &mut device) { + if bookkeeper.word_to_action(&sentence, &mut stream) { state.already_commanded = true; } state.length = sentence.len(); @@ -215,9 +233,12 @@ fn main() -> Result<()> { init_april_api(1); // Initialize April ASR. Required to load a Model. - let device = VirtualDevice::default() - .map_err(|e| anyhow!("failed to create global uinput virtual device: {e}"))?; - + let key = if let Some(key) = args().into_iter().skip(1).next() { + let key_bytes = hex::decode(key.as_bytes())?; + Some(Key::::from_slice(&key_bytes).clone()) + } else { + None + }; let conf: config::RawConfig = { let reader = File::open("config.yml")?; serde_yaml::from_reader(reader)? @@ -277,9 +298,20 @@ fn main() -> Result<()> { current_action: None, }; - thread::spawn(move || { - inference_loop(VirtualInput(device), state, bookkeeper, session_rx, bert) - }); + let socket = match (UnixStream::connect("/run/tempest.socket"), key) { + (Ok(stream), Some(key)) => Some(AuthenticatedUnixStream { stream, key }), + (Err(e), Some(_)) => { + error!("failed to connect to the daemon socket: {e}"); + error!("bindings to keyboard shortcuts require connection to the daemon, they will not work for this session."); + None + } + _ => { + error!("token supplied to connect to the daemon is either nonexistent or incorrect"); + None + } + }; + + thread::spawn(move || inference_loop(socket, state, bookkeeper, session_rx, bert)); stream.play()?; diff --git a/src/state.rs b/client/src/state.rs similarity index 100% rename from src/state.rs rename to client/src/state.rs diff --git a/config.yml b/config.yml index 45c47f0..f42049b 100644 --- a/config.yml +++ b/config.yml @@ -6,6 +6,8 @@ infer_phrase: tempest listen actions: - phrase: focus up keys: [super, dot] + - phrase: krunner + keys: [leftalt, space] - phrase: focus down keys: [super, comma] - phrase: stack windows diff --git a/daemon/Cargo.toml b/daemon/Cargo.toml new file mode 100644 index 0000000..042297b --- /dev/null +++ b/daemon/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "tempest-daemon" +version = "0.2.2" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +aes-gcm = "0.10.3" +anyhow = "1.0.80" +env_logger = "0.11.3" +hex = "0.4.3" +log = "0.4.20" +mouse-keyboard-input = "0.4.1" +serde = { version = "1.0.197", features = ["derive"] } +serde_yaml = "0.9.34" diff --git a/src/config.rs b/daemon/src/config.rs similarity index 90% rename from src/config.rs rename to daemon/src/config.rs index aee0f80..dc1b28e 100644 --- a/src/config.rs +++ b/daemon/src/config.rs @@ -1,7 +1,6 @@ use mouse_keyboard_input::key_codes::*; use serde::Deserialize; use std::collections::BTreeMap; -use trie_rs::TrieBuilder; #[derive(Deserialize)] pub struct RawConfig { @@ -14,7 +13,7 @@ pub struct RawConfig { pub ollama_endpoint: String, } -#[derive(Deserialize, Clone)] +#[derive(Deserialize, Debug, Clone)] pub enum Action { Keys(Vec), Command(Vec), @@ -47,15 +46,11 @@ pub struct RawBinding { action: RawAction, } +#[derive(Debug)] pub struct Config { pub model_path: String, pub actions: BTreeMap, - pub word_trie: trie_rs::Trie, - pub keys: Vec, - pub abstract_triggers: trie_rs::Trie, pub modes: BTreeMap, - pub ollama_model: String, - pub ollama_endpoint: String, } fn decode_key(key: S) -> u16 @@ -329,7 +324,7 @@ where } } -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Mode { Wake, Rest, @@ -342,28 +337,11 @@ impl From for Config { let wake_phrase = value.wake_phrase.to_lowercase(); let rest_phrase = value.rest_phrase.to_lowercase(); let infer_phrase = value.infer_phrase.to_lowercase(); - let mut trie_builder = TrieBuilder::new(); - for phrase in value.actions.iter().map(|b| b.phrase.to_lowercase()) { - trie_builder.push(phrase); - } - let word_trie = trie_builder.build(); - let keys = value - .actions - .iter() - .map(|b| b.phrase.to_lowercase()) - .collect(); let actions = value .actions .into_iter() .map(|b| (b.phrase.to_lowercase(), b.action.into())) .collect(); - - let mut trie_builder = TrieBuilder::new(); - trie_builder.push(wake_phrase.clone()); - trie_builder.push(rest_phrase.clone()); - trie_builder.push(infer_phrase.clone()); - - let abstract_triggers = trie_builder.build(); let modes = [ (wake_phrase, Mode::Wake), (rest_phrase, Mode::Rest), @@ -374,13 +352,8 @@ impl From for Config { Self { model_path: value.model_path, - abstract_triggers, modes, - keys, actions, - word_trie, - ollama_model: value.ollama_model, - ollama_endpoint: value.ollama_endpoint, } } } diff --git a/daemon/src/main.rs b/daemon/src/main.rs new file mode 100644 index 0000000..2e03e03 --- /dev/null +++ b/daemon/src/main.rs @@ -0,0 +1,70 @@ +use aes_gcm::{ + aead::{Aead, KeyInit, Nonce, OsRng}, + Aes256Gcm, +}; +use anyhow::Result; +use std::fs::{self, File, Permissions}; +use std::io::{prelude::*, BufReader}; +use std::os::unix::fs::PermissionsExt; +use std::os::unix::net::UnixListener; +use virtualdevice::VirtualInput; + +mod config; +mod virtualdevice; + +fn main() -> Result<()> { + env_logger::init(); + let socket_path = "/run/tempest.socket"; + // Remove the socket if it exists + let _ = fs::remove_file(socket_path); + let listener = UnixListener::bind(socket_path)?; + fs::set_permissions(socket_path, Permissions::from_mode(0o622))?; + + let key = Aes256Gcm::generate_key(OsRng); + let cipher = Aes256Gcm::new(&key); + println!("please use this token to authenticate with the daemon"); + println!("{}", hex::encode(key)); + + let conf: config::RawConfig = { + let reader = File::open("config.yml")?; + serde_yaml::from_reader(reader)? + }; + let conf: config::Config = conf.into(); + let mut device = VirtualInput::new()?; + + loop { + log::info!("listening for connections"); + match listener.accept() { + Ok((socket, addr)) => { + log::info!("Got a client: {:?} - {:?}", socket.peer_addr(), addr); + let mut reader = BufReader::new(&socket); + let mut response = String::new(); + while let Ok(bytes_read) = reader.read_line(&mut response) { + if bytes_read == 0 { + log::info!("Got an end of stream"); + break; + } + let bytes = hex::decode(&response.trim())?; + let nonce_bytes = &bytes[..12]; + let ciphertext = &bytes[12..]; + + log::debug!("nonce: {nonce_bytes:?}"); + log::debug!("ciphertext: {ciphertext:?}"); + let nonce = Nonce::::clone_from_slice(nonce_bytes); + let Ok(resp) = cipher.decrypt(&nonce, ciphertext.as_ref()) else { + log::error!("failed to decrypt ciphertext sent by client"); + continue; + }; + let resp = std::str::from_utf8(&resp)?; + if let Some(config::Action::Keys(keys)) = conf.actions.get(resp) { + device.key_chord(keys.as_slice()); + } + response = String::new(); + } + } + Err(e) => log::error!("accept function failed: {:?}", e), + } + } + + Ok(()) +} diff --git a/daemon/src/virtualdevice.rs b/daemon/src/virtualdevice.rs new file mode 100644 index 0000000..7476a3f --- /dev/null +++ b/daemon/src/virtualdevice.rs @@ -0,0 +1,24 @@ +use anyhow::{anyhow, Result}; +use log::error; +use mouse_keyboard_input::VirtualDevice; +pub struct VirtualInput(VirtualDevice); + +impl VirtualInput { + pub fn new() -> Result { + let device = VirtualDevice::default() + .map_err(|e| anyhow!("failed to create global uinput virtual device: {e}"))?; + Ok(Self(device)) + } + pub fn key_chord(&mut self, keys: &[u16]) { + for &key in keys { + if let Err(e) = self.0.press(key) { + error!("failed to press key {key}: {e}"); + } + } + for &key in keys.iter().rev() { + if let Err(e) = self.0.release(key) { + error!("failed to release key {key}: {e}"); + } + } + } +}