diff --git a/app/src/main/java/com/litongjava/whisper/cpp/android/java/bean/WhisperSegment.java b/app/src/main/java/com/litongjava/whisper/cpp/android/java/bean/WhisperSegment.java new file mode 100644 index 0000000..fd84355 --- /dev/null +++ b/app/src/main/java/com/litongjava/whisper/cpp/android/java/bean/WhisperSegment.java @@ -0,0 +1,47 @@ +package com.litongjava.whisper.cpp.android.java.bean; + +/** + * Created by litonglinux@qq.com on 10/21/2023_7:48 AM + */ +public class WhisperSegment { + private long start, end; + private String sentence; + + public WhisperSegment() { + } + + public WhisperSegment(long start, long end, String sentence) { + this.start = start; + this.end = end; + this.sentence = sentence; + } + + public long getStart() { + return start; + } + + public long getEnd() { + return end; + } + + public String getSentence() { + return sentence; + } + + public void setStart(long start) { + this.start = start; + } + + public void setEnd(long end) { + this.end = end; + } + + public void setSentence(String sentence) { + this.sentence = sentence; + } + + @Override + public String toString() { + return "["+start+" --> "+end+"]:"+sentence; + } +} diff --git a/app/src/main/java/com/litongjava/whisper/cpp/android/java/demo/app/App.java b/app/src/main/java/com/litongjava/whisper/cpp/android/java/demo/app/App.java new file mode 100644 index 0000000..3383f81 --- /dev/null +++ b/app/src/main/java/com/litongjava/whisper/cpp/android/java/demo/app/App.java @@ -0,0 +1,13 @@ +package com.litongjava.whisper.cpp.android.java.demo.app; + +import android.app.Application; + +import com.blankj.utilcode.util.Utils; + +public class App extends Application { + @Override + public void onCreate() { + super.onCreate(); + Utils.init(this); + } +} diff --git a/app/src/main/java/com/litongjava/whisper/cpp/android/java/services/WhisperService.java b/app/src/main/java/com/litongjava/whisper/cpp/android/java/services/WhisperService.java index c9c256c..0f06f49 100644 --- a/app/src/main/java/com/litongjava/whisper/cpp/android/java/services/WhisperService.java +++ b/app/src/main/java/com/litongjava/whisper/cpp/android/java/services/WhisperService.java @@ -6,101 +6,89 @@ import androidx.annotation.RequiresApi; +import com.litongjava.whisper.cpp.android.java.bean.WhisperSegment; import com.litongjava.whisper.cpp.android.java.media.WaveEncoder; +import com.litongjava.whisper.cpp.android.java.single.LocalWhisper; import com.litongjava.whisper.cpp.android.java.utils.AssetUtils; -import com.whispercppdemo.whisper.WhisperContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.util.List; import java.util.concurrent.ExecutionException; public class WhisperService { private Logger log = LoggerFactory.getLogger(this.getClass()); - private WhisperContext whisperContext; + + private final Object lock = new Object(); @RequiresApi(api = Build.VERSION_CODES.O) public void loadModel(Context context, TextView tv) { + String modelFilePath = LocalWhisper.modelFilePath; + String msg = "load model from :" + modelFilePath + "\n"; + outputMsg(tv, msg); - File filesDir = context.getFilesDir(); - String modelFilePath = "models/ggml-tiny.bin"; - File modelFile = AssetUtils.copyFileIfNotExists(context, filesDir, modelFilePath); - modelFilePath = modelFile.getAbsolutePath(); + long start = System.currentTimeMillis(); + LocalWhisper.INSTANCE.init(); + long end = System.currentTimeMillis(); + msg = "model load successful:" + (end - start) + "ms"; + outputMsg(tv, msg); - String msg = "load model from :" + modelFilePath + "\n"; - log.info(msg); - tv.append(msg); - if (whisperContext == null) { - long start = System.currentTimeMillis(); - whisperContext = WhisperContext.createContextFromFile(modelFilePath); -// AopManager.me().addSingletonObject(whisperContext); - long end = System.currentTimeMillis(); - msg = "model load successful:" + (end - start) + "ms\n"; - log.info(msg); - tv.append(msg); - } else { - msg = "model loaded\n"; - log.info(msg); - tv.append(msg); - } } public void transcribeSample(Context context, TextView tv) { + String msg = ""; + long start = System.currentTimeMillis(); String sampleFilePath = "samples/jfk.wav"; File filesDir = context.getFilesDir(); File sampleFile = AssetUtils.copyFileIfNotExists(context, filesDir, sampleFilePath); - log.info("transcribe file from :{}", sampleFile.getAbsolutePath()); + long end = System.currentTimeMillis(); + msg = "copy file:" + (end - start) + "ms"; + outputMsg(tv, msg); + + msg = "transcribe file from :" + sampleFile.getAbsolutePath(); + outputMsg(tv, msg); + + start = System.currentTimeMillis(); float[] audioData = new float[0]; // 读取音频样本 try { audioData = WaveEncoder.decodeWaveFile(sampleFile); } catch (IOException e) { e.printStackTrace(); + return; } + end = System.currentTimeMillis(); + msg = "decode wave file:" + (end - start) + "ms"; + outputMsg(tv, msg); - String transcription = null; // 转录音频数据 - - String msg = ""; + start = System.currentTimeMillis(); + List transcription = null; try { - if (whisperContext == null) { - msg = "please load model or wait model loaded"; - log.info(msg); - - } else { - long start = System.currentTimeMillis(); - transcription = whisperContext.transcribeData(audioData); - long end = System.currentTimeMillis(); - msg = "Transcript successful:" + (end - start) + "ms"; - log.info(msg); - tv.append(msg + "\n"); - - msg = "Transcription:" + transcription; - log.info(msg); - tv.append(msg + "\n"); - } - + //transcription = LocalWhisper.INSTANCE.transcribeData(audioData); + transcription = LocalWhisper.INSTANCE.transcribeDataWithTime(audioData); } catch (ExecutionException e) { e.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } + end = System.currentTimeMillis(); + msg = "Transcript successful:" + (end - start) + "ms"; + outputMsg(tv, msg); + msg = "Transcription:" + transcription.toString(); + outputMsg(tv, msg); + } + private void outputMsg(TextView tv, String msg) { + tv.append(msg + "\n"); + log.info(msg); } @RequiresApi(api = Build.VERSION_CODES.O) public void release() { - if (whisperContext != null) { - try { - whisperContext.release(); - } catch (ExecutionException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - whisperContext = null; - } + //noting to do } -} \ No newline at end of file +} diff --git a/app/src/main/java/com/litongjava/whisper/cpp/android/java/single/LocalWhisper.java b/app/src/main/java/com/litongjava/whisper/cpp/android/java/single/LocalWhisper.java new file mode 100644 index 0000000..ba41879 --- /dev/null +++ b/app/src/main/java/com/litongjava/whisper/cpp/android/java/single/LocalWhisper.java @@ -0,0 +1,48 @@ +package com.litongjava.whisper.cpp.android.java.single; + +import android.app.Application; +import android.os.Build; + +import androidx.annotation.RequiresApi; + +import com.blankj.utilcode.util.Utils; +import com.litongjava.whisper.cpp.android.java.bean.WhisperSegment; +import com.litongjava.whisper.cpp.android.java.utils.AssetUtils; +import com.whispercppdemo.whisper.WhisperContext; + +import java.io.File; +import java.util.List; +import java.util.concurrent.ExecutionException; + +//import com.litongjava.whisper.android.java.bean.WhisperSegment; +//import com.litongjava.whisper.android.java.utils.AssetUtils; + +public enum LocalWhisper { + INSTANCE; + + public static final String modelFilePath = "models/ggml-tiny.bin"; + private WhisperContext whisperContext; + + @RequiresApi(api = Build.VERSION_CODES.O) + LocalWhisper() { + Application context = Utils.getApp(); + File filesDir = context.getFilesDir(); + File modelFile = AssetUtils.copyFileIfNotExists(context, filesDir, modelFilePath); + String realModelFilePath = modelFile.getAbsolutePath(); + whisperContext = WhisperContext.createContextFromFile(realModelFilePath); + } + + public synchronized String transcribeData(float[] data) throws ExecutionException, InterruptedException { + return whisperContext.transcribeData(data); + } + + public List transcribeDataWithTime(float[] audioData) throws ExecutionException, InterruptedException { + return whisperContext.transcribeDataWithTime(audioData); + } + + public void init() { + //noting to do.but init + } + + +} diff --git a/app/src/main/java/com/whispercppdemo/whisper/WhisperContext.java b/app/src/main/java/com/whispercppdemo/whisper/WhisperContext.java index 1f415b5..f3ff60f 100644 --- a/app/src/main/java/com/whispercppdemo/whisper/WhisperContext.java +++ b/app/src/main/java/com/whispercppdemo/whisper/WhisperContext.java @@ -6,7 +6,11 @@ import androidx.annotation.RequiresApi; +import com.litongjava.whisper.cpp.android.java.bean.WhisperSegment; + import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -33,17 +37,53 @@ public String call() throws Exception { } int numThreads = WhisperCpuConfig.getPreferredThreadCount(); Log.d(LOG_TAG, "Selecting " + numThreads + " threads"); - WhisperLib.fullTranscribe(ptr, numThreads, data); - int textCount = WhisperLib.getTextSegmentCount(ptr); + StringBuilder result = new StringBuilder(); - for (int i = 0; i < textCount; i++) { - result.append(WhisperLib.getTextSegment(ptr, i)); + synchronized (this) { + + WhisperLib.fullTranscribe(ptr, numThreads, data); + int textCount = WhisperLib.getTextSegmentCount(ptr); + for (int i = 0; i < textCount; i++) { + String sentence = WhisperLib.getTextSegment(ptr, i); + result.append(sentence); + } } return result.toString(); } }).get(); } + public List transcribeDataWithTime(float[] data) throws ExecutionException, InterruptedException { + return executorService.submit(new Callable>() { + @RequiresApi(api = Build.VERSION_CODES.O) + @Override + public List call() throws Exception { + if (ptr == 0L) { + throw new IllegalStateException(); + } + int numThreads = WhisperCpuConfig.getPreferredThreadCount(); + Log.d(LOG_TAG, "Selecting " + numThreads + " threads"); + + List segments = new ArrayList<>(); + synchronized (this) { +// StringBuilder result = new StringBuilder(); + WhisperLib.fullTranscribe(ptr, numThreads, data); + int textCount = WhisperLib.getTextSegmentCount(ptr); + for (int i = 0; i < textCount; i++) { + long start = WhisperLib.getTextSegmentT0(ptr, i); + String sentence = WhisperLib.getTextSegment(ptr, i); + long end = WhisperLib.getTextSegmentT1(ptr, i); +// result.append(); + segments.add(new WhisperSegment(start, end, sentence)); + + } +// return result.toString(); + } + return segments; + } + }).get(); + } + @RequiresApi(api = Build.VERSION_CODES.O) public String benchMemory(int nthreads) throws ExecutionException, InterruptedException { return executorService.submit(() -> WhisperLib.benchMemcpy(nthreads)).get(); diff --git a/app/src/main/java/com/whispercppdemo/whisper/WhisperLib.java b/app/src/main/java/com/whispercppdemo/whisper/WhisperLib.java index 7a38177..bf99cc7 100644 --- a/app/src/main/java/com/whispercppdemo/whisper/WhisperLib.java +++ b/app/src/main/java/com/whispercppdemo/whisper/WhisperLib.java @@ -6,41 +6,32 @@ import androidx.annotation.RequiresApi; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.InputStream; @RequiresApi(api = Build.VERSION_CODES.O) public class WhisperLib { private static final String LOG_TAG = "LibWhisper"; - private static Logger log = LoggerFactory.getLogger(WhisperLib.class); static { Log.d(LOG_TAG, "Primary ABI: " + Build.SUPPORTED_ABIS[0]); - log.info("Primary ABI: " + Build.SUPPORTED_ABIS[0]); boolean loadVfpv4 = false; boolean loadV8fp16 = false; - if (Utils.isArmEabiV7a()) { - String cpuInfo = Utils.cpuInfo(); + if (WhisperUtils.isArmEabiV7a()) { + String cpuInfo = WhisperUtils.cpuInfo(); if (cpuInfo != null) { Log.d(LOG_TAG, "CPU info: " + cpuInfo); - log.info("CPU info: " + cpuInfo); if (cpuInfo.contains("vfpv4")) { Log.d(LOG_TAG, "CPU supports vfpv4"); - log.info("CPU supports vfpv4"); loadVfpv4 = true; } } - } else if (Utils.isArmEabiV8a()) { - String cpuInfo = Utils.cpuInfo(); + } else if (WhisperUtils.isArmEabiV8a()) { + String cpuInfo = WhisperUtils.cpuInfo(); if (cpuInfo != null) { Log.d(LOG_TAG, "CPU info: " + cpuInfo); - log.info("CPU info: " + cpuInfo); if (cpuInfo.contains("fphp")) { Log.d(LOG_TAG, "CPU supports fp16 arithmetic"); - log.info("CPU supports fp16 arithmetic"); loadV8fp16 = true; } } @@ -48,15 +39,12 @@ public class WhisperLib { if (loadVfpv4) { Log.d(LOG_TAG, "Loading libwhisper_vfpv4.so"); - log.info("Loading libwhisper_vfpv4.so"); System.loadLibrary("whisper_vfpv4"); } else if (loadV8fp16) { Log.d(LOG_TAG, "Loading libwhisper_v8fp16_va.so"); - log.info("Loading libwhisper_v8fp16_va.so"); System.loadLibrary("whisper_v8fp16_va"); } else { Log.d(LOG_TAG, "Loading libwhisper.so"); - log.info("Loading libwhisper.so"); System.loadLibrary("whisper"); } } @@ -75,6 +63,10 @@ public class WhisperLib { public static native String getTextSegment(long contextPtr, int index); + public static native long getTextSegmentT0(long contextPtr, int index); + + public static native long getTextSegmentT1(long contextPtr, int index); + public static native String getSystemInfo(); public static native String benchMemcpy(int nthread); diff --git a/app/src/main/java/com/whispercppdemo/whisper/Utils.java b/app/src/main/java/com/whispercppdemo/whisper/WhisperUtils.java similarity index 96% rename from app/src/main/java/com/whispercppdemo/whisper/Utils.java rename to app/src/main/java/com/whispercppdemo/whisper/WhisperUtils.java index f825cb1..5781db5 100644 --- a/app/src/main/java/com/whispercppdemo/whisper/Utils.java +++ b/app/src/main/java/com/whispercppdemo/whisper/WhisperUtils.java @@ -8,7 +8,7 @@ import java.io.File; import java.nio.file.Path; -public class Utils { +public class WhisperUtils { private static final String LOG_TAG = "LibWhisper"; diff --git a/readme.md b/readme.md index 529a254..2b8c421 100644 --- a/readme.md +++ b/readme.md @@ -33,8 +33,12 @@ A sample Android app using java code and [whisper.cpp library](https://github.co │ │ │ │ │ │ │ ├── cpp │ │ │ │ │ │ │ │ ├── android │ │ │ │ │ │ │ │ │ ├── java +│ │ │ │ │ │ │ │ │ │ ├── bean +│ │ │ │ │ │ │ │ │ │ │ ├── WhisperSegment.java │ │ │ │ │ │ │ │ │ │ ├── demo │ │ │ │ │ │ │ │ │ │ │ ├── MainActivity.java +│ │ │ │ │ │ │ │ │ │ │ ├── app +│ │ │ │ │ │ │ │ │ │ │ │ ├── App.java │ │ │ │ │ │ │ │ │ │ │ ├── ui │ │ │ │ │ │ │ │ │ │ ├── media │ │ │ │ │ │ │ │ │ │ │ ├── WaveEncoder.java @@ -42,15 +46,17 @@ A sample Android app using java code and [whisper.cpp library](https://github.co │ │ │ │ │ │ │ │ │ │ │ ├── Recorder.java │ │ │ │ │ │ │ │ │ │ ├── services │ │ │ │ │ │ │ │ │ │ │ ├── WhisperService.java +│ │ │ │ │ │ │ │ │ │ ├── single +│ │ │ │ │ │ │ │ │ │ │ ├── LocalWhisper.java │ │ │ │ │ │ │ │ │ │ ├── utils │ │ │ │ │ │ │ │ │ │ │ ├── AssetUtils.java │ │ │ │ │ ├── whispercppdemo │ │ │ │ │ │ ├── whisper │ │ │ │ │ │ │ ├── CpuInfo.java -│ │ │ │ │ │ │ ├── Utils.java │ │ │ │ │ │ │ ├── WhisperContext.java │ │ │ │ │ │ │ ├── WhisperCpuConfig.java │ │ │ │ │ │ │ ├── WhisperLib.java +│ │ │ │ │ │ │ ├── WhisperUtils.java │ │ │ ├── jniLibs │ │ │ │ ├── arm64-v8a │ │ │ │ │ ├── libwhisper.so @@ -133,9 +139,7 @@ A sample Android app using java code and [whisper.cpp library](https://github.co ├── gradlew ├── gradlew.bat ├── local.properties -├── readmd.md -├── readmd_files -│ ├── 1.jpg +├── readme.md ├── readme_cn.md ├── settings.gradle ``` @@ -201,837 +205,4 @@ Modify app/build.gradle to add jniLibs to sourceSets implementation 'com.litongjava:android-view-inject:1.0' implementation 'com.litongjava:jfinal-aop:1.0.1' implementation 'com.litongjava:litongjava-android-utils:1.0.0' -``` -### 1.3.4.Whisper Cpp Java code -#### 1.3.4.1.CpuInfo -``` -package com.whispercppdemo.whisper; - -import android.os.Build; -import android.util.Log; - -import androidx.annotation.RequiresApi; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class CpuInfo { - private static final String LOG_TAG = "WhisperCpuConfig"; - - private List lines; - - public CpuInfo(List lines) { - this.lines = lines; - } - - @RequiresApi(api = Build.VERSION_CODES.N) - public int getHighPerfCpuCount0() { - try { - return getHighPerfCpuCountByFrequencies(); - } catch (Exception e) { - Log.d(LOG_TAG, "Couldn't read CPU frequencies", e); - return getHighPerfCpuCountByVariant(); - } - } - - @RequiresApi(api = Build.VERSION_CODES.N) - private int getHighPerfCpuCountByFrequencies() { - List frequencies = getCpuValues("processor", line -> { - try { - return getMaxCpuFrequency(Integer.parseInt(line.trim())); - } catch (IOException e) { - e.printStackTrace(); - } - return 0; - } - ); - Log.d(LOG_TAG, "Binned cpu frequencies (frequency, count): " + binnedValues(frequencies)); - return countDroppingMin(frequencies); - } - - @RequiresApi(api = Build.VERSION_CODES.N) - private int getHighPerfCpuCountByVariant() { - List variants = getCpuValues("CPU variant", line -> Integer.parseInt(line.trim().substring(line.indexOf("0x") + 2), 16)); - Log.d(LOG_TAG, "Binned cpu variants (variant, count): " + binnedValues(variants)); - return countKeepingMin(variants); - } - - @RequiresApi(api = Build.VERSION_CODES.N) - private Map binnedValues(List values) { - Map countMap = new HashMap<>(); - for (int value : values) { - countMap.put(value, countMap.getOrDefault(value, 0) + 1); - } - return countMap; - } - - @RequiresApi(api = Build.VERSION_CODES.N) - private List getCpuValues(String property, Mapper mapper) { - List values = new ArrayList<>(); - for (String line : lines) { - if (line.startsWith(property)) { - values.add(mapper.map(line.substring(line.indexOf(':') + 1))); - } - } - values.sort(Integer::compareTo); - return values; - } - - @RequiresApi(api = Build.VERSION_CODES.N) - private int countDroppingMin(List values) { - int min = values.stream().mapToInt(i -> i).min().orElse(Integer.MAX_VALUE); - return (int) values.stream().filter(value -> value > min).count(); - } - - @RequiresApi(api = Build.VERSION_CODES.N) - private int countKeepingMin(List values) { - int min = values.stream().mapToInt(i -> i).min().orElse(Integer.MAX_VALUE); - return (int) values.stream().filter(value -> value.equals(min)).count(); - } - - @RequiresApi(api = Build.VERSION_CODES.N) - public static int getHighPerfCpuCount() { - try { - return readCpuInfo().getHighPerfCpuCount0(); - } catch (Exception e) { - Log.d(LOG_TAG, "Couldn't read CPU info", e); - return Math.max(Runtime.getRuntime().availableProcessors() - 4, 0); - } - } - - private static CpuInfo readCpuInfo() throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader("/proc/cpuinfo"))) { - List lines = new ArrayList<>(); - String line; - while ((line = reader.readLine()) != null) { - lines.add(line); - } - return new CpuInfo(lines); - } - } - - private static int getMaxCpuFrequency(int cpuIndex) throws IOException { - String path = "/sys/devices/system/cpu/cpu" + cpuIndex + "/cpufreq/cpuinfo_max_freq"; - try (BufferedReader reader = new BufferedReader(new FileReader(path))) { - return Integer.parseInt(reader.readLine()); - } - } - - private interface Mapper { - int map(String line); - } -} -``` -#### 1.3.4.2.Utils -``` -package com.whispercppdemo.whisper; - -import android.os.Build; -import android.util.Log; - -import androidx.annotation.RequiresApi; - -import java.io.File; -import java.nio.file.Path; - -public class Utils { - private static final String LOG_TAG = "LibWhisper"; - - - public static boolean isArmEabiV7a() { - return Build.SUPPORTED_ABIS[0].equals("armeabi-v7a"); - } - - public static boolean isArmEabiV8a() { - return Build.SUPPORTED_ABIS[0].equals("arm64-v8a"); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public static String cpuInfo() { - try { - Path path = new File("/proc/cpuinfo").toPath(); - return new String(java.nio.file.Files.readAllBytes(path)); - } catch (Exception e) { - Log.w(LOG_TAG, "Couldn't read /proc/cpuinfo", e); - return null; - } - - } -} -#### 1.3.4.3.WhisperContext -package com.whispercppdemo.whisper; - -import android.content.res.AssetManager; -import android.os.Build; -import android.util.Log; - -import androidx.annotation.RequiresApi; - -import java.io.InputStream; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class WhisperContext { - - private static final String LOG_TAG = "LibWhisper"; - private long ptr; - private final ExecutorService executorService; - - private WhisperContext(long ptr) { - this.ptr = ptr; - this.executorService = Executors.newSingleThreadExecutor(); - } - - public String transcribeData(float[] data) throws ExecutionException, InterruptedException { - return executorService.submit(new Callable() { - @RequiresApi(api = Build.VERSION_CODES.O) - @Override - public String call() throws Exception { - if (ptr == 0L) { - throw new IllegalStateException(); - } - int numThreads = WhisperCpuConfig.getPreferredThreadCount(); - Log.d(LOG_TAG, "Selecting " + numThreads + " threads"); - WhisperLib.fullTranscribe(ptr, numThreads, data); - int textCount = WhisperLib.getTextSegmentCount(ptr); - StringBuilder result = new StringBuilder(); - for (int i = 0; i < textCount; i++) { - result.append(WhisperLib.getTextSegment(ptr, i)); - } - return result.toString(); - } - }).get(); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public String benchMemory(int nthreads) throws ExecutionException, InterruptedException { - return executorService.submit(() -> WhisperLib.benchMemcpy(nthreads)).get(); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public String benchGgmlMulMat(int nthreads) throws ExecutionException, InterruptedException { - return executorService.submit(() -> WhisperLib.benchGgmlMulMat(nthreads)).get(); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public void release() throws ExecutionException, InterruptedException { - executorService.submit(() -> { - if (ptr != 0L) { - WhisperLib.freeContext(ptr); - ptr = 0; - } - }).get(); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public static WhisperContext createContextFromFile(String filePath) { - long ptr = WhisperLib.initContext(filePath); - if (ptr == 0L) { - throw new RuntimeException("Couldn't create context with path " + filePath); - } - return new WhisperContext(ptr); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public static WhisperContext createContextFromInputStream(InputStream stream) { - long ptr = WhisperLib.initContextFromInputStream(stream); - if (ptr == 0L) { - throw new RuntimeException("Couldn't create context from input stream"); - } - return new WhisperContext(ptr); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public static WhisperContext createContextFromAsset(AssetManager assetManager, String assetPath) { - long ptr = WhisperLib.initContextFromAsset(assetManager, assetPath); - if (ptr == 0L) { - throw new RuntimeException("Couldn't create context from asset " + assetPath); - } - return new WhisperContext(ptr); - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public static String getSystemInfo() { - return WhisperLib.getSystemInfo(); - } -} -``` -#### 1.3.4.4.WhisperCpuConfig -``` -package com.whispercppdemo.whisper; - -import android.os.Build; - -import androidx.annotation.RequiresApi; - -public class WhisperCpuConfig { - @RequiresApi(api = Build.VERSION_CODES.N) - public static int getPreferredThreadCount() { - return Math.max(CpuInfo.getHighPerfCpuCount(), 2); - } -} -``` -#### 1.3.4.5.WhisperLib -``` -package com.whispercppdemo.whisper; - -import android.content.res.AssetManager; -import android.os.Build; -import android.util.Log; - -import androidx.annotation.RequiresApi; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.InputStream; - -@RequiresApi(api = Build.VERSION_CODES.O) -public class WhisperLib { - private static final String LOG_TAG = "LibWhisper"; - private static Logger log = LoggerFactory.getLogger(WhisperLib.class); - - static { - - Log.d(LOG_TAG, "Primary ABI: " + Build.SUPPORTED_ABIS[0]); - log.info("Primary ABI: " + Build.SUPPORTED_ABIS[0]); - boolean loadVfpv4 = false; - boolean loadV8fp16 = false; - if (Utils.isArmEabiV7a()) { - String cpuInfo = Utils.cpuInfo(); - if (cpuInfo != null) { - Log.d(LOG_TAG, "CPU info: " + cpuInfo); - log.info("CPU info: " + cpuInfo); - if (cpuInfo.contains("vfpv4")) { - Log.d(LOG_TAG, "CPU supports vfpv4"); - log.info("CPU supports vfpv4"); - loadVfpv4 = true; - } - } - } else if (Utils.isArmEabiV8a()) { - String cpuInfo = Utils.cpuInfo(); - if (cpuInfo != null) { - Log.d(LOG_TAG, "CPU info: " + cpuInfo); - log.info("CPU info: " + cpuInfo); - if (cpuInfo.contains("fphp")) { - Log.d(LOG_TAG, "CPU supports fp16 arithmetic"); - log.info("CPU supports fp16 arithmetic"); - loadV8fp16 = true; - } - } - } - - if (loadVfpv4) { - Log.d(LOG_TAG, "Loading libwhisper_vfpv4.so"); - log.info("Loading libwhisper_vfpv4.so"); - System.loadLibrary("whisper_vfpv4"); - } else if (loadV8fp16) { - Log.d(LOG_TAG, "Loading libwhisper_v8fp16_va.so"); - log.info("Loading libwhisper_v8fp16_va.so"); - System.loadLibrary("whisper_v8fp16_va"); - } else { - Log.d(LOG_TAG, "Loading libwhisper.so"); - log.info("Loading libwhisper.so"); - System.loadLibrary("whisper"); - } - } - - public static native long initContextFromInputStream(InputStream inputStream); - - public static native long initContextFromAsset(AssetManager assetManager, String assetPath); - - public static native long initContext(String modelPath); - - public static native void freeContext(long contextPtr); - - public static native void fullTranscribe(long contextPtr, int numThreads, float[] audioData); - - public static native int getTextSegmentCount(long contextPtr); - - public static native String getTextSegment(long contextPtr, int index); - - public static native String getSystemInfo(); - - public static native String benchMemcpy(int nthread); - - public static native String benchGgmlMulMat(int nthread); -} -``` -### 1.3.5.添加工具类 -#### 1.3.5.1.AssetUtils -``` -package com.litongjava.whisper.cpp.android.java.utils; - -import android.content.Context; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; - -public class AssetUtils { - private static Logger log = LoggerFactory.getLogger(AssetUtils.class); - - public static File copyFileIfNotExists(Context context, File distDir, String filename) { - File dstFile = new File(distDir, filename); - if (dstFile.exists()) { - return dstFile; - } else { - File parentFile = dstFile.getParentFile(); - log.info("parentFile:{}", parentFile); - if (!parentFile.exists()) { - parentFile.mkdirs(); - } - AssetUtils.copyFileFromAssets(context, filename, dstFile); - } - return dstFile; - } - - public static void copyDirectoryFromAssets(Context appCtx, String srcDir, String dstDir) { - if (srcDir.isEmpty() || dstDir.isEmpty()) { - return; - } - try { - if (!new File(dstDir).exists()) { - new File(dstDir).mkdirs(); - } - for (String fileName : appCtx.getAssets().list(srcDir)) { - String srcSubPath = srcDir + File.separator + fileName; - String dstSubPath = dstDir + File.separator + fileName; - if (new File(srcSubPath).isDirectory()) { - copyDirectoryFromAssets(appCtx, srcSubPath, dstSubPath); - } else { - copyFileFromAssets(appCtx, srcSubPath, dstSubPath); - } - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - public static void copyFileFromAssets(Context appCtx, String srcPath, String dstPath) { - File dstFile = new File(dstPath); - copyFileFromAssets(appCtx, srcPath, dstFile); - } - - public static void copyFileFromAssets(Context appCtx, String srcPath, File dstFile) { - if (srcPath.isEmpty()) { - return; - } - InputStream is = null; - OutputStream os = null; - try { - is = new BufferedInputStream(appCtx.getAssets().open(srcPath)); - - os = new BufferedOutputStream(new FileOutputStream(dstFile)); - byte[] buffer = new byte[1024]; - int length = 0; - while ((length = is.read(buffer)) != -1) { - os.write(buffer, 0, length); - } - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - os.close(); - is.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - } -} -``` -### 1.3.5.2.WaveEncoder -``` -package com.litongjava.whisper.cpp.android.java.media; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.ShortBuffer; - -public class WaveEncoder { - - public static float[] decodeWaveFile(File file) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (FileInputStream fis = new FileInputStream(file)) { - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = fis.read(buffer)) != -1) { - baos.write(buffer, 0, bytesRead); - } - } - ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray()); - byteBuffer.order(ByteOrder.LITTLE_ENDIAN); - - int channel = byteBuffer.getShort(22); - byteBuffer.position(44); - - ShortBuffer shortBuffer = byteBuffer.asShortBuffer(); - short[] shortArray = new short[shortBuffer.limit()]; - shortBuffer.get(shortArray); - - float[] output = new float[shortArray.length / channel]; - - for (int index = 0; index < output.length; index++) { - if (channel == 1) { - output[index] = Math.max(-1f, Math.min(1f, shortArray[index] / 32767.0f)); - } else { - output[index] = Math.max(-1f, Math.min(1f, (shortArray[2 * index] + shortArray[2 * index + 1]) / 32767.0f / 2.0f)); - } - } - return output; - } - - public static void encodeWaveFile(File file, short[] data) throws IOException { - try (FileOutputStream fos = new FileOutputStream(file)) { - fos.write(headerBytes(data.length * 2)); - - ByteBuffer buffer = ByteBuffer.allocate(data.length * 2); - buffer.order(ByteOrder.LITTLE_ENDIAN); - buffer.asShortBuffer().put(data); - - byte[] bytes = new byte[buffer.limit()]; - buffer.get(bytes); - - fos.write(bytes); - } - } - - private static byte[] headerBytes(int totalLength) { - if (totalLength < 44) - throw new IllegalArgumentException("Total length must be at least 44 bytes"); - - ByteBuffer buffer = ByteBuffer.allocate(44); - buffer.order(ByteOrder.LITTLE_ENDIAN); - - buffer.put((byte) 'R'); - buffer.put((byte) 'I'); - buffer.put((byte) 'F'); - buffer.put((byte) 'F'); - - buffer.putInt(totalLength - 8); - - buffer.put((byte) 'W'); - buffer.put((byte) 'A'); - buffer.put((byte) 'V'); - buffer.put((byte) 'E'); - - buffer.put((byte) 'f'); - buffer.put((byte) 'm'); - buffer.put((byte) 't'); - buffer.put((byte) ' '); - - buffer.putInt(16); - buffer.putShort((short) 1); - buffer.putShort((short) 1); - buffer.putInt(16000); - buffer.putInt(32000); - buffer.putShort((short) 2); - buffer.putShort((short) 16); - - buffer.put((byte) 'd'); - buffer.put((byte) 'a'); - buffer.put((byte) 't'); - buffer.put((byte) 'a'); - - buffer.putInt(totalLength - 44); - buffer.position(0); - - byte[] bytes = new byte[buffer.limit()]; - buffer.get(bytes); - - return bytes; - } -} -``` -### 1.3.6.transcript wav file -#### 1.3.6.1.WhisperService -``` -//load model -whisperContext = WhisperContext.createContextFromFile(modelPath); -// transcript -transcription = whisperContext.transcribeData(audioData); -``` - -``` -package com.litongjava.whisper.cpp.android.java.services; - -import android.content.Context; -import android.os.Build; -import android.widget.TextView; - -import androidx.annotation.RequiresApi; - -import com.litongjava.whisper.cpp.android.java.media.WaveEncoder; -import com.litongjava.whisper.cpp.android.java.utils.AssetUtils; -import com.whispercppdemo.whisper.WhisperContext; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.ExecutionException; - -public class WhisperService { - private Logger log = LoggerFactory.getLogger(this.getClass()); - private WhisperContext whisperContext; - - @RequiresApi(api = Build.VERSION_CODES.O) - public void loadModel(Context context, TextView tv) { - - File filesDir = context.getFilesDir(); - String modelFilePath = "models/ggml-tiny.bin"; - File modelFile = AssetUtils.copyFileIfNotExists(context, filesDir, modelFilePath); - modelFilePath = modelFile.getAbsolutePath(); - - String msg = "load model from :" + modelFilePath + "\n"; - log.info(msg); - tv.append(msg); - if (whisperContext == null) { - long start = System.currentTimeMillis(); - whisperContext = WhisperContext.createContextFromFile(modelFilePath); -// AopManager.me().addSingletonObject(whisperContext); - long end = System.currentTimeMillis(); - msg = "model load successful:" + (end - start) + "ms\n"; - log.info(msg); - tv.append(msg); - } else { - msg = "model loaded\n"; - log.info(msg); - tv.append(msg); - } - } - - public void transcribeSample(Context context, TextView tv) { - String sampleFilePath = "samples/jfk.wav"; - File filesDir = context.getFilesDir(); - File sampleFile = AssetUtils.copyFileIfNotExists(context, filesDir, sampleFilePath); - log.info("transcribe file from :{}", sampleFile.getAbsolutePath()); - float[] audioData = new float[0]; // 读取音频样本 - try { - audioData = WaveEncoder.decodeWaveFile(sampleFile); - } catch (IOException e) { - e.printStackTrace(); - } - - String transcription = null; // 转录音频数据 - - String msg = ""; - try { - if (whisperContext == null) { - msg = "please load model or wait model loaded"; - log.info(msg); - - } else { - long start = System.currentTimeMillis(); - transcription = whisperContext.transcribeData(audioData); - long end = System.currentTimeMillis(); - msg = "Transcript successful:" + (end - start) + "ms"; - log.info(msg); - tv.append(msg + "\n"); - - msg = "Transcription:" + transcription; - log.info(msg); - tv.append(msg + "\n"); - } - - - } catch (ExecutionException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - - - } - - @RequiresApi(api = Build.VERSION_CODES.O) - public void release() { - if (whisperContext != null) { - try { - whisperContext.release(); - } catch (ExecutionException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - whisperContext = null; - } - } -} -``` - -#### 1.3.6.2 MainActivity -activity_main.xml -``` - - - - - -