From d30e4eda705950cb280374804292a1e748542f15 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 2 Sep 2023 10:03:32 +0200 Subject: [PATCH 01/12] add definition --- package-lock.json | 1 + packages/selfie-segmentation/README.md | 41 +++++++++++--- .../selfie-segmentation/src/definitions.ts | 56 ++++++++++++++++++- packages/selfie-segmentation/src/web.ts | 22 ++++++-- 4 files changed, 107 insertions(+), 13 deletions(-) diff --git a/package-lock.json b/package-lock.json index f5d82ee..1c182e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6215,6 +6215,7 @@ } }, "packages/selfie-segmentation": { + "name": "@capacitor-mlkit/selfie-segmentation", "version": "5.1.0", "funding": [ { diff --git a/packages/selfie-segmentation/README.md b/packages/selfie-segmentation/README.md index fad249d..ab35683 100644 --- a/packages/selfie-segmentation/README.md +++ b/packages/selfie-segmentation/README.md @@ -37,27 +37,54 @@ const echo = async () => { -* [`echo(...)`](#echo) +* [`processImage(...)`](#processimage) +* [Interfaces](#interfaces) -### echo(...) +### processImage(...) ```typescript -echo(options: { value: string; }) => Promise<{ value: string; }> +processImage(options: ProcessImageOptions) => Promise ``` -| Param | Type | -| ------------- | ------------------------------- | -| **`options`** | { value: string; } | +Performs segmentation on an input image. -**Returns:** Promise<{ value: string; }> +Only available on Android and iOS. + +| Param | Type | +| ------------- | ------------------------------------------------------------------- | +| **`options`** | ProcessImageOptions | + +**Returns:** Promise<ProcessImageResult> + +**Since:** 5.2.0 -------------------- + +### Interfaces + + +#### ProcessImageResult + +| Prop | Type | Description | Since | +| ------------ | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | +| **`mask`** | number[] | Returns a mask that indicates the foreground and background segmentation. This mask’s dimensions could vary, depending on whether a raw size mask is requested via options. | 5.2.0 | +| **`width`** | number | Returns the width of the mask. | 5.2.0 | +| **`height`** | number | Returns the height of the mask. | 5.2.0 | + + +#### ProcessImageOptions + +| Prop | Type | Description | Since | +| ----------------- | -------------------- | ----------------------------------------------------------------------------------- | ----- | +| **`path`** | string | The local path to the image file. | 5.2.0 | +| **`rawSizeMask`** | boolean | Asks the segmenter to return the raw size mask which matches the model output size. | 5.2.0 | + ## Terms & Privacy diff --git a/packages/selfie-segmentation/src/definitions.ts b/packages/selfie-segmentation/src/definitions.ts index e3f7b6a..649d323 100644 --- a/packages/selfie-segmentation/src/definitions.ts +++ b/packages/selfie-segmentation/src/definitions.ts @@ -1,3 +1,57 @@ export interface SelfieSegmentationPlugin { - echo(options: { value: string }): Promise<{ value: string }>; + /** + * Performs segmentation on an input image. + * + * Only available on Android and iOS. + * + * @since 5.2.0 + */ + processImage(options: ProcessImageOptions): Promise; +} + +/** + * @since 5.2.0 + */ +export interface ProcessImageOptions { + /** + * The local path to the image file. + * + * @since 5.2.0 + */ + path: string; + + /** + * Asks the segmenter to return the raw size mask which matches the model output size. + * + * @since 5.2.0 + */ + rawSizeMask?: boolean; +} + +/** + * @since 5.2.0 + * @see https://developers.google.com/android/reference/com/google/mlkit/vision/segmentation/SegmentationMask + */ +export interface ProcessImageResult { + /** + * Returns a mask that indicates the foreground and background segmentation. + * + * This mask’s dimensions could vary, depending on whether a raw size mask is requested via options. + * + * @since 5.2.0 + */ + mask: number[]; + + /** + * Returns the width of the mask. + * + * @since 5.2.0 + */ + width: number; + /** + * Returns the height of the mask. + * + * @since 5.2.0 + */ + height: number; } diff --git a/packages/selfie-segmentation/src/web.ts b/packages/selfie-segmentation/src/web.ts index 99ba43b..e581361 100644 --- a/packages/selfie-segmentation/src/web.ts +++ b/packages/selfie-segmentation/src/web.ts @@ -1,13 +1,25 @@ -import { WebPlugin } from '@capacitor/core'; +import { CapacitorException, ExceptionCode, WebPlugin } from '@capacitor/core'; -import type { SelfieSegmentationPlugin } from './definitions'; +import type { + SelfieSegmentationPlugin, + ProcessImageOptions, + ProcessImageResult, +} from './definitions'; export class SelfieSegmentationWeb extends WebPlugin implements SelfieSegmentationPlugin { - async echo(options: { value: string }): Promise<{ value: string }> { - console.log('ECHO', options); - return options; + public async processImage( + _options: ProcessImageOptions, + ): Promise { + throw this.createUnavailableException(); + } + + private createUnavailableException(): CapacitorException { + return new CapacitorException( + 'This Selfie Segmentation plugin method is not available on this platform.', + ExceptionCode.Unavailable, + ); } } From 5464dbe6fede26bb2424851c323e15bad6281e00 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 2 Sep 2023 11:15:44 +0200 Subject: [PATCH 02/12] changed to enableRawSizeMask --- packages/selfie-segmentation/src/definitions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/selfie-segmentation/src/definitions.ts b/packages/selfie-segmentation/src/definitions.ts index 649d323..b5f45fc 100644 --- a/packages/selfie-segmentation/src/definitions.ts +++ b/packages/selfie-segmentation/src/definitions.ts @@ -25,7 +25,7 @@ export interface ProcessImageOptions { * * @since 5.2.0 */ - rawSizeMask?: boolean; + enableRawSizeMask?: boolean; } /** From ade1ab9b09402ad014113d33ecc1214a6290ec2c Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 2 Sep 2023 13:24:03 +0200 Subject: [PATCH 03/12] feat: add Android implementation --- packages/selfie-segmentation/README.md | 10 +-- .../selfie-segmentation/android/build.gradle | 2 +- .../ProcessImageResultCallback.java | 11 +++ .../SelfieSegmentation.java | 68 ++++++++++++++++- .../SelfieSegmentationPlugin.java | 73 +++++++++++++++++-- .../classes/ProcessImageOptions.java | 23 ++++++ .../classes/ProcessImageResult.java | 41 +++++++++++ 7 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/ProcessImageResultCallback.java create mode 100644 packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java create mode 100644 packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java diff --git a/packages/selfie-segmentation/README.md b/packages/selfie-segmentation/README.md index ab35683..3bab1ad 100644 --- a/packages/selfie-segmentation/README.md +++ b/packages/selfie-segmentation/README.md @@ -1,6 +1,6 @@ # @capacitor-mlkit/selfie-segmentation -Unofficial Capacitor plugin for [ML Kit Face Segmentation](https://developers.google.com/ml-kit/vision/selfie-segmentation).[^1] +Unofficial Capacitor plugin for [ML Kit Selfie Segmentation](https://developers.google.com/ml-kit/vision/selfie-segmentation).[^1] ## Installation @@ -80,10 +80,10 @@ Only available on Android and iOS. #### ProcessImageOptions -| Prop | Type | Description | Since | -| ----------------- | -------------------- | ----------------------------------------------------------------------------------- | ----- | -| **`path`** | string | The local path to the image file. | 5.2.0 | -| **`rawSizeMask`** | boolean | Asks the segmenter to return the raw size mask which matches the model output size. | 5.2.0 | +| Prop | Type | Description | Since | +| ----------------------- | -------------------- | ----------------------------------------------------------------------------------- | ----- | +| **`path`** | string | The local path to the image file. | 5.2.0 | +| **`enableRawSizeMask`** | boolean | Asks the segmenter to return the raw size mask which matches the model output size. | 5.2.0 | diff --git a/packages/selfie-segmentation/android/build.gradle b/packages/selfie-segmentation/android/build.gradle index 6df5175..3b1f28b 100644 --- a/packages/selfie-segmentation/android/build.gradle +++ b/packages/selfie-segmentation/android/build.gradle @@ -12,7 +12,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:8.0.0' + classpath 'com.android.tools.build:gradle:8.0.2' } } diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/ProcessImageResultCallback.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/ProcessImageResultCallback.java new file mode 100644 index 0000000..f1febc2 --- /dev/null +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/ProcessImageResultCallback.java @@ -0,0 +1,11 @@ +package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation; + +import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageResult; + +public interface ProcessImageResultCallback { + void success(ProcessImageResult result); + + void cancel(); + + void error(Exception exception); +} diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java index 0fa0305..c28dddd 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java @@ -1,11 +1,71 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation; -import android.util.Log; +import android.net.Uri; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.segmentation.Segmentation; +import com.google.mlkit.vision.segmentation.Segmenter; +import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions; +import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageOptions; +import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageResult; public class SelfieSegmentation { - public String echo(String value) { - Log.i("Echo", value); - return value; + @NonNull + private final SelfieSegmentationPlugin plugin; + + public SelfieSegmentation(@NonNull SelfieSegmentationPlugin plugin) { + this.plugin = plugin; + } + + @Nullable + public InputImage createInputImageFromFilePath(@NonNull String path) { + try { + return InputImage.fromFilePath(this.plugin.getContext(), Uri.parse(path)); + } catch (Exception exception) { + return null; + } + } + + public void processImage(ProcessImageOptions options, ProcessImageResultCallback callback) { + InputImage inputImage = options.getInputImage(); + boolean enableRawSizeMask = options.isRawSizeMaskEnabled(); + + SelfieSegmenterOptions.Builder builder = new SelfieSegmenterOptions.Builder(); + builder.setDetectorMode(SelfieSegmenterOptions.SINGLE_IMAGE_MODE); + if (enableRawSizeMask) { + builder.enableRawSizeMask(); + } + SelfieSegmenterOptions selfieSegmenterOptions = builder.build(); + + final Segmenter segmenter = Segmentation.getClient(selfieSegmenterOptions); + plugin + .getActivity() + .runOnUiThread( + () -> { + segmenter + .process(inputImage) + .addOnSuccessListener( + mask -> { + segmenter.close(); + ProcessImageResult result = new ProcessImageResult(mask); + callback.success(result); + } + ) + .addOnCanceledListener( + () -> { + segmenter.close(); + callback.cancel(); + } + ) + .addOnFailureListener( + exception -> { + segmenter.close(); + callback.error(exception); + } + ); + } + ); } } diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java index cd7ba56..628c13f 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java @@ -1,22 +1,81 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation; -import com.getcapacitor.JSObject; +import com.getcapacitor.Logger; import com.getcapacitor.Plugin; import com.getcapacitor.PluginCall; import com.getcapacitor.PluginMethod; import com.getcapacitor.annotation.CapacitorPlugin; +import com.google.mlkit.vision.common.InputImage; +import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageOptions; +import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageResult; @CapacitorPlugin(name = "SelfieSegmentation") public class SelfieSegmentationPlugin extends Plugin { - private SelfieSegmentation implementation = new SelfieSegmentation(); + public static final String TAG = "SelfieSegmentation"; + public static final String ERROR_PROCESS_IMAGE_CANCELED = "processImage canceled."; + public static final String ERROR_PATH_MISSING = "path must be provided."; + public static final String ERROR_LOAD_IMAGE_FAILED = "image could not be loaded."; + + private SelfieSegmentation implementation; + + @Override + public void load() { + try { + implementation = new SelfieSegmentation(this); + } catch (Exception exception) { + Logger.error(TAG, exception.getMessage(), exception); + } + } @PluginMethod - public void echo(PluginCall call) { - String value = call.getString("value"); + public void processImage(PluginCall call) { + try { + String path = call.getString("path", null); + if (path == null) { + call.reject(ERROR_PATH_MISSING); + return; + } + Boolean enableRawSizeMask = call.getBoolean("enableRawSizeMask", false); + + InputImage image = implementation.createInputImageFromFilePath(path); + if (image == null) { + call.reject(ERROR_LOAD_IMAGE_FAILED); + return; + } + ProcessImageOptions options = new ProcessImageOptions(image, enableRawSizeMask); + + implementation.processImage( + options, + new ProcessImageResultCallback() { + @Override + public void success(ProcessImageResult result) { + try { + call.resolve(result.toJSObject()); + } catch (Exception exception) { + String message = exception.getMessage(); + Logger.error(TAG, message, exception); + call.reject(message); + } + } + + @Override + public void cancel() { + call.reject(ERROR_PROCESS_IMAGE_CANCELED); + } - JSObject ret = new JSObject(); - ret.put("value", implementation.echo(value)); - call.resolve(ret); + @Override + public void error(Exception exception) { + String message = exception.getMessage(); + Logger.error(TAG, message, exception); + call.reject(message); + } + } + ); + } catch (Exception exception) { + String message = exception.getMessage(); + Logger.error(TAG, message, exception); + call.reject(message); + } } } diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java new file mode 100644 index 0000000..8c6ded0 --- /dev/null +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java @@ -0,0 +1,23 @@ +package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes; + +import com.google.mlkit.vision.common.InputImage; + +public class ProcessImageOptions { + + private InputImage inputImage; + + private boolean enableRawSizeMask; + + public ProcessImageOptions(InputImage inputImage, boolean enableRawSizeMask) { + this.inputImage = inputImage; + this.enableRawSizeMask = enableRawSizeMask; + } + + public InputImage getInputImage() { + return inputImage; + } + + public boolean isRawSizeMaskEnabled() { + return enableRawSizeMask; + } +} diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java new file mode 100644 index 0000000..2a2c374 --- /dev/null +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java @@ -0,0 +1,41 @@ +package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes; + +import com.getcapacitor.JSArray; +import com.getcapacitor.JSObject; +import com.google.mlkit.vision.segmentation.SegmentationMask; +import java.nio.ByteBuffer; +import org.json.JSONException; + +public class ProcessImageResult { + + private SegmentationMask segmentationMask; + + public ProcessImageResult(SegmentationMask segmentationMask) { + this.segmentationMask = segmentationMask; + } + + public JSObject toJSObject() throws JSONException { + JSArray maskResult = this.createMaskResult(); + + JSObject result = new JSObject(); + result.put("mask", maskResult); + result.put("width", segmentationMask.getWidth()); + result.put("height", segmentationMask.getHeight()); + return result; + } + + private JSArray createMaskResult() throws JSONException { + JSArray result = new JSArray(); + + ByteBuffer mask = segmentationMask.getBuffer(); + int maskWidth = segmentationMask.getWidth(); + int maskHeight = segmentationMask.getHeight(); + + for (int y = 0; y < maskHeight; y++) { + for (int x = 0; x < maskWidth; x++) { + result.put(mask.getFloat()); + } + } + return result; + } +} From 788674d32562923b4caee192d7b9f9b1fbfc2170 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 2 Sep 2023 20:56:43 +0200 Subject: [PATCH 04/12] fixed minor issues --- packages/selfie-segmentation/README.md | 8 ++++---- packages/selfie-segmentation/android/build.gradle | 2 +- packages/selfie-segmentation/src/definitions.ts | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/selfie-segmentation/README.md b/packages/selfie-segmentation/README.md index 3bab1ad..8b3b719 100644 --- a/packages/selfie-segmentation/README.md +++ b/packages/selfie-segmentation/README.md @@ -80,10 +80,10 @@ Only available on Android and iOS. #### ProcessImageOptions -| Prop | Type | Description | Since | -| ----------------------- | -------------------- | ----------------------------------------------------------------------------------- | ----- | -| **`path`** | string | The local path to the image file. | 5.2.0 | -| **`enableRawSizeMask`** | boolean | Asks the segmenter to return the raw size mask which matches the model output size. | 5.2.0 | +| Prop | Type | Description | Default | Since | +| ----------------------- | -------------------- | ----------------------------------------------------------------------------------- | ------------------ | ----- | +| **`path`** | string | The local path to the image file. | | 5.2.0 | +| **`enableRawSizeMask`** | boolean | Asks the segmenter to return the raw size mask which matches the model output size. | false | 5.2.0 | diff --git a/packages/selfie-segmentation/android/build.gradle b/packages/selfie-segmentation/android/build.gradle index 3b1f28b..6df5175 100644 --- a/packages/selfie-segmentation/android/build.gradle +++ b/packages/selfie-segmentation/android/build.gradle @@ -12,7 +12,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:8.0.2' + classpath 'com.android.tools.build:gradle:8.0.0' } } diff --git a/packages/selfie-segmentation/src/definitions.ts b/packages/selfie-segmentation/src/definitions.ts index b5f45fc..077441e 100644 --- a/packages/selfie-segmentation/src/definitions.ts +++ b/packages/selfie-segmentation/src/definitions.ts @@ -24,6 +24,7 @@ export interface ProcessImageOptions { * Asks the segmenter to return the raw size mask which matches the model output size. * * @since 5.2.0 + * @default false */ enableRawSizeMask?: boolean; } From e6f464d62c4449a52632f3f4512d309d80e93e2b Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sun, 3 Sep 2023 10:44:05 +0200 Subject: [PATCH 05/12] feat(ios): add implementation --- .../ios/Plugin.xcodeproj/project.pbxproj | 16 +++++++ .../Plugin/Classes/ProcessImageOptions.swift | 23 +++++++++ .../Plugin/Classes/ProcessImageResult.swift | 47 ++++++++++++++++++ .../ios/Plugin/SelfieSegmentation.swift | 48 +++++++++++++++++-- .../ios/Plugin/SelfieSegmentationPlugin.m | 2 +- .../ios/Plugin/SelfieSegmentationPlugin.swift | 39 ++++++++++++--- 6 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift create mode 100644 packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift diff --git a/packages/selfie-segmentation/ios/Plugin.xcodeproj/project.pbxproj b/packages/selfie-segmentation/ios/Plugin.xcodeproj/project.pbxproj index a6d581f..865a496 100644 --- a/packages/selfie-segmentation/ios/Plugin.xcodeproj/project.pbxproj +++ b/packages/selfie-segmentation/ios/Plugin.xcodeproj/project.pbxproj @@ -10,6 +10,8 @@ 03FC29A292ACC40490383A1F /* Pods_Plugin.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 3B2A61DA5A1F2DD4F959604D /* Pods_Plugin.framework */; }; 20C0B05DCFC8E3958A738AF2 /* Pods_PluginTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = F6753A823D3815DB436415E3 /* Pods_PluginTests.framework */; }; 2F98D68224C9AAE500613A4C /* SelfieSegmentation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F98D68124C9AAE400613A4C /* SelfieSegmentation.swift */; }; + 4A4E0DE52AA478A800BED263 /* ProcessImageOptions.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A4E0DE32AA478A800BED263 /* ProcessImageOptions.swift */; }; + 4A4E0DE62AA478A800BED263 /* ProcessImageResult.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A4E0DE42AA478A800BED263 /* ProcessImageResult.swift */; }; 50ADFF92201F53D600D50D53 /* Plugin.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 50ADFF88201F53D600D50D53 /* Plugin.framework */; }; 50ADFF97201F53D600D50D53 /* SelfieSegmentationTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50ADFF96201F53D600D50D53 /* SelfieSegmentationTests.swift */; }; 50ADFF99201F53D600D50D53 /* SelfieSegmentationPlugin.h in Headers */ = {isa = PBXBuildFile; fileRef = 50ADFF8B201F53D600D50D53 /* SelfieSegmentationPlugin.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -31,6 +33,8 @@ /* Begin PBXFileReference section */ 2F98D68124C9AAE400613A4C /* SelfieSegmentation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SelfieSegmentation.swift; sourceTree = ""; }; 3B2A61DA5A1F2DD4F959604D /* Pods_Plugin.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Plugin.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 4A4E0DE32AA478A800BED263 /* ProcessImageOptions.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = ProcessImageOptions.swift; path = Classes/ProcessImageOptions.swift; sourceTree = ""; }; + 4A4E0DE42AA478A800BED263 /* ProcessImageResult.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = ProcessImageResult.swift; path = Classes/ProcessImageResult.swift; sourceTree = ""; }; 50ADFF88201F53D600D50D53 /* Plugin.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Plugin.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 50ADFF8B201F53D600D50D53 /* SelfieSegmentationPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SelfieSegmentationPlugin.h; sourceTree = ""; }; 50ADFF8C201F53D600D50D53 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; @@ -69,6 +73,15 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 4A4E0DE22AA4788400BED263 /* Classes */ = { + isa = PBXGroup; + children = ( + 4A4E0DE32AA478A800BED263 /* ProcessImageOptions.swift */, + 4A4E0DE42AA478A800BED263 /* ProcessImageResult.swift */, + ); + name = Classes; + sourceTree = ""; + }; 50ADFF7E201F53D600D50D53 = { isa = PBXGroup; children = ( @@ -92,6 +105,7 @@ 50ADFF8A201F53D600D50D53 /* Plugin */ = { isa = PBXGroup; children = ( + 4A4E0DE22AA4788400BED263 /* Classes */, 50E1A94720377CB70090CE1A /* SelfieSegmentationPlugin.swift */, 2F98D68124C9AAE400613A4C /* SelfieSegmentation.swift */, 50ADFF8B201F53D600D50D53 /* SelfieSegmentationPlugin.h */, @@ -364,8 +378,10 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4A4E0DE62AA478A800BED263 /* ProcessImageResult.swift in Sources */, 50E1A94820377CB70090CE1A /* SelfieSegmentationPlugin.swift in Sources */, 2F98D68224C9AAE500613A4C /* SelfieSegmentation.swift in Sources */, + 4A4E0DE52AA478A800BED263 /* ProcessImageOptions.swift in Sources */, 50ADFFA82020EE4F00D50D53 /* SelfieSegmentationPlugin.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift new file mode 100644 index 0000000..7dc31e7 --- /dev/null +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift @@ -0,0 +1,23 @@ +import Foundation +import MLKitVision + +@objc class ProcessImageOptions: NSObject { + private var visionImage: VisionImage + private var enableRawSizeMask: Bool + + init( + visionImage: VisionImage, + enableRawSizeMask: Bool + ) { + self.visionImage = visionImage + self.enableRawSizeMask = enableRawSizeMask + } + + func getVisionImage() -> VisionImage { + return visionImage + } + + func shouldEnableRawSizeMask() -> Bool { + return enableRawSizeMask + } +} diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift new file mode 100644 index 0000000..ef8781c --- /dev/null +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift @@ -0,0 +1,47 @@ +import Foundation +import Capacitor +import MLKitVision +import MLKitSegmentationSelfie + +@objc class ProcessImageResult: NSObject { + let segmentationMask: SegmentationMask + + init(segmentationMask: SegmentationMask) { + self.segmentationMask = segmentationMask + } + + func toJSObject() -> JSObject { + let (maskResult, maskWidth, maskHeight) = createMaskResult(mask: segmentationMask) + + var result = JSObject() + result["mask"] = maskResult + result["width"] = maskWidth + result["height"] = maskHeight + + return result + } + + private func createMaskResult(mask: SegmentationMask) -> (JSArray, Int, Int) { + var result = JSArray() + + let maskWidth = CVPixelBufferGetWidth(mask.buffer) + let maskHeight = CVPixelBufferGetHeight(mask.buffer) + + CVPixelBufferLockBaseAddress(mask.buffer, CVPixelBufferLockFlags.readOnly) + let maskBytesPerRow = CVPixelBufferGetBytesPerRow(mask.buffer) + var maskAddress = + CVPixelBufferGetBaseAddress(mask.buffer)!.bindMemory( + to: Float32.self, capacity: maskBytesPerRow * maskHeight) + + for _ in 0...(maskHeight - 1) { + for col in 0...(maskWidth - 1) { + // Gets the confidence of the pixel in the mask being in the foreground. + let foregroundConfidence: Float32 = maskAddress[col] + result.append(foregroundConfidence) + } + maskAddress += maskBytesPerRow / MemoryLayout.size + } + + return (result, maskWidth, maskHeight) + } +} diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift index 977f2f1..fc3e495 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift @@ -1,8 +1,50 @@ import Foundation +import MLKitVision +import MLKitSegmentationSelfie @objc public class SelfieSegmentation: NSObject { - @objc public func echo(_ value: String) -> String { - print(value) - return value + public let plugin: SelfieSegmentationPlugin + + init(plugin: SelfieSegmentationPlugin) { + self.plugin = plugin + } + + @objc func createVisionImageFromFilePath(_ path: String) -> VisionImage? { + guard let url = URL.init(string: path) else { + return nil + } + if FileManager.default.fileExists(atPath: url.path) { + guard let image = UIImage.init(contentsOfFile: url.path) else { + return nil + } + return VisionImage.init( + image: image + ) + } else { + return nil + } + } + + @objc func processImage(_ options: ProcessImageOptions, completion: @escaping (ProcessImageResult?, Error?) -> Void) { + let visionImage = options.getVisionImage() + let enableRawSizeMask = options.shouldEnableRawSizeMask() + + let selfieSegmenterOptions: SelfieSegmenterOptions = SelfieSegmenterOptions() + selfieSegmenterOptions.segmenterMode = .singleImage + selfieSegmenterOptions.shouldEnableRawSizeMask = enableRawSizeMask + + let segmenter = Segmenter.segmenter( + options: selfieSegmenterOptions + ) + + do { + let mask: SegmentationMask = try segmenter.results( + in: visionImage + ) + let result = ProcessImageResult(segmentationMask: mask) + completion(result, nil) + } catch let error { + completion(nil, error) + } } } diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.m b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.m index e850f82..6ec0c6b 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.m +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.m @@ -4,5 +4,5 @@ // Define the plugin using the CAP_PLUGIN Macro, and // each method the plugin supports using the CAP_PLUGIN_METHOD macro. CAP_PLUGIN(SelfieSegmentationPlugin, "SelfieSegmentation", - CAP_PLUGIN_METHOD(echo, CAPPluginReturnPromise); + CAP_PLUGIN_METHOD(processImage, CAPPluginReturnPromise); ) diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift index 69f9a98..1c35a18 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift @@ -7,12 +7,39 @@ import Capacitor */ @objc(SelfieSegmentationPlugin) public class SelfieSegmentationPlugin: CAPPlugin { - private let implementation = SelfieSegmentation() + public let tag = "SelfieSegmentation" + public let errorPathMissing = "path must be provided." + public let errorLoadImageFailed = "image could not be loaded." - @objc func echo(_ call: CAPPluginCall) { - let value = call.getString("value") ?? "" - call.resolve([ - "value": implementation.echo(value) - ]) + private var implementation: SelfieSegmentation? + + override public func load() { + implementation = SelfieSegmentation(plugin: self) + } + + @objc func processImage(_ call: CAPPluginCall) { + guard let path = call.getString("path") else { + call.reject(errorPathMissing) + return + } + let enableRawSizeMask = call.getBool("enableRawSizeMask", false) + + guard let visionImage = implementation?.createVisionImageFromFilePath(path) else { + call.reject(errorLoadImageFailed) + return + } + + let options = ProcessImageOptions(visionImage: visionImage, enableRawSizeMask: enableRawSizeMask) + + implementation?.processImage(options, completion: { result, error in + if let error = error { + CAPLog.print("[", self.tag, "] ", error) + call.reject(error.localizedDescription, nil, error) + return + } + if let result = result?.toJSObject() as? JSObject { + call.resolve(result) + } + }) } } From a09ceb811f1a7d7edb4f9a76f3b277d43780e66b Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Fri, 8 Sep 2023 22:49:52 +0200 Subject: [PATCH 06/12] refactor: return selfie segmented image --- packages/selfie-segmentation/README.md | 19 ++--- .../SelfieSegmentation.java | 78 +++++++++++++++++-- .../SelfieSegmentationPlugin.java | 6 +- .../classes/ProcessImageOptions.java | 34 ++++++-- .../classes/ProcessImageResult.java | 55 +++++++------ .../Plugin/Classes/ProcessImageOptions.swift | 9 +-- .../ios/Plugin/SelfieSegmentation.swift | 3 +- .../ios/Plugin/SelfieSegmentationPlugin.swift | 3 +- .../selfie-segmentation/src/definitions.ts | 23 +++--- 9 files changed, 158 insertions(+), 72 deletions(-) diff --git a/packages/selfie-segmentation/README.md b/packages/selfie-segmentation/README.md index 8b3b719..b7d62e3 100644 --- a/packages/selfie-segmentation/README.md +++ b/packages/selfie-segmentation/README.md @@ -71,19 +71,20 @@ Only available on Android and iOS. #### ProcessImageResult -| Prop | Type | Description | Since | -| ------------ | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| **`mask`** | number[] | Returns a mask that indicates the foreground and background segmentation. This mask’s dimensions could vary, depending on whether a raw size mask is requested via options. | 5.2.0 | -| **`width`** | number | Returns the width of the mask. | 5.2.0 | -| **`height`** | number | Returns the height of the mask. | 5.2.0 | +| Prop | Type | Description | Since | +| ------------ | ------------------- | ------------------------------------- | ----- | +| **`path`** | string | The path to the segmented image file. | 5.2.0 | +| **`width`** | number | Returns the width of the image file. | 5.2.0 | +| **`height`** | number | Returns the height of the image file. | 5.2.0 | #### ProcessImageOptions -| Prop | Type | Description | Default | Since | -| ----------------------- | -------------------- | ----------------------------------------------------------------------------------- | ------------------ | ----- | -| **`path`** | string | The local path to the image file. | | 5.2.0 | -| **`enableRawSizeMask`** | boolean | Asks the segmenter to return the raw size mask which matches the model output size. | false | 5.2.0 | +| Prop | Type | Description | Since | +| ------------ | ------------------- | --------------------------------- | ----- | +| **`path`** | string | The local path to the image file. | 5.2.0 | +| **`width`** | number | Scale the image to this width. | 5.2.0 | +| **`height`** | number | Scale the image to this height. | 5.2.0 | diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java index c28dddd..d16ef8a 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java @@ -1,5 +1,6 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation; +import android.graphics.Bitmap; import android.net.Uri; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -9,6 +10,14 @@ import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions; import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageOptions; import io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes.ProcessImageResult; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Objects; public class SelfieSegmentation { @@ -30,13 +39,10 @@ public InputImage createInputImageFromFilePath(@NonNull String path) { public void processImage(ProcessImageOptions options, ProcessImageResultCallback callback) { InputImage inputImage = options.getInputImage(); - boolean enableRawSizeMask = options.isRawSizeMaskEnabled(); SelfieSegmenterOptions.Builder builder = new SelfieSegmenterOptions.Builder(); builder.setDetectorMode(SelfieSegmenterOptions.SINGLE_IMAGE_MODE); - if (enableRawSizeMask) { - builder.enableRawSizeMask(); - } + // builder.enableRawSizeMask(); SelfieSegmenterOptions selfieSegmenterOptions = builder.build(); final Segmenter segmenter = Segmentation.getClient(selfieSegmenterOptions); @@ -47,10 +53,68 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback segmenter .process(inputImage) .addOnSuccessListener( - mask -> { + segmentationMask -> { segmenter.close(); - ProcessImageResult result = new ProcessImageResult(mask); - callback.success(result); + + ByteBuffer mask = segmentationMask.getBuffer(); + // int maskWidth = segmentationMask.getWidth(); + // int maskHeight = segmentationMask.getHeight(); + + Bitmap mPictureBitmap = inputImage.getBitmapInternal(); + Objects.requireNonNull(mPictureBitmap).setHasAlpha(true); + + ByteBuffer pixels = ByteBuffer.allocateDirect(mPictureBitmap.getAllocationByteCount()); + mPictureBitmap.copyPixelsToBuffer(pixels); + + final boolean bigEndian = pixels.order() == ByteOrder.BIG_ENDIAN; + final int ALPHA = bigEndian ? 3 : 0; + final int RED = bigEndian ? 2 : 1; + final int GREEN = bigEndian ? 1 : 2; + final int BLUE = bigEndian ? 0 : 3; + + for (int i = 0; i < pixels.capacity() >> 2; i++) { + float confidence = mask.getFloat(); + + if (confidence >= 0.9f) { + // byte alpha = pixels.get((i << 2) + ALPHA); + byte red = pixels.get((i << 2) + RED); + byte green = pixels.get((i << 2) + GREEN); + byte blue = pixels.get((i << 2) + BLUE); + + pixels.put((i << 2) + ALPHA, (byte) (0xff)); + pixels.put((i << 2) + RED, (byte) (red * confidence)); + pixels.put((i << 2) + GREEN, (byte) (green * confidence)); + pixels.put((i << 2) + BLUE, (byte) (blue * confidence)); + } else { + pixels.putInt(i << 2, 0x00000000); // transparent + } + } + + mPictureBitmap.copyPixelsFromBuffer(pixels.rewind()); + + // Reset byteBuffer pointer to beginning + mask.rewind(); + + // Create an image file name + String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); + String imageFileName = "PNG_" + timeStamp + "_"; + + try { + File image = File.createTempFile(imageFileName, ".png"); + + OutputStream stream = new FileOutputStream(image); + mPictureBitmap.compress(Bitmap.CompressFormat.PNG, 100, stream); + stream.close(); + + ProcessImageResult result = new ProcessImageResult( + image.getAbsolutePath(), + mPictureBitmap.getWidth(), + mPictureBitmap.getHeight() + ); + callback.success(result); + } catch (Exception exception) { + callback.error(exception); + } } ) .addOnCanceledListener( diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java index 628c13f..7c3f791 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java @@ -36,14 +36,16 @@ public void processImage(PluginCall call) { call.reject(ERROR_PATH_MISSING); return; } - Boolean enableRawSizeMask = call.getBoolean("enableRawSizeMask", false); + + Integer width = call.getInt("width", null); + Integer height = call.getInt("height", null); InputImage image = implementation.createInputImageFromFilePath(path); if (image == null) { call.reject(ERROR_LOAD_IMAGE_FAILED); return; } - ProcessImageOptions options = new ProcessImageOptions(image, enableRawSizeMask); + ProcessImageOptions options = new ProcessImageOptions(image, width, height); implementation.processImage( options, diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java index 8c6ded0..11f9e4c 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java @@ -1,23 +1,43 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes; +import android.graphics.Bitmap; import com.google.mlkit.vision.common.InputImage; public class ProcessImageOptions { private InputImage inputImage; - private boolean enableRawSizeMask; - - public ProcessImageOptions(InputImage inputImage, boolean enableRawSizeMask) { - this.inputImage = inputImage; - this.enableRawSizeMask = enableRawSizeMask; + public ProcessImageOptions(InputImage inputImage, Integer width, Integer height) { + this.inputImage = scaledImage(inputImage, width, height); } public InputImage getInputImage() { return inputImage; } - public boolean isRawSizeMaskEnabled() { - return enableRawSizeMask; + private InputImage scaledImage(InputImage inputImage, Integer width, Integer height) { + float scaleX = (width != null) ? width * 1f / inputImage.getWidth() : 0f; + float scaleY = (height != null) ? height * 1f / inputImage.getHeight() : 0f; + + if (scaleX > 0f || scaleY > 0f) { + if (scaleX > 0f && scaleY == 0f) { + scaleY = scaleX; + } + if (scaleX == 0f && scaleY > 0f) { + scaleX = scaleY; + } + + return InputImage.fromBitmap( + Bitmap.createScaledBitmap( + inputImage.getBitmapInternal(), + (int) (inputImage.getWidth() * scaleX), + (int) (inputImage.getHeight() * scaleY), + false + ), + inputImage.getRotationDegrees() + ); + } + + return inputImage; } } diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java index 2a2c374..f75fae2 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java @@ -1,41 +1,46 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes; -import com.getcapacitor.JSArray; import com.getcapacitor.JSObject; -import com.google.mlkit.vision.segmentation.SegmentationMask; -import java.nio.ByteBuffer; import org.json.JSONException; public class ProcessImageResult { - private SegmentationMask segmentationMask; + // private SegmentationMask segmentationMask; + private String imagePath; + private int width; + private int height; - public ProcessImageResult(SegmentationMask segmentationMask) { - this.segmentationMask = segmentationMask; + public ProcessImageResult(String imagePath, int width, int height) { + // this.segmentationMask = segmentationMask; + this.imagePath = imagePath; + this.width = width; + this.height = height; } public JSObject toJSObject() throws JSONException { - JSArray maskResult = this.createMaskResult(); + // JSArray maskResult = this.createMaskResult(); JSObject result = new JSObject(); - result.put("mask", maskResult); - result.put("width", segmentationMask.getWidth()); - result.put("height", segmentationMask.getHeight()); - return result; - } - - private JSArray createMaskResult() throws JSONException { - JSArray result = new JSArray(); - - ByteBuffer mask = segmentationMask.getBuffer(); - int maskWidth = segmentationMask.getWidth(); - int maskHeight = segmentationMask.getHeight(); - - for (int y = 0; y < maskHeight; y++) { - for (int x = 0; x < maskWidth; x++) { - result.put(mask.getFloat()); - } - } + // result.put("mask", maskResult); + // result.put("width", segmentationMask.getWidth()); + // result.put("height", segmentationMask.getHeight()); + result.put("path", imagePath); + result.put("width", width); + result.put("height", height); return result; } + // private JSArray createMaskResult() throws JSONException { + // JSArray result = new JSArray(); + // + // ByteBuffer mask = segmentationMask.getBuffer(); + // int maskWidth = segmentationMask.getWidth(); + // int maskHeight = segmentationMask.getHeight(); + // + // for (int y = 0; y < maskHeight; y++) { + // for (int x = 0; x < maskWidth; x++) { + // result.put(mask.getFloat()); + // } + // } + // return result; + // } } diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift index 7dc31e7..fcf830e 100644 --- a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift @@ -3,21 +3,14 @@ import MLKitVision @objc class ProcessImageOptions: NSObject { private var visionImage: VisionImage - private var enableRawSizeMask: Bool init( - visionImage: VisionImage, - enableRawSizeMask: Bool + visionImage: VisionImage ) { self.visionImage = visionImage - self.enableRawSizeMask = enableRawSizeMask } func getVisionImage() -> VisionImage { return visionImage } - - func shouldEnableRawSizeMask() -> Bool { - return enableRawSizeMask - } } diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift index fc3e495..a8d4f28 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift @@ -27,11 +27,10 @@ import MLKitSegmentationSelfie @objc func processImage(_ options: ProcessImageOptions, completion: @escaping (ProcessImageResult?, Error?) -> Void) { let visionImage = options.getVisionImage() - let enableRawSizeMask = options.shouldEnableRawSizeMask() let selfieSegmenterOptions: SelfieSegmenterOptions = SelfieSegmenterOptions() selfieSegmenterOptions.segmenterMode = .singleImage - selfieSegmenterOptions.shouldEnableRawSizeMask = enableRawSizeMask + selfieSegmenterOptions.shouldEnableRawSizeMask = true let segmenter = Segmenter.segmenter( options: selfieSegmenterOptions diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift index 1c35a18..e0b20ea 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift @@ -22,14 +22,13 @@ public class SelfieSegmentationPlugin: CAPPlugin { call.reject(errorPathMissing) return } - let enableRawSizeMask = call.getBool("enableRawSizeMask", false) guard let visionImage = implementation?.createVisionImageFromFilePath(path) else { call.reject(errorLoadImageFailed) return } - let options = ProcessImageOptions(visionImage: visionImage, enableRawSizeMask: enableRawSizeMask) + let options = ProcessImageOptions(visionImage: visionImage) implementation?.processImage(options, completion: { result, error in if let error = error { diff --git a/packages/selfie-segmentation/src/definitions.ts b/packages/selfie-segmentation/src/definitions.ts index 077441e..f40d317 100644 --- a/packages/selfie-segmentation/src/definitions.ts +++ b/packages/selfie-segmentation/src/definitions.ts @@ -21,36 +21,39 @@ export interface ProcessImageOptions { path: string; /** - * Asks the segmenter to return the raw size mask which matches the model output size. + * Scale the image to this width. * * @since 5.2.0 - * @default false */ - enableRawSizeMask?: boolean; + width?: number; + + /** + * Scale the image to this height. + * + * @since 5.2.0 + */ + height?: number; } /** * @since 5.2.0 - * @see https://developers.google.com/android/reference/com/google/mlkit/vision/segmentation/SegmentationMask */ export interface ProcessImageResult { /** - * Returns a mask that indicates the foreground and background segmentation. - * - * This mask’s dimensions could vary, depending on whether a raw size mask is requested via options. + * The path to the segmented image file. * * @since 5.2.0 */ - mask: number[]; + path: string; /** - * Returns the width of the mask. + * Returns the width of the image file. * * @since 5.2.0 */ width: number; /** - * Returns the height of the mask. + * Returns the height of the image file. * * @since 5.2.0 */ From cc4d9fac8aa83f9114a9013344b01c9c38eb3ec3 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 9 Sep 2023 09:26:44 +0200 Subject: [PATCH 07/12] feat: added confidence threshold --- packages/selfie-segmentation/README.md | 11 ++++++----- .../mlkit/selfiesegmentation/SelfieSegmentation.java | 3 ++- .../selfiesegmentation/SelfieSegmentationPlugin.java | 4 +++- .../classes/ProcessImageOptions.java | 6 +++++- packages/selfie-segmentation/src/definitions.ts | 8 ++++++++ 5 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/selfie-segmentation/README.md b/packages/selfie-segmentation/README.md index b7d62e3..7de3831 100644 --- a/packages/selfie-segmentation/README.md +++ b/packages/selfie-segmentation/README.md @@ -80,11 +80,12 @@ Only available on Android and iOS. #### ProcessImageOptions -| Prop | Type | Description | Since | -| ------------ | ------------------- | --------------------------------- | ----- | -| **`path`** | string | The local path to the image file. | 5.2.0 | -| **`width`** | number | Scale the image to this width. | 5.2.0 | -| **`height`** | number | Scale the image to this height. | 5.2.0 | +| Prop | Type | Description | Default | Since | +| ---------------- | ------------------- | --------------------------------- | ---------------- | ----- | +| **`path`** | string | The local path to the image file. | | 5.2.0 | +| **`width`** | number | Scale the image to this width. | | 5.2.0 | +| **`height`** | number | Scale the image to this height. | | 5.2.0 | +| **`confidence`** | number | Sets the confidence threshold. | 0.9 | 5.2.0 | diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java index d16ef8a..608df65 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java @@ -39,6 +39,7 @@ public InputImage createInputImageFromFilePath(@NonNull String path) { public void processImage(ProcessImageOptions options, ProcessImageResultCallback callback) { InputImage inputImage = options.getInputImage(); + Float threshold = options.getConfidence(); SelfieSegmenterOptions.Builder builder = new SelfieSegmenterOptions.Builder(); builder.setDetectorMode(SelfieSegmenterOptions.SINGLE_IMAGE_MODE); @@ -75,7 +76,7 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback for (int i = 0; i < pixels.capacity() >> 2; i++) { float confidence = mask.getFloat(); - if (confidence >= 0.9f) { + if (confidence >= threshold) { // byte alpha = pixels.get((i << 2) + ALPHA); byte red = pixels.get((i << 2) + RED); byte green = pixels.get((i << 2) + GREEN); diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java index 7c3f791..808e1b9 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java @@ -40,12 +40,14 @@ public void processImage(PluginCall call) { Integer width = call.getInt("width", null); Integer height = call.getInt("height", null); + Float confidence = call.getFloat("confidence", 0.9f); + InputImage image = implementation.createInputImageFromFilePath(path); if (image == null) { call.reject(ERROR_LOAD_IMAGE_FAILED); return; } - ProcessImageOptions options = new ProcessImageOptions(image, width, height); + ProcessImageOptions options = new ProcessImageOptions(image, width, height, confidence); implementation.processImage( options, diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java index 11f9e4c..a92870a 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java @@ -6,14 +6,18 @@ public class ProcessImageOptions { private InputImage inputImage; + private Float confidence; - public ProcessImageOptions(InputImage inputImage, Integer width, Integer height) { + public ProcessImageOptions(InputImage inputImage, Integer width, Integer height, Float confidence) { this.inputImage = scaledImage(inputImage, width, height); + + this.confidence = confidence; } public InputImage getInputImage() { return inputImage; } + public Float getConfidence() { return confidence; }; private InputImage scaledImage(InputImage inputImage, Integer width, Integer height) { float scaleX = (width != null) ? width * 1f / inputImage.getWidth() : 0f; diff --git a/packages/selfie-segmentation/src/definitions.ts b/packages/selfie-segmentation/src/definitions.ts index f40d317..56f5c7d 100644 --- a/packages/selfie-segmentation/src/definitions.ts +++ b/packages/selfie-segmentation/src/definitions.ts @@ -33,6 +33,14 @@ export interface ProcessImageOptions { * @since 5.2.0 */ height?: number; + + /** + * Sets the confidence threshold. + * + * @since 5.2.0 + * @default 0.9 + */ + confidence?: number; } /** From 4a489e60ecf2c5b8a235ac66f54714712cd183dc Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 9 Sep 2023 10:01:50 +0200 Subject: [PATCH 08/12] lint(android) --- .../selfiesegmentation/classes/ProcessImageOptions.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java index a92870a..c2eefed 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java @@ -17,7 +17,10 @@ public ProcessImageOptions(InputImage inputImage, Integer width, Integer height, public InputImage getInputImage() { return inputImage; } - public Float getConfidence() { return confidence; }; + + public Float getConfidence() { + return confidence; + } private InputImage scaledImage(InputImage inputImage, Integer width, Integer height) { float scaleX = (width != null) ? width * 1f / inputImage.getWidth() : 0f; From 0cb414a8ce0c6c9df9846c31799ed22f333c3b95 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sat, 9 Sep 2023 11:41:11 +0200 Subject: [PATCH 09/12] cleaned up code --- packages/selfie-segmentation/README.md | 12 +++---- .../SelfieSegmentation.java | 32 ++++++++--------- .../SelfieSegmentationPlugin.java | 5 ++- .../classes/ProcessImageOptions.java | 15 ++++---- .../classes/ProcessImageResult.java | 35 +++++-------------- .../selfie-segmentation/src/definitions.ts | 3 +- 6 files changed, 41 insertions(+), 61 deletions(-) diff --git a/packages/selfie-segmentation/README.md b/packages/selfie-segmentation/README.md index 7de3831..d253246 100644 --- a/packages/selfie-segmentation/README.md +++ b/packages/selfie-segmentation/README.md @@ -80,12 +80,12 @@ Only available on Android and iOS. #### ProcessImageOptions -| Prop | Type | Description | Default | Since | -| ---------------- | ------------------- | --------------------------------- | ---------------- | ----- | -| **`path`** | string | The local path to the image file. | | 5.2.0 | -| **`width`** | number | Scale the image to this width. | | 5.2.0 | -| **`height`** | number | Scale the image to this height. | | 5.2.0 | -| **`confidence`** | number | Sets the confidence threshold. | 0.9 | 5.2.0 | +| Prop | Type | Description | Default | Since | +| ---------------- | ------------------- | ----------------------------------------------------------------------------------------- | ---------------- | ----- | +| **`path`** | string | The local path to the image file. | | 5.2.0 | +| **`width`** | number | Scale the image to this width. If no `height` is given, it will respect the aspect ratio. | | 5.2.0 | +| **`height`** | number | Scale the image to this height. If no `width` is given, it will respect the aspect ratio. | | 5.2.0 | +| **`confidence`** | number | Sets the confidence threshold. | 0.9 | 5.2.0 | diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java index 608df65..3c7e6e5 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentation.java @@ -1,5 +1,6 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation; +import android.annotation.SuppressLint; import android.graphics.Bitmap; import android.net.Uri; import androidx.annotation.NonNull; @@ -43,14 +44,14 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback SelfieSegmenterOptions.Builder builder = new SelfieSegmenterOptions.Builder(); builder.setDetectorMode(SelfieSegmenterOptions.SINGLE_IMAGE_MODE); - // builder.enableRawSizeMask(); SelfieSegmenterOptions selfieSegmenterOptions = builder.build(); final Segmenter segmenter = Segmentation.getClient(selfieSegmenterOptions); + plugin .getActivity() .runOnUiThread( - () -> { + () -> segmenter .process(inputImage) .addOnSuccessListener( @@ -58,14 +59,12 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback segmenter.close(); ByteBuffer mask = segmentationMask.getBuffer(); - // int maskWidth = segmentationMask.getWidth(); - // int maskHeight = segmentationMask.getHeight(); - Bitmap mPictureBitmap = inputImage.getBitmapInternal(); - Objects.requireNonNull(mPictureBitmap).setHasAlpha(true); + Bitmap bitmap = inputImage.getBitmapInternal(); + Objects.requireNonNull(bitmap).setHasAlpha(true); - ByteBuffer pixels = ByteBuffer.allocateDirect(mPictureBitmap.getAllocationByteCount()); - mPictureBitmap.copyPixelsToBuffer(pixels); + ByteBuffer pixels = ByteBuffer.allocateDirect(bitmap.getAllocationByteCount()); + bitmap.copyPixelsToBuffer(pixels); final boolean bigEndian = pixels.order() == ByteOrder.BIG_ENDIAN; final int ALPHA = bigEndian ? 3 : 0; @@ -77,7 +76,6 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback float confidence = mask.getFloat(); if (confidence >= threshold) { - // byte alpha = pixels.get((i << 2) + ALPHA); byte red = pixels.get((i << 2) + RED); byte green = pixels.get((i << 2) + GREEN); byte blue = pixels.get((i << 2) + BLUE); @@ -91,12 +89,10 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback } } - mPictureBitmap.copyPixelsFromBuffer(pixels.rewind()); - - // Reset byteBuffer pointer to beginning - mask.rewind(); + bitmap.copyPixelsFromBuffer(pixels.rewind()); // Create an image file name + @SuppressLint("SimpleDateFormat") String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); String imageFileName = "PNG_" + timeStamp + "_"; @@ -104,14 +100,15 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback File image = File.createTempFile(imageFileName, ".png"); OutputStream stream = new FileOutputStream(image); - mPictureBitmap.compress(Bitmap.CompressFormat.PNG, 100, stream); + bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream); stream.close(); ProcessImageResult result = new ProcessImageResult( image.getAbsolutePath(), - mPictureBitmap.getWidth(), - mPictureBitmap.getHeight() + bitmap.getWidth(), + bitmap.getHeight() ); + callback.success(result); } catch (Exception exception) { callback.error(exception); @@ -129,8 +126,7 @@ public void processImage(ProcessImageOptions options, ProcessImageResultCallback segmenter.close(); callback.error(exception); } - ); - } + ) ); } } diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java index 808e1b9..533c0fe 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java @@ -13,10 +13,13 @@ public class SelfieSegmentationPlugin extends Plugin { public static final String TAG = "SelfieSegmentation"; + public static final String ERROR_PROCESS_IMAGE_CANCELED = "processImage canceled."; public static final String ERROR_PATH_MISSING = "path must be provided."; public static final String ERROR_LOAD_IMAGE_FAILED = "image could not be loaded."; + public static final float CONFIDENCE = 0.9f; + private SelfieSegmentation implementation; @Override @@ -40,7 +43,7 @@ public void processImage(PluginCall call) { Integer width = call.getInt("width", null); Integer height = call.getInt("height", null); - Float confidence = call.getFloat("confidence", 0.9f); + Float confidence = call.getFloat("confidence", CONFIDENCE); InputImage image = implementation.createInputImageFromFilePath(path); if (image == null) { diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java index c2eefed..79755ab 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageOptions.java @@ -2,11 +2,13 @@ import android.graphics.Bitmap; import com.google.mlkit.vision.common.InputImage; +import java.util.Objects; public class ProcessImageOptions { - private InputImage inputImage; - private Float confidence; + private final InputImage inputImage; + + private final Float confidence; public ProcessImageOptions(InputImage inputImage, Integer width, Integer height, Float confidence) { this.inputImage = scaledImage(inputImage, width, height); @@ -27,16 +29,11 @@ private InputImage scaledImage(InputImage inputImage, Integer width, Integer hei float scaleY = (height != null) ? height * 1f / inputImage.getHeight() : 0f; if (scaleX > 0f || scaleY > 0f) { - if (scaleX > 0f && scaleY == 0f) { - scaleY = scaleX; - } - if (scaleX == 0f && scaleY > 0f) { - scaleX = scaleY; - } + if (scaleX > 0f && scaleY == 0f) scaleY = scaleX; else if (scaleY > 0f && scaleX == 0f) scaleX = scaleY; return InputImage.fromBitmap( Bitmap.createScaledBitmap( - inputImage.getBitmapInternal(), + Objects.requireNonNull(inputImage.getBitmapInternal()), (int) (inputImage.getWidth() * scaleX), (int) (inputImage.getHeight() * scaleY), false diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java index f75fae2..ffe3363 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/classes/ProcessImageResult.java @@ -1,46 +1,29 @@ package io.capawesome.capacitorjs.plugins.mlkit.selfiesegmentation.classes; import com.getcapacitor.JSObject; -import org.json.JSONException; public class ProcessImageResult { - // private SegmentationMask segmentationMask; - private String imagePath; - private int width; - private int height; + private final String imagePath; + + private final int width; + private final int height; public ProcessImageResult(String imagePath, int width, int height) { - // this.segmentationMask = segmentationMask; this.imagePath = imagePath; + this.width = width; this.height = height; } - public JSObject toJSObject() throws JSONException { - // JSArray maskResult = this.createMaskResult(); - + public JSObject toJSObject() { JSObject result = new JSObject(); - // result.put("mask", maskResult); - // result.put("width", segmentationMask.getWidth()); - // result.put("height", segmentationMask.getHeight()); + result.put("path", imagePath); + result.put("width", width); result.put("height", height); + return result; } - // private JSArray createMaskResult() throws JSONException { - // JSArray result = new JSArray(); - // - // ByteBuffer mask = segmentationMask.getBuffer(); - // int maskWidth = segmentationMask.getWidth(); - // int maskHeight = segmentationMask.getHeight(); - // - // for (int y = 0; y < maskHeight; y++) { - // for (int x = 0; x < maskWidth; x++) { - // result.put(mask.getFloat()); - // } - // } - // return result; - // } } diff --git a/packages/selfie-segmentation/src/definitions.ts b/packages/selfie-segmentation/src/definitions.ts index 56f5c7d..1e8d6b6 100644 --- a/packages/selfie-segmentation/src/definitions.ts +++ b/packages/selfie-segmentation/src/definitions.ts @@ -22,13 +22,14 @@ export interface ProcessImageOptions { /** * Scale the image to this width. + * If no `height` is given, it will respect the aspect ratio. * * @since 5.2.0 */ width?: number; - /** * Scale the image to this height. + * If no `width` is given, it will respect the aspect ratio. * * @since 5.2.0 */ From b3a757eb1c9628e759836d925ca5889b536c10b7 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sun, 10 Sep 2023 11:16:24 +0200 Subject: [PATCH 10/12] feat(ios): add implementation --- .../Plugin/Classes/ProcessImageOptions.swift | 59 +++++- .../Plugin/Classes/ProcessImageResult.swift | 47 ++--- .../ios/Plugin/SelfieSegmentation.swift | 186 ++++++++++++++++-- .../ios/Plugin/SelfieSegmentationPlugin.swift | 20 +- 4 files changed, 253 insertions(+), 59 deletions(-) diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift index fcf830e..41b55e9 100644 --- a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift @@ -1,16 +1,63 @@ import Foundation -import MLKitVision + +extension UIImage { + public func scaledImage(width: Int?, height: Int?) -> UIImage { + let newWidth: CGFloat + let newHeight: CGFloat + + if let width = width { + newWidth = CGFloat(width) + if let height = height { + newHeight = CGFloat(height) + } else { + let scaleFactor = newWidth / self.size.width + newHeight = self.size.height * scaleFactor + } + } else + if let height = height { + newHeight = CGFloat(height) + if let width = width { + newWidth = CGFloat(width) + } else { + let scaleFactor = newHeight / self.size.height + newWidth = self.size.width * scaleFactor + } + } else { + return self + } + + let newSize = CGSize(width: newWidth, height: newHeight) + + if newSize.width >= size.width && newSize.height >= size.height { + return self + } + + UIGraphicsBeginImageContextWithOptions(newSize, false, scale) + defer { UIGraphicsEndImageContext() } + draw(in: CGRect(origin: .zero, size: newSize)) + return UIGraphicsGetImageFromCurrentImageContext() ?? self + } +} @objc class ProcessImageOptions: NSObject { - private var visionImage: VisionImage + private var image: UIImage + private var confidence: CGFloat init( - visionImage: VisionImage + image: UIImage, + width: Int?, + height: Int?, + confidence: CGFloat ) { - self.visionImage = visionImage + self.image = image.scaledImage(width: width, height: height) + self.confidence = confidence + } + + func getImage() -> UIImage { + return image } - func getVisionImage() -> VisionImage { - return visionImage + func getConfidence() -> CGFloat { + return confidence } } diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift index ef8781c..7a8e382 100644 --- a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift @@ -1,47 +1,32 @@ import Foundation import Capacitor -import MLKitVision -import MLKitSegmentationSelfie @objc class ProcessImageResult: NSObject { - let segmentationMask: SegmentationMask + let image: UIImage - init(segmentationMask: SegmentationMask) { - self.segmentationMask = segmentationMask + init(image: UIImage) { + self.image = image } func toJSObject() -> JSObject { - let (maskResult, maskWidth, maskHeight) = createMaskResult(mask: segmentationMask) - var result = JSObject() - result["mask"] = maskResult - result["width"] = maskWidth - result["height"] = maskHeight - - return result - } - private func createMaskResult(mask: SegmentationMask) -> (JSArray, Int, Int) { - var result = JSArray() + if let data = image.pngData() { + do { + let path = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] + let name = "photo-"+UUID().uuidString+".png" + let url = path.appendingPathComponent(name) + try data.write(to: url) - let maskWidth = CVPixelBufferGetWidth(mask.buffer) - let maskHeight = CVPixelBufferGetHeight(mask.buffer) - - CVPixelBufferLockBaseAddress(mask.buffer, CVPixelBufferLockFlags.readOnly) - let maskBytesPerRow = CVPixelBufferGetBytesPerRow(mask.buffer) - var maskAddress = - CVPixelBufferGetBaseAddress(mask.buffer)!.bindMemory( - to: Float32.self, capacity: maskBytesPerRow * maskHeight) - - for _ in 0...(maskHeight - 1) { - for col in 0...(maskWidth - 1) { - // Gets the confidence of the pixel in the mask being in the foreground. - let foregroundConfidence: Float32 = maskAddress[col] - result.append(foregroundConfidence) + result["path"] = url.absoluteString + } catch { + result["path"] = "data:image/png;base64," + data.base64EncodedString() } - maskAddress += maskBytesPerRow / MemoryLayout.size + + result["width"] = Int(image.size.width) + result["height"] = Int(image.size.height) } - return (result, maskWidth, maskHeight) + return result } } diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift index a8d4f28..b536b62 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentation.swift @@ -9,41 +9,191 @@ import MLKitSegmentationSelfie self.plugin = plugin } - @objc func createVisionImageFromFilePath(_ path: String) -> VisionImage? { + @objc func createImageFromFilePath(_ path: String) -> UIImage? { guard let url = URL.init(string: path) else { return nil } if FileManager.default.fileExists(atPath: url.path) { - guard let image = UIImage.init(contentsOfFile: url.path) else { - return nil - } - return VisionImage.init( - image: image - ) + return UIImage.init(contentsOfFile: url.path) } else { return nil } } + enum ProcessError: Error { + case createImageBuffer + } + + private var segmenter: Segmenter? + @objc func processImage(_ options: ProcessImageOptions, completion: @escaping (ProcessImageResult?, Error?) -> Void) { - let visionImage = options.getVisionImage() + let image = options.getImage() + let threshold = options.getConfidence() + + let visionImage = VisionImage.init(image: image) + visionImage.orientation = image.imageOrientation let selfieSegmenterOptions: SelfieSegmenterOptions = SelfieSegmenterOptions() selfieSegmenterOptions.segmenterMode = .singleImage - selfieSegmenterOptions.shouldEnableRawSizeMask = true - let segmenter = Segmenter.segmenter( + segmenter = Segmenter.segmenter( options: selfieSegmenterOptions ) - do { - let mask: SegmentationMask = try segmenter.results( - in: visionImage - ) - let result = ProcessImageResult(segmentationMask: mask) - completion(result, nil) - } catch let error { - completion(nil, error) + segmenter?.process(visionImage) { mask, error in + self.segmenter = nil + + guard error == nil, let mask = mask else { + return completion(nil, error) + } + + do { + guard let imageBuffer = self.createImageBuffer(from: image) else { + throw ProcessError.createImageBuffer + } + + self.applySegmentationMask( + mask: mask, to: imageBuffer, threshold: threshold + ) + + let image = self.createImage(from: imageBuffer) + let result = ProcessImageResult(image: image) + + completion(result, nil) + } catch { + completion(nil, error) + } + } + } + + func createImageBuffer(from image: UIImage) -> CVImageBuffer? { + guard let cgImage = image.cgImage else { return nil } + let width = cgImage.width + let height = cgImage.height + + var buffer: CVPixelBuffer? + CVPixelBufferCreate( + kCFAllocatorDefault, + width, + height, + kCVPixelFormatType_32BGRA, + nil, + &buffer) + guard let imageBuffer = buffer else { return nil } + + let flags = CVPixelBufferLockFlags(rawValue: 0) + CVPixelBufferLockBaseAddress(imageBuffer, flags) + let baseAddress = CVPixelBufferGetBaseAddress(imageBuffer) + let colorSpace = CGColorSpaceCreateDeviceRGB() + let bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer) + let context = CGContext( + data: baseAddress, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: bytesPerRow, + space: colorSpace, + bitmapInfo: (CGImageAlphaInfo.premultipliedFirst.rawValue + | CGBitmapInfo.byteOrder32Little.rawValue)) + + if let context = context { + let rect = CGRect.init(x: 0, y: 0, width: width, height: height) + context.draw(cgImage, in: rect) + CVPixelBufferUnlockBaseAddress(imageBuffer, flags) + return imageBuffer + } else { + CVPixelBufferUnlockBaseAddress(imageBuffer, flags) + return nil + } + } + + // func createSampleBuffer(with imageBuffer: CVImageBuffer) -> CMSampleBuffer? { + // var timingInfo = CMSampleTimingInfo() + //// guard CMSampleBufferGetSampleTimingInfo(sampleBuffer, at: 0, timingInfoOut: &timingInfo) == 0 else { + //// return nil + //// } + // var outputSampleBuffer: CMSampleBuffer? + // var newFormatDescription: CMFormatDescription? + // CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: imageBuffer, formatDescriptionOut: &newFormatDescription) + // guard let formatDescription = newFormatDescription else { + // return nil + // } + // CMSampleBufferCreateReadyWithImageBuffer(allocator: nil, imageBuffer: imageBuffer, formatDescription: formatDescription, sampleTiming: &timingInfo, sampleBufferOut: &outputSampleBuffer) + // guard let buffer = outputSampleBuffer else { + // return nil + // } + // return buffer + // } + + func createImage( + from imageBuffer: CVImageBuffer + ) -> UIImage { + let ciImage = CIImage(cvPixelBuffer: imageBuffer) + let context = CIContext(options: nil) + let cgImage = context.createCGImage(ciImage, from: ciImage.extent)! + return UIImage(cgImage: cgImage) + } + + func applySegmentationMask( + mask: SegmentationMask, to imageBuffer: CVImageBuffer, threshold: CGFloat + ) { + let bgraBytesPerPixel = 4 + + assert( + CVPixelBufferGetPixelFormatType(imageBuffer) == kCVPixelFormatType_32BGRA, + "Image buffer must have 32BGRA pixel format type") + + let width = CVPixelBufferGetWidth(mask.buffer) + let height = CVPixelBufferGetHeight(mask.buffer) + assert(CVPixelBufferGetWidth(imageBuffer) == width, "Width must match") + assert(CVPixelBufferGetHeight(imageBuffer) == height, "Height must match") + + let writeFlags = CVPixelBufferLockFlags(rawValue: 0) + CVPixelBufferLockBaseAddress(imageBuffer, writeFlags) + CVPixelBufferLockBaseAddress(mask.buffer, CVPixelBufferLockFlags.readOnly) + + let maskBytesPerRow = CVPixelBufferGetBytesPerRow(mask.buffer) + var maskAddress = + CVPixelBufferGetBaseAddress(mask.buffer)!.bindMemory( + to: Float32.self, capacity: maskBytesPerRow * height) + + let imageBytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer) + var imageAddress = CVPixelBufferGetBaseAddress(imageBuffer)!.bindMemory( + to: UInt8.self, capacity: imageBytesPerRow * height) + + for _ in 0...(height - 1) { + for col in 0...(width - 1) { + let pixelOffset = col * bgraBytesPerPixel + let blueOffset = pixelOffset + let greenOffset = pixelOffset + 1 + let redOffset = pixelOffset + 2 + let alphaOffset = pixelOffset + 3 + + let confidence: CGFloat = CGFloat(maskAddress[col]) + + if confidence >= threshold { + let red = CGFloat(imageAddress[redOffset]) + let green = CGFloat(imageAddress[greenOffset]) + let blue = CGFloat(imageAddress[blueOffset]) + // let alpha = CGFloat(imageAddress[alphaOffset]) + + imageAddress[redOffset] = UInt8(red * confidence) + imageAddress[greenOffset] = UInt8(green * confidence) + imageAddress[blueOffset] = UInt8(blue * confidence) + imageAddress[alphaOffset] = UInt8(0xff) + } else { + imageAddress[redOffset] = UInt8(0x00) + imageAddress[greenOffset] = UInt8(0x00) + imageAddress[blueOffset] = UInt8(0x00) + imageAddress[alphaOffset] = UInt8(0x00) + } + } + + imageAddress += imageBytesPerRow / MemoryLayout.size + maskAddress += maskBytesPerRow / MemoryLayout.size } + + CVPixelBufferUnlockBaseAddress(imageBuffer, writeFlags) + CVPixelBufferUnlockBaseAddress(mask.buffer, CVPixelBufferLockFlags.readOnly) } } diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift index e0b20ea..8175d3b 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift @@ -8,9 +8,12 @@ import Capacitor @objc(SelfieSegmentationPlugin) public class SelfieSegmentationPlugin: CAPPlugin { public let tag = "SelfieSegmentation" + public let errorPathMissing = "path must be provided." public let errorLoadImageFailed = "image could not be loaded." + public let defaultConfidence: Float = 0.9 + private var implementation: SelfieSegmentation? override public func load() { @@ -23,12 +26,20 @@ public class SelfieSegmentationPlugin: CAPPlugin { return } - guard let visionImage = implementation?.createVisionImageFromFilePath(path) else { + let width = call.getInt("width") + let height = call.getInt("height") + + let confidence = call.getFloat("confidence", defaultConfidence) + + guard let image = implementation?.createImageFromFilePath(path) else { call.reject(errorLoadImageFailed) return } - let options = ProcessImageOptions(visionImage: visionImage) + let options = ProcessImageOptions(image: image, + width: width, + height: height, + confidence: CGFloat(confidence)) implementation?.processImage(options, completion: { result, error in if let error = error { @@ -36,8 +47,9 @@ public class SelfieSegmentationPlugin: CAPPlugin { call.reject(error.localizedDescription, nil, error) return } - if let result = result?.toJSObject() as? JSObject { - call.resolve(result) + + if let result = result { + call.resolve(result.toJSObject()) } }) } From d041d853ad5d0b578a427bb9b72f80472defb250 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Sun, 10 Sep 2023 11:50:01 +0200 Subject: [PATCH 11/12] feat(ios): fixed missing import --- .../ios/Plugin/Classes/ProcessImageOptions.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift index 41b55e9..a68f342 100644 --- a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageOptions.swift @@ -1,4 +1,5 @@ import Foundation +import UIKit extension UIImage { public func scaledImage(width: Int?, height: Int?) -> UIImage { From 462a68624fb1f88efa9ba3204390b10cbaac93e5 Mon Sep 17 00:00:00 2001 From: Robin Genz Date: Sun, 10 Sep 2023 22:23:20 +0200 Subject: [PATCH 12/12] refactoring --- .../SelfieSegmentationPlugin.java | 2 +- .../Plugin/Classes/ProcessImageResult.swift | 18 ++++++++---------- .../ios/Plugin/SelfieSegmentationPlugin.swift | 9 +++++++-- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java index 533c0fe..fd16c45 100644 --- a/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java +++ b/packages/selfie-segmentation/android/src/main/java/io/capawesome/capacitorjs/plugins/mlkit/selfiesegmentation/SelfieSegmentationPlugin.java @@ -16,7 +16,7 @@ public class SelfieSegmentationPlugin extends Plugin { public static final String ERROR_PROCESS_IMAGE_CANCELED = "processImage canceled."; public static final String ERROR_PATH_MISSING = "path must be provided."; - public static final String ERROR_LOAD_IMAGE_FAILED = "image could not be loaded."; + public static final String ERROR_LOAD_IMAGE_FAILED = "The image could not be loaded."; public static final float CONFIDENCE = 0.9f; diff --git a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift index 7a8e382..626a5a7 100644 --- a/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift +++ b/packages/selfie-segmentation/ios/Plugin/Classes/ProcessImageResult.swift @@ -8,21 +8,19 @@ import Capacitor self.image = image } - func toJSObject() -> JSObject { + func toJSObject() throws -> JSObject { var result = JSObject() if let data = image.pngData() { - do { - let path = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask)[0] - let name = "photo-"+UUID().uuidString+".png" - let url = path.appendingPathComponent(name) - try data.write(to: url) - - result["path"] = url.absoluteString - } catch { - result["path"] = "data:image/png;base64," + data.base64EncodedString() + let uniqueFileNameWithExtension = UUID().uuidString + ".png" + var directory = URL(fileURLWithPath: NSTemporaryDirectory()) + if let cachesDirectory = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first { + directory = cachesDirectory } + let url = directory.appendingPathComponent(uniqueFileNameWithExtension) + try data.write(to: url) + result["path"] = url.absoluteString result["width"] = Int(image.size.width) result["height"] = Int(image.size.height) } diff --git a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift index 8175d3b..1aa7551 100644 --- a/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift +++ b/packages/selfie-segmentation/ios/Plugin/SelfieSegmentationPlugin.swift @@ -10,7 +10,8 @@ public class SelfieSegmentationPlugin: CAPPlugin { public let tag = "SelfieSegmentation" public let errorPathMissing = "path must be provided." - public let errorLoadImageFailed = "image could not be loaded." + public let errorLoadImageFailed = "The image could not be loaded." + public let errorWriteFileFailed = "The result could not be created." public let defaultConfidence: Float = 0.9 @@ -49,7 +50,11 @@ public class SelfieSegmentationPlugin: CAPPlugin { } if let result = result { - call.resolve(result.toJSObject()) + do { + call.resolve(try result.toJSObject()) + } catch { + call.reject(self.errorWriteFileFailed) + } } }) }