python

example/main.cpp

@@ -26,5 +26,12 @@ int main(int argc, char** argv) {
 
     vits.tts(phonemes, "./babylon_output.wav");
 
+    std::vector<int64_t> phoneme_ids = dp.g2p_tokens(text);
+
+    for (const auto& id : phoneme_ids) {
+        std::cout << id << " ";
+    }
+    std::cout << std::endl;
+
     return 0;
 }

include/babylon.h

@@ -15,6 +15,8 @@ BABYLON_EXPORT int babylon_g2p_init(const char* model_path, const char* language
 
 BABYLON_EXPORT char* babylon_g2p(const char* text);
 
+BABYLON_EXPORT int* babylon_g2p_tokens(const char* text);
+
 BABYLON_EXPORT void babylon_g2p_free(void);
 
 BABYLON_EXPORT int babylon_tts_init(const char* model_path);

src/babylon.cpp

@@ -37,6 +37,30 @@ extern "C" {
         return strdup(phonemes.c_str());
     }
 
+    BABYLON_EXPORT int* babylon_g2p_tokens(const char* text) {
+        if (dp == nullptr) {
+            std::cerr << "DeepPhonemizer session not initialized." << std::endl;
+            return nullptr;
+        }
+
+        std::vector<int64_t> phoneme_ids;
+        try {
+            phoneme_ids = dp->g2p_tokens(text);
+        } 
+        catch (const std::exception& e) {
+            std::cerr << e.what() << std::endl;
+        }
+
+        phoneme_ids.push_back(-1); // Sentinel value
+
+        int* phoneme_ids_arr = new int[phoneme_ids.size()];
+        for (size_t i = 0; i < phoneme_ids.size(); i++) {
+            phoneme_ids_arr[i] = phoneme_ids[i];
+        }
+
+        return phoneme_ids_arr;
+    }
+
     BABYLON_EXPORT void babylon_g2p_free(void) {
         delete dp;
     }

wrappers/babylon.py

@@ -16,6 +16,9 @@
 babylon_lib.babylon_g2p.argtypes = [ctypes.c_char_p]
 babylon_lib.babylon_g2p.restype = ctypes.c_char_p
 
+babylon_lib.babylon_g2p_tokens.argtypes = [ctypes.c_char_p]
+babylon_lib.babylon_g2p_tokens.restype = ctypes.POINTER(ctypes.c_int)
+
 babylon_lib.babylon_g2p_free.argtypes = []
 babylon_lib.babylon_g2p_free.restype = None
 
@@ -37,6 +40,19 @@ def g2p(text):
     result = babylon_lib.babylon_g2p(text.encode('utf-8'))
     return result.decode('utf-8')
 
+# Use G2P with tokens
+def g2p_tokens(text):
+    result_ptr = babylon_lib.babylon_g2p_tokens(text.encode('utf-8'))
+
+    # Convert the pointer to a Python list, stopping at -1
+    tokens = []
+    i = 0
+    while result_ptr[i] != -1:
+        tokens.append(result_ptr[i])
+        i += 1
+
+    return tokens
+
 # Free G2P resources
 def free_g2p():
     babylon_lib.babylon_g2p_free()
@@ -65,6 +81,9 @@ def free_tts():
         print('G2P initialized successfully')
         phonemes = g2p(sequence)
         print(f'Phonemes: {phonemes}')
+
+        tokens = g2p_tokens(sequence)
+        print(f'Tokens: {tokens}')
     else:
         print('Failed to initialize G2P')