Skip to content

Commit

Permalink
Update python calls for FWP tokenization
Browse files Browse the repository at this point in the history
  • Loading branch information
broken committed May 6, 2022
1 parent 65e5f1d commit e3c3870
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions tensorflow_text/python/ops/fast_sentencepiece_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def tokenize(self, inputs):
# normal.
(output_values, row_splits) = (
gen_fast_sentencepiece_tokenizer
.tf_text_fast_sentencepiece_tokenize(
.tf_sentencepiece_tokenize_op(
self._converted_model, input_tensor, 0, 0, self._add_bos,
self._add_eos, self._reverse))
tokens = tf.RaggedTensor.from_nested_row_splits(
Expand Down Expand Up @@ -122,7 +122,7 @@ def detokenize(self, input): # pylint: disable=redefined-builtin
tokens = self.detokenize(input_tensor.values)
return input_tensor.with_values(tokens)
else:
return gen_fast_sentencepiece_tokenizer.tf_text_fast_sentencepiece_detokenize(
return gen_fast_sentencepiece_tokenizer.tf_sentencepiece_detokenize_op(
self._converted_model_detokenizer, input_tensor.flat_values,
input_tensor.row_splits)
else:
Expand Down

0 comments on commit e3c3870

Please sign in to comment.