Skip to content

Commit

Permalink
Remove tensorflow_text depenency on tf_hub library.
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 601915859
  • Loading branch information
tf-text-github-robot committed Jan 29, 2024
1 parent c96dbef commit b08cb1f
Show file tree
Hide file tree
Showing 318 changed files with 354 additions and 362 deletions.
14 changes: 0 additions & 14 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -54,20 +54,6 @@ http_archive(
],
)

# NOTE: according to
# https://docs.bazel.build/versions/master/external.html#transitive-dependencies
# we should list the transitive dependencies of @org_tensorflow_hub in this
# WORKSPACE file. Still, all of them are already listed by tf_workspace() which
# is called later in this file.
http_archive(
name = "org_tensorflow_hub",
strip_prefix = "hub-c83a2362abdad2baae67508aa17efb42bf7c7dd6",
sha256 = "b7f5e53605a3b2d6da827b817385d476017300e8e14cccc91b269c86d5f5a99f",
urls = [
"https://github.com/tensorflow/hub/archive/c83a2362abdad2baae67508aa17efb42bf7c7dd6.zip"
],
)

http_archive(
name = "org_tensorflow",
patch_args = ["-p1"],
Expand Down
13 changes: 7 additions & 6 deletions docs/api_docs/python/text/HubModuleSplitter.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,11 @@ as the support for ragged tensors and high-rank tensors.
The Hub module should be supported by `hub.load()
<https://www.tensorflow.org/hub/api_docs/python/hub/load>`_ If a v1 module, it
should have a graph variant with an empty set of tags; we consider that graph
variant to be the module and ignore everything else. The module should have a
signature named `default` that takes a <a href="../text.md"><code>text</code></a> input (a rank-1 tensor of
strings to split into pieces) and returns a dictionary of tensors, let's say
`output_dict`, such that:
variant to be the module and ignore everything else. The module should have a
signature named `default` that takes a
<a href="../text.md"><code>text</code></a> input (a rank-1 tensor of strings to
split into pieces) and returns a dictionary of tensors, let's say `output_dict`,
such that:

* `output_dict['num_pieces']` is a rank-1 tensor of integers, where
num_pieces[i] is the number of pieces that text[i] was split into.
Expand All @@ -68,7 +69,7 @@ class is not using them.

```
>>> HUB_MODULE = "https://tfhub.dev/google/zh_segmentation/1"
>>> segmenter = HubModuleSplitter(HUB_MODULE)
>>> segmenter = HubModuleSplitter(hub.resolve(HUB_MODULE))
>>> segmenter.split(["新华社北京"])
<tf.RaggedTensor [[b'\xe6\x96\xb0\xe5\x8d\x8e\xe7\xa4\xbe',
b'\xe5\x8c\x97\xe4\xba\xac']]>
Expand All @@ -78,7 +79,7 @@ You can also use this tokenizer to return the split strings and their offsets:

```
>>> HUB_MODULE = "https://tfhub.dev/google/zh_segmentation/1"
>>> segmenter = HubModuleSplitter(HUB_MODULE)
>>> segmenter = HubModuleSplitter(hub.resolve(HUB_MODULE))
>>> pieces, starts, ends = segmenter.split_with_offsets(["新华社北京"])
>>> print("pieces: %s starts: %s ends: %s" % (pieces, starts, ends))
pieces: <tf.RaggedTensor [[b'\xe6\x96\xb0\xe5\x8d\x8e\xe7\xa4\xbe',
Expand Down
4 changes: 2 additions & 2 deletions docs/api_docs/python/text/HubModuleTokenizer.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ tokenize() instead of the more general and less informatively named split().

```
>>> HUB_MODULE = "https://tfhub.dev/google/zh_segmentation/1"
>>> segmenter = HubModuleTokenizer(HUB_MODULE)
>>> segmenter = HubModuleTokenizer(hub.resolve(HUB_MODULE))
>>> segmenter.tokenize(["新华社北京"])
<tf.RaggedTensor [[b'\xe6\x96\xb0\xe5\x8d\x8e\xe7\xa4\xbe',
b'\xe5\x8c\x97\xe4\xba\xac']]>
Expand All @@ -56,7 +56,7 @@ You can also use this tokenizer to return the split strings and their offsets:

```
>>> HUB_MODULE = "https://tfhub.dev/google/zh_segmentation/1"
>>> segmenter = HubModuleTokenizer(HUB_MODULE)
>>> segmenter = HubModuleTokenizer(hub.resolve(HUB_MODULE))
>>> pieces, starts, ends = segmenter.tokenize_with_offsets(["新华社北京"])
>>> print("pieces: %s starts: %s ends: %s" % (pieces, starts, ends))
pieces: <tf.RaggedTensor [[b'\xe6\x96\xb0\xe5\x8d\x8e\xe7\xa4\xbe',
Expand Down
2 changes: 1 addition & 1 deletion oss_scripts/build_docs.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# coding=utf-8
# Copyright 2023 TF.Text Authors.
# Copyright 2024 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down
3 changes: 1 addition & 2 deletions oss_scripts/pip_package/setup.nightly.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# coding=utf-8
# Copyright 2023 TF.Text Authors.
# Copyright 2024 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -73,7 +73,6 @@ def finalize_options(self):
cmdclass={'install': InstallPlatlib},
distclass=BinaryDistribution,
install_requires=[
'tensorflow_hub>=0.15.0',
],
extras_require={
'tests': [
Expand Down
3 changes: 1 addition & 2 deletions oss_scripts/pip_package/setup.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# coding=utf-8
# Copyright 2023 TF.Text Authors.
# Copyright 2024 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -81,7 +81,6 @@ def finalize_options(self):
'tensorflow-macos>=2.15.0, <2.16; platform_machine == "arm64" and'
' platform_system == "Darwin"'
),
'tensorflow_hub>=0.15.0',
],
extras_require={
'tensorflow_cpu': [
Expand Down
6 changes: 3 additions & 3 deletions tensorflow_text/BUILD
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
# Placeholder: load py_library
# Placeholder: load py_test
load("@org_tensorflow//tensorflow/lite:build_def.bzl", "tflite_cc_shared_object")
load("//tensorflow_text:tftext.bzl", "py_tf_text_library")

# [internal] load build_test.bzl
load("@org_tensorflow//tensorflow/lite:build_def.bzl", "tflite_cc_shared_object")

# Visibility rules
package(
Expand Down Expand Up @@ -527,9 +526,10 @@ py_library(
srcs = ["python/ops/hub_module_splitter.py"],
deps = [
":splitter",
"@org_tensorflow_hub//tensorflow_hub",
# python/eager:monitoring tensorflow dep,
# python/ops:array_ops_stack tensorflow dep,
# python/ops/ragged:ragged_tensor tensorflow dep,
# python/saved_model:load tensorflow dep,
],
)

Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# coding=utf-8
# Copyright 2023 TF.Text Authors.
# Copyright 2024 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand Down
5 changes: 4 additions & 1 deletion tensorflow_text/core/kernels/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ load("//tensorflow_text:tftext.bzl", "tf_cc_library", "tflite_cc_library")
licenses(["notice"])

# Visibility rules
package(default_visibility = ["//visibility:public"])
package(default_visibility = [
"//abuse/hades:__subpackages__",
"//visibility:public",
])

exports_files(["LICENSE"])

Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/boise_offset_converter.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/boise_offset_converter.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter_kernel.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter_kernel.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter_test.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter_tflite.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/byte_splitter_tflite.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/constrained_sequence.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/constrained_sequence.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/darts_clone_trie_builder.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/darts_clone_trie_builder.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/darts_clone_trie_test.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/darts_clone_trie_wrapper.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/disjoint_set_forest.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/disjoint_set_forest_test.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/fast_bert_normalizer.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/fast_bert_normalizer_test.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/fast_bert_normalizer_tflite.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/fast_wordpiece_tokenizer.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion tensorflow_text/core/kernels/fast_wordpiece_tokenizer.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2023 TF.Text Authors.
// Copyright 2024 TF.Text Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
Loading

0 comments on commit b08cb1f

Please sign in to comment.