Skip to content

Commit

Permalink
Add docker build scripts and enable aarch64 pip wheels.
Browse files Browse the repository at this point in the history
Also enable python 3.12 builds.

Usage:
```
bash oss_scripts/docker_builds.sh
```
or to build for a single python version,
```
PYTHON_VERSION=3.12 bash oss_scripts/docker_builds.sh
```
PiperOrigin-RevId: 692953635
  • Loading branch information
cantonios authored and tf-text-github-robot committed Nov 5, 2024
1 parent 9474dd9 commit 5180fd9
Show file tree
Hide file tree
Showing 16 changed files with 2,911 additions and 1,552 deletions.
1 change: 1 addition & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ python_init_repositories(
"3.9": "//oss_scripts/pip_package:requirements_lock_3_9.txt",
"3.10": "//oss_scripts/pip_package:requirements_lock_3_10.txt",
"3.11": "//oss_scripts/pip_package:requirements_lock_3_11.txt",
"3.12": "//oss_scripts/pip_package:requirements_lock_3_12.txt",
},
default_python_version = "system",
)
Expand Down
37 changes: 37 additions & 0 deletions oss_scripts/build.Dockerfile.aarch64
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Constructs the environment within which we will build the tensorflow-text
# pip wheels.

FROM linaro/tensorflow-arm64-build:2.16-multipython
LABEL maintainer="TensorFlow-Text team <[email protected]>"

ARG PYTHON_VERSION

ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHON_BIN_PATH=/usr/bin/python${PYTHON_VERSION}

# Install supplementary Python interpreters
RUN ln -s ${PYTHON_BIN_PATH} /usr/local/bin/python && \
ln -s ${PYTHON_BIN_PATH} /usr/local/bin/python3 && \
ln -s ${PYTHON_BIN_PATH} /usr/bin/python

RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -yqq \
apt-utils \
build-essential \
checkinstall \
libffi-dev

# Install pip dependencies needed for tensorflow-text
RUN --mount=type=cache,target=/root/.cache \
${PYTHON_BIN_PATH} -m pip install -U pip && \
${PYTHON_BIN_PATH} -m pip install -U \
absl-py \
auditwheel \
etils[epath] \
patchelf \
setuptools \
twine \
wheel;

WORKDIR "/tmp/tensorflow_text"
32 changes: 32 additions & 0 deletions oss_scripts/build.Dockerfile.x86_64
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Constructs the environment within which we will build the tensorflow-text
# pip wheels.

ARG PYTHON_VERSION
FROM tensorflow/build:2.18-python${PYTHON_VERSION}
LABEL maintainer="TensorFlow-Text team <[email protected]>"

ENV DEBIAN_FRONTEND=noninteractive

# Install supplementary Python interpreters
RUN mkdir /tmp/python
RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -yqq \
apt-utils \
build-essential \
checkinstall \
libffi-dev

# Install pip dependencies needed for tensorflow text.
RUN --mount=type=cache,target=/root/.cache \
python${PYTHON_VERSION} -m pip install -U pip && \
python${PYTHON_VERSION} -m pip install -U \
absl-py \
auditwheel \
etils[epath] \
patchelf \
setuptools \
twine \
wheel;

WORKDIR "/tmp/tensorflow_text"
14 changes: 14 additions & 0 deletions oss_scripts/build_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright 2024 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r"""Tool to generate external api_docs.
python build_docs.py --output_dir=/tmp/text_api
Expand Down
13 changes: 3 additions & 10 deletions oss_scripts/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,10 @@ if [[ $(pip show tensorflow) == *tensorflow* ]] ||
echo 'Using installed tensorflow.'
else
echo 'Installing tensorflow.'
if is_macos; then
# Only Apple Silicon will be installed with tensorflow-macos.
if [[ x"$(arch)" == x"arm64" ]]; then
pip install tensorflow-macos==2.16.1
else
pip install tensorflow==2.16.1
fi
if [[ "$IS_NIGHTLY" == "nightly" ]]; then
pip install tf-nightly
else
pip install tensorflow==2.16.1
pip install tensorflow==2.18.0
fi
fi

Expand All @@ -58,8 +53,6 @@ curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelrc -o
# This line breaks Windows builds, so we remove it.
sed -i -e 's/build --noincompatible_remove_legacy_whole_archive//' .bazelrc

# the next line is temporary to aid in transition
write_to_bazelrc "build:manylinux2010 --config=release_cpu_linux"
write_to_bazelrc "build:manylinux2014 --config=release_cpu_linux"

if (which python3) | grep -q "python3"; then
Expand Down
49 changes: 49 additions & 0 deletions oss_scripts/docker_builds.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/bin/bash
# This script builds a docker container, and pip wheels for all supported
# Python versions. Run from the root tensorflow-text directory:
#
# ./oss_scripts/docker_builds.sh

set -e -x

# If a specific PYTHON_VERSION is specified, only build that one.
# Otherwisebuild all supported versions.
python_versions=("3.9" "3.10" "3.11" "3.12")
if [[ ! -z ${PYTHON_VERSION+x} ]]; then
python_versions=("$PYTHON_VERSION")
fi

# Clean previous images.
for python_version in ${python_versions[@]}
do
docker rmi -f tensorflow_text:${python_version} || true
done

arch=$(uname -m)
build_args=()
if [ "$arch" == "x86_64" ]; then
build_args+=("--config=release_cpu_linux")
build_args+=("[email protected]_config_platform//:platform")
auditwheel_platform="manylinux2014_x86_64"
elif [ "$arch" == "aarch64" ]; then
build_args+=("--crosstool_top=@ml2014_aarch64_config_aarch64//crosstool:toolchain")
auditwheel_platform="manylinux2014_aarch64"
fi

# Build wheel for each Python version.
for python_version in ${python_versions[@]}
do
DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache \
--build-arg HERMETIC_PYTHON_VERSION=${python_version} --build-arg PYTHON_VERSION=${python_version} \
-t tensorflow_text:${python_version} - < "oss_scripts/build.Dockerfile.${arch}"

docker run --rm -a stdin -a stdout -a stderr \
--env PYTHON_VERSION=${python_version} \
--env HERMETIC_PYTHON_VERSION=${python_version} \
--env BUILD_ARGS=${build_args} \
--env AUDITWHEEL_PLATFORM=${auditwheel_platform} \
--env IS_NIGHTLY=${IS_NIGHTLY} \
-v $PWD:/tmp/tensorflow_text \
--name tensorflow_text tensorflow_text:${python_version} \
bash oss_scripts/run_build.sh
done
2 changes: 2 additions & 0 deletions oss_scripts/pip_package/requirements.in
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# ai-edge-litert
setuptools==70.0.0
tensorflow
tf-keras
tensorflow-datasets
Expand Down
Loading

0 comments on commit 5180fd9

Please sign in to comment.