Skip to content

Commit

Permalink
Merge pull request #1323 from tensorflow/test_692953635
Browse files Browse the repository at this point in the history
Add docker build scripts and enable aarch64 pip wheels.
  • Loading branch information
cantonios authored Nov 6, 2024
2 parents 9474dd9 + 5180fd9 commit 1779b3a
Show file tree
Hide file tree
Showing 16 changed files with 2,911 additions and 1,552 deletions.
1 change: 1 addition & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ python_init_repositories(
"3.9": "//oss_scripts/pip_package:requirements_lock_3_9.txt",
"3.10": "//oss_scripts/pip_package:requirements_lock_3_10.txt",
"3.11": "//oss_scripts/pip_package:requirements_lock_3_11.txt",
"3.12": "//oss_scripts/pip_package:requirements_lock_3_12.txt",
},
default_python_version = "system",
)
Expand Down
37 changes: 37 additions & 0 deletions oss_scripts/build.Dockerfile.aarch64
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Constructs the environment within which we will build the tensorflow-text
# pip wheels.

FROM linaro/tensorflow-arm64-build:2.16-multipython
LABEL maintainer="TensorFlow-Text team <[email protected]>"

ARG PYTHON_VERSION

ENV DEBIAN_FRONTEND=noninteractive
ENV PYTHON_BIN_PATH=/usr/bin/python${PYTHON_VERSION}

# Install supplementary Python interpreters
RUN ln -s ${PYTHON_BIN_PATH} /usr/local/bin/python && \
ln -s ${PYTHON_BIN_PATH} /usr/local/bin/python3 && \
ln -s ${PYTHON_BIN_PATH} /usr/bin/python

RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -yqq \
apt-utils \
build-essential \
checkinstall \
libffi-dev

# Install pip dependencies needed for tensorflow-text
RUN --mount=type=cache,target=/root/.cache \
${PYTHON_BIN_PATH} -m pip install -U pip && \
${PYTHON_BIN_PATH} -m pip install -U \
absl-py \
auditwheel \
etils[epath] \
patchelf \
setuptools \
twine \
wheel;

WORKDIR "/tmp/tensorflow_text"
32 changes: 32 additions & 0 deletions oss_scripts/build.Dockerfile.x86_64
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Constructs the environment within which we will build the tensorflow-text
# pip wheels.

ARG PYTHON_VERSION
FROM tensorflow/build:2.18-python${PYTHON_VERSION}
LABEL maintainer="TensorFlow-Text team <[email protected]>"

ENV DEBIAN_FRONTEND=noninteractive

# Install supplementary Python interpreters
RUN mkdir /tmp/python
RUN --mount=type=cache,target=/var/cache/apt \
apt update && \
apt install -yqq \
apt-utils \
build-essential \
checkinstall \
libffi-dev

# Install pip dependencies needed for tensorflow text.
RUN --mount=type=cache,target=/root/.cache \
python${PYTHON_VERSION} -m pip install -U pip && \
python${PYTHON_VERSION} -m pip install -U \
absl-py \
auditwheel \
etils[epath] \
patchelf \
setuptools \
twine \
wheel;

WORKDIR "/tmp/tensorflow_text"
14 changes: 14 additions & 0 deletions oss_scripts/build_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.

# Copyright 2024 TF.Text Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

r"""Tool to generate external api_docs.
python build_docs.py --output_dir=/tmp/text_api
Expand Down
13 changes: 3 additions & 10 deletions oss_scripts/configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,10 @@ if [[ $(pip show tensorflow) == *tensorflow* ]] ||
echo 'Using installed tensorflow.'
else
echo 'Installing tensorflow.'
if is_macos; then
# Only Apple Silicon will be installed with tensorflow-macos.
if [[ x"$(arch)" == x"arm64" ]]; then
pip install tensorflow-macos==2.16.1
else
pip install tensorflow==2.16.1
fi
if [[ "$IS_NIGHTLY" == "nightly" ]]; then
pip install tf-nightly
else
pip install tensorflow==2.16.1
pip install tensorflow==2.18.0
fi
fi

Expand All @@ -58,8 +53,6 @@ curl https://raw.githubusercontent.com/tensorflow/tensorflow/master/.bazelrc -o
# This line breaks Windows builds, so we remove it.
sed -i -e 's/build --noincompatible_remove_legacy_whole_archive//' .bazelrc

# the next line is temporary to aid in transition
write_to_bazelrc "build:manylinux2010 --config=release_cpu_linux"
write_to_bazelrc "build:manylinux2014 --config=release_cpu_linux"

if (which python3) | grep -q "python3"; then
Expand Down
49 changes: 49 additions & 0 deletions oss_scripts/docker_builds.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/bin/bash
# This script builds a docker container, and pip wheels for all supported
# Python versions. Run from the root tensorflow-text directory:
#
# ./oss_scripts/docker_builds.sh

set -e -x

# If a specific PYTHON_VERSION is specified, only build that one.
# Otherwisebuild all supported versions.
python_versions=("3.9" "3.10" "3.11" "3.12")
if [[ ! -z ${PYTHON_VERSION+x} ]]; then
python_versions=("$PYTHON_VERSION")
fi

# Clean previous images.
for python_version in ${python_versions[@]}
do
docker rmi -f tensorflow_text:${python_version} || true
done

arch=$(uname -m)
build_args=()
if [ "$arch" == "x86_64" ]; then
build_args+=("--config=release_cpu_linux")
build_args+=("[email protected]_config_platform//:platform")
auditwheel_platform="manylinux2014_x86_64"
elif [ "$arch" == "aarch64" ]; then
build_args+=("--crosstool_top=@ml2014_aarch64_config_aarch64//crosstool:toolchain")
auditwheel_platform="manylinux2014_aarch64"
fi

# Build wheel for each Python version.
for python_version in ${python_versions[@]}
do
DOCKER_BUILDKIT=1 docker build --progress=plain --no-cache \
--build-arg HERMETIC_PYTHON_VERSION=${python_version} --build-arg PYTHON_VERSION=${python_version} \
-t tensorflow_text:${python_version} - < "oss_scripts/build.Dockerfile.${arch}"

docker run --rm -a stdin -a stdout -a stderr \
--env PYTHON_VERSION=${python_version} \
--env HERMETIC_PYTHON_VERSION=${python_version} \
--env BUILD_ARGS=${build_args} \
--env AUDITWHEEL_PLATFORM=${auditwheel_platform} \
--env IS_NIGHTLY=${IS_NIGHTLY} \
-v $PWD:/tmp/tensorflow_text \
--name tensorflow_text tensorflow_text:${python_version} \
bash oss_scripts/run_build.sh
done
2 changes: 2 additions & 0 deletions oss_scripts/pip_package/requirements.in
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# ai-edge-litert
setuptools==70.0.0
tensorflow
tf-keras
tensorflow-datasets
Expand Down
Loading

0 comments on commit 1779b3a

Please sign in to comment.