diff --git a/CHANGES.md b/CHANGES.md index ccf8a79..af6c599 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,6 +16,10 @@ * base: update to Debian 12. by @ericonr in https://github.com/cnpem/epics-in-docker/pull/84 * Refer to up to date README for new/updated `RUNTIME_PACKAGES`. +* Prune unused artifacts from non-static builds by @henriquesimoes in + https://github.com/cnpem/epics-in-docker/pull/59 + * Refer to README for instructions on how to configure this procedure when + needed. ## v0.12.0 diff --git a/Dockerfile b/Dockerfile index 29dd12c..71dcc4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,9 +42,18 @@ RUN ln -s ${ENTRYPOINT} ./entrypoint ENTRYPOINT ["./entrypoint"] +FROM build-image AS pruned-build + +ARG APP_DIRS +ARG RUNDIR +ARG SKIP_PRUNE + +RUN if [ "$SKIP_PRUNE" != 1 ]; then lnls-prune-artifacts ${APP_DIRS} ${RUNDIR}; fi + + FROM base AS no-build -COPY --from=build-image /opt /opt +COPY --from=pruned-build /opt /opt FROM build-image AS build-stage @@ -70,11 +79,15 @@ RUN rm -rf .git/ FROM build-stage AS dynamic-build ARG JOBS=1 +ARG APP_DIRS ARG RUNDIR ARG SKIP_TESTS +ARG SKIP_PRUNE RUN make distclean && make -j ${JOBS} && make $([ "$SKIP_TESTS" != 1 ] && echo runtests) && make clean && make -C ${RUNDIR} +RUN if [ "$SKIP_PRUNE" != 1 ]; then lnls-prune-artifacts ${APP_DIRS} ${PWD} ${RUNDIR}; fi + FROM base AS dynamic-link diff --git a/README.md b/README.md index 6a21b7d..427e5d7 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,24 @@ services: By default, the IOC will be built with statically linked EPICS libraries. If you **need** to link them dynamically, you must define the build target as -`dynamic-link`. This will increase the resulting image size, since unused -dependencies will also be copied. +`dynamic-link`. + +For non-static builds, an automatic pruning step is executed to make the IOC +image size smaller by removing unused artifacts. This step preserves all ELF +executables inside `RUNDIR` and the IOC repository, as well as their dependent +EPICS modules based on the linkage information. Source code, GUI files, and +other directories in used modules are assumed not to be needed at runtime and +also removed. Additional paths can be provided in `args` under the `APP_DIRS` +key to extend the list of where used applications and shared libraries are, +including any `dlopen(3)`ed libraries. If the resulting image fails at runtime, +a careful look at error messages might provide clues for directories to add to +`APP_DIRS`; if that isn't possible, or if it's believed that the directories +added to `APP_DIRS` should have been detected automatically, please open an +issue. Please do so as well if the pruning step errors out during the build. In +case it is necessary to build a working image before these issues can be fixed, +the pruning step can be skipped entirely by setting the `SKIP_PRUNE=1` +argument; note that this is highly discouraged, because it increases the image +size significantly. The resulting image contains a standard IOC run script, `lnls-run`, which will be run inside `RUNDIR` and will launch the container's command under procServ, diff --git a/base/Dockerfile b/base/Dockerfile index a012812..4cdf643 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -35,7 +35,6 @@ RUN apt update -y && \ ca-certificates COPY lnls-get-n-unpack.sh /usr/local/bin/lnls-get-n-unpack -COPY lnls-run.sh /usr/local/bin/lnls-run ENV EPICS_IN_DOCKER=/opt/epics-in-docker RUN mkdir $EPICS_IN_DOCKER @@ -68,3 +67,6 @@ RUN $EPICS_IN_DOCKER/install_motor.sh ARG DEBIAN_VERSION COPY opcua_versions.sh install_opcua.sh $EPICS_IN_DOCKER RUN $EPICS_IN_DOCKER/install_opcua.sh + +COPY lnls-prune-artifacts.sh /usr/local/bin/lnls-prune-artifacts +COPY lnls-run.sh /usr/local/bin/lnls-run diff --git a/base/install_modules.sh b/base/install_modules.sh index de211f9..91aec84 100755 --- a/base/install_modules.sh +++ b/base/install_modules.sh @@ -96,6 +96,7 @@ echo PYTHON=python3 >> pyDevSup/configure/CONFIG_SITE install_module pyDevSup PYDEVSUP " EPICS_BASE " +echo 'python3*/linux*/' > pyDevSup/.lnls-keep-paths mkdir snmp cd snmp diff --git a/base/lnls-prune-artifacts.sh b/base/lnls-prune-artifacts.sh new file mode 100755 index 0000000..81f7f59 --- /dev/null +++ b/base/lnls-prune-artifacts.sh @@ -0,0 +1,203 @@ +#!/usr/bin/env bash + +set -Eeu + +# Filter out from the $1 list of paths any exact match of a parent directory +# from any path in the $2 exclude list. +# +# Both list are treated as newline-separated strings, and must consist +# exclusively of absolute paths. +filter_out_paths() { + list="$1" + exclude_list="$2" + + while read -r path; do + if [ "${path:0:1}" != "/" ]; then + >&2 echo "error: filter_out_paths() expects absolute paths, but got '$path'" + exit 1 + fi + + while [ "$path" != "/" ]; do + list=$(echo "$list" | grep -xv "$path") + + path=$(dirname "$path") + done + done <<< "$exclude_list" + + echo "$list" +} + +find_elf_executables() { + targets=$@ + + # Loop on entire lines to properly handle filenames with spaces + while read -r executable; do + read -r -N 4 magic < "$executable" + + # Output only ELF binaries + if [ "$magic" = $'\x7fELF' ]; then + echo $executable + fi + done < <(find $targets -type f -executable) +} + +find_shared_libraries() { + elf_files=$(find_elf_executables $@) + + echo "$elf_files" | grep -E "\.so(.[0-9]+)*$" | sort -u +} + +find_linked_libraries() { + executables=$(find_elf_executables $@) + + # Depend on the glibc-specific behavior of supporting multiple executables + # to be queried at once + linked=$(ldd $executables 2>/dev/null | grep '=>') + + # We grep out not found libraries, since they cannot be kept if we don't + # know where they are. + # + # Final binary may be actually runnable, since rpath of another binary may + # pull those not-found libraries + found="$(echo "$linked" | grep -v "not found")" + + # Get their full path + libs=$(echo "$found" | cut -d' ' -f 3) + + echo "$libs" | sort -u +} + +get_all_epics_modules() { + release_defs=$(grep = ${EPICS_RELEASE_FILE} | cut -d'=' -f 2) + + echo "$release_defs" | grep $EPICS_MODULES_PATH + + echo $EPICS_BASE_PATH +} + +get_used_epics_modules() { + linked_libs=$(find_linked_libraries $@) + all_modules=$(get_all_epics_modules) + + unused_modules=$(filter_out_paths "$all_modules" "$linked_libs") + + filter_out_paths "$all_modules" "$unused_modules" +} + +# Traverse ancestor directories of each provided path, and concatenate all +# their .lnls-keep-paths defined entries as absolute paths. +get_defined_paths_to_keep() { + for path; do + if [ "${path:0:1}" != "/" ]; then + >&2 echo "error: get_defined_paths_to_keep() expects absolute paths, but got '$path'" + exit 1 + fi + + while true; do + keep_path_file="$path/.lnls-keep-paths" + + if [ -f "$keep_path_file" ]; then + keep_paths=$(cat "$keep_path_file") + + for keep_path in $keep_paths; do + # output it as an absolute path + realpath "$path"/$keep_path + done + fi + + [ "$path" == "/" ] && break + + path=$(dirname $path) + done + done | sort -u +} + +prune_directories() { + local targets="$1" + local keep_paths="$2" + + remove_dirs=$(filter_out_paths "$targets" "$keep_paths") + + while read -r remove_dir; do + # if we already removed it because of its parent directory, move on to + # the next. + [ ! -d "$remove_dir" ] && continue + + size=$(du -hs "$remove_dir" | cut -f 1) + + echo "Removing directory '$remove_dir' ($size)..." + rm -rf "$remove_dir" + done <<< "$remove_dirs" +} + +prune_module_directories() { + module=$1 + + module_dirs=$(find $module -type d) + keep_paths=$(cat << EOF +$(find_shared_libraries $module) +$(find $module -type f -regex ".*\.\(cmd\|db\|template\|req\|substitutions\)" -printf "%h\n" | sort -u) +$(get_defined_paths_to_keep $module) +EOF + ) + + prune_directories "$module_dirs" "$keep_paths" +} + +clean_up_epics_modules() { + targets=$(echo $@ | sed -E "s|\s+|\n|g") + + all_modules=$(get_all_epics_modules) + used_modules=$(get_used_epics_modules $targets) + + keep_paths=$(printf "$targets\n$used_modules") + prune_directories "$all_modules" "$keep_paths" + + # Filter out targets to provide a way to disable module pruning in special + # cases + prune_dirs=$(filter_out_paths "$used_modules" "$targets") + + for dir in $prune_dirs; do + echo "Pruning module '$dir'..." + prune_module_directories $dir + done +} + +remove_static_libraries() { + for target; do + libs=$(find $target -type f -name *.a) + + if [ -n "$libs" ]; then + size=$(du -hsc $libs | tail -n 1 | cut -f 1) + + echo "Removing static libraries from $target ($size)" + rm -f $libs + fi + done +} + +remove_unused_shared_libraries() { + target_libs=$(find_shared_libraries $@) + linked_libs=$(find_linked_libraries $@) + remove_libs=$(find_shared_libraries /opt /usr/local) + + for lib in $target_libs $linked_libs; do + remove_libs=$(echo "$remove_libs" | grep -vx $lib) + done + + keep_paths=$(get_defined_paths_to_keep $remove_libs) + + for lib in $remove_libs; do + # if library is not found inside any $keep_dirs, remove it + if find $keep_paths -path "$lib" -exec false {} +; then + size=$(du -hs $lib | cut -f 1) + + echo "Removing shared library '$lib' ($size)" + rm -f ${lib%.so*}.so* + fi + done +} + +clean_up_epics_modules $@ +remove_static_libraries /opt /usr/local +remove_unused_shared_libraries $@ diff --git a/images/docker-compose-mca.yml b/images/docker-compose-mca.yml index 66f3c48..5d571cb 100644 --- a/images/docker-compose-mca.yml +++ b/images/docker-compose-mca.yml @@ -8,6 +8,6 @@ services: labels: org.opencontainers.image.source: https://github.com/cnpem/epics-in-docker args: - REPONAME: mca + APP_DIRS: /opt/epics/modules/mca RUNDIR: /opt/epics/modules/mca/iocBoot/iocAmptek RUNTIME_PACKAGES: libpcap0.8 libnet1 libusb-1.0-0 diff --git a/images/docker-compose-motorpigcs2.yml b/images/docker-compose-motorpigcs2.yml index 26011df..c89eeb3 100644 --- a/images/docker-compose-motorpigcs2.yml +++ b/images/docker-compose-motorpigcs2.yml @@ -8,5 +8,5 @@ services: labels: org.opencontainers.image.source: https://github.com/cnpem/epics-in-docker args: - REPONAME: motorpigcs2 + APP_DIRS: /opt/epics/modules/motor/modules/motorPIGCS2 RUNDIR: /opt/epics/modules/motor/modules/motorPIGCS2/iocs/pigcs2IOC/iocBoot/iocPIGCS2 diff --git a/images/docker-compose-opcua.yml b/images/docker-compose-opcua.yml index 95dfc34..89c478e 100644 --- a/images/docker-compose-opcua.yml +++ b/images/docker-compose-opcua.yml @@ -8,6 +8,6 @@ services: labels: org.opencontainers.image.source: https://github.com/cnpem/epics-in-docker args: - REPONAME: opcua + APP_DIRS: /opt/epics/modules/opcua RUNDIR: /opt/epics/modules/opcua/iocBoot/iocUaDemoServer RUNTIME_PACKAGES: libxml2 libssl3 diff --git a/images/docker-compose-pvagw.yml b/images/docker-compose-pvagw.yml index 5d819d7..775b781 100644 --- a/images/docker-compose-pvagw.yml +++ b/images/docker-compose-pvagw.yml @@ -8,5 +8,7 @@ services: labels: org.opencontainers.image.source: https://github.com/cnpem/epics-in-docker args: + APP_DIRS: /opt/epics/modules/p4p RUNDIR: /opt/epics/modules/p4p/bin/linux-x86_64 - RUNTIME_PACKAGES: python3-numpy python3-ply + ENTRYPOINT: ./pvagw + RUNTIME_PACKAGES: python3-numpy python3-ply libevent-pthreads-2.1-7