make-dist

#!/bin/sh
### make-dist: create an Emacs distribution tar file from current srcdir

## Copyright (C) 1995, 1997-1998, 2000-2018 Free Software Foundation,
## Inc.

## This file is part of GNU Emacs.

## GNU Emacs is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.

## GNU Emacs is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.

## You should have received a copy of the GNU General Public License
## along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.

### Commentary:

## This basically creates a duplicate directory structure, and then
## hard links into it only those files that should be distributed.
## This means that if you add a file with an odd name, you should make
## sure that this script will include it.

### Code:

progname="$0"

### Exit if a command fails.
#set -e

### Print out each line we read, for debugging's sake.
#set -v

LANGUAGE=C
LC_ALL=C
LC_MESSAGES=
LANG=
export LANGUAGE LC_ALL LC_MESSAGES LANG

## Remove unnecessary restrictions on file access.
umask 022

update=yes
check=yes
clean_up=no
make_tar=no
default_gzip=gzip
newer=""
with_tests=no
changelog=yes
verbose=no

while [ $# -gt 0 ]; do
  case "$1" in
    ## This option tells make-dist to delete the staging directory
    ## when done.  It is useless to use this unless you make a tar file.
    "--clean-up" )
      clean_up=yes
    ;;
    ## This option tells make-dist to make a tar file.
    "--tar" )
      make_tar=yes
    ;;
    ## This option tells make-dist not to recompile or do analogous things.
    "--no-update" )
      update=no
    ;;
    ## This option says don't check for bad file names, etc.
    "--no-check" )
      check=no
    ;;
    "--no-changelog" )
      changelog=no
    ;;
    ## This option tells make-dist to make the distribution normally, then
    ## remove all files older than the given timestamp file.  This is useful
    ## for creating incremental or patch distributions.
    "--newer")
      newer="$2"
      new_extension=".new"
      shift
    ;;
    ## This option tells make-dist to use 'bzip2' instead of gzip.
    "--bzip2")
      default_gzip="bzip2"
    ;;
    ## Same with xz.
    "--xz")
      default_gzip="xz"
    ;;
    "--no-compress")
      default_gzip="cat"
    ;;

    "--snapshot")
      clean_up=yes
      make_tar=yes
      update=no
     ;;

    ## Include the test/ directory.
    ## This option is mainly for the hydra build server.
    "--tests")
      with_tests=yes
     ;;

    "--verbose")
      verbose=yes
     ;;

    "--help")
      printf '%s\n' "Usage: ${progname} [options]"
      echo ""
      echo "  --bzip2	use bzip2 instead of gzip"
      echo "  --clean-up	delete staging directories when done"
      echo "  --xz		use xz instead of gzip"
      echo "  --no-compress	don't compress"
      echo "  --newer=TIME	don't include files older than TIME"
      echo "  --no-check	don't check for bad file names etc."
      echo "  --no-update	don't recompile or do analogous things"
      echo "  --no-changelog	don't generate the top-level ChangeLog"
      echo "  --snapshot	same as --clean-up --no-update --tar --no-check"
      echo "  --tar		make a tar file"
      echo "  --tests	include the test/ directory"
      echo "  --verbose		noisier output"
      echo ""
      exit 0
    ;;

    * )
      printf '%s\n' "${progname}: Unrecognized argument: $1" >&2
      exit 1
    ;;
  esac
  shift
done

### Make sure we're running in the right place.
if [ ! -d src -o ! -f src/lisp.h -o ! -d lisp -o ! -f lisp/subr.el ]; then
  printf '%s\n' "${progname}: Can't find 'src/lisp.h' and 'lisp/subr.el'." >&2
  printf '%s\n' "${progname} must be run in the top directory of the Emacs" >&2
  printf '%s\n' "distribution tree.  cd to that directory and try again." >&2
  exit 1
fi

### Find where to run Emacs.
### (Accept only absolute file names.)
if [ $update = yes ];
then
  if [ -f src/emacs ];
  then
    EMACS=`pwd`/src/emacs
  else
    case $EMACS in
      /*) ;;
      *)
	if [ ! -f "$EMACS" ]; then
	  printf '%s\n' "$0: You must set the EMACS environment variable " \
	       "to an absolute file name." 2>&1
	  exit 1
	fi;;
    esac
  fi
fi

### Find out which version of Emacs this is.
version=`
  sed -n 's/^AC_INIT(GNU Emacs,[	 ]*\([^	 ,)]*\).*/\1/p' <configure.ac
` || version=
if [ ! "${version}" ]; then
  printf '%s\n' \
    "${progname}: can't find current Emacs version in './src/emacs.c'" >&2
  exit 1
fi

echo Version number is $version

if [ $update = yes ]; then
  if ! grep -q "tree holds version  *${version}" README; then
    echo "WARNING: README has the wrong version number"
    echo "Consider running M-x set-version from admin/admin.el"
    sleep 5
  fi
fi

### Make sure we don't already have a directory  emacs-${version}.

emacsname="emacs-${version}${new_extension}"

if [ -d ${emacsname} ]
then
  echo Directory "${emacsname}" already exists >&2
  exit 1
fi

### Make sure the subdirectory is available.
tempparent="make-dist.tmp.$$"
if [ -d ${tempparent} ]; then
  printf '%s\n' "${progname}: staging directory '${tempparent}' already exists.
Perhaps a previous invocation of '${progname}' failed to clean up after
itself.  Check that directories whose names are of the form
'make-dist.tmp.NNNNN' don't contain any important information, remove
them, and try again." >&2
  exit 1
fi

if [ $check = yes ]; then

  echo "Sanity checking (use --no-check to disable this)..."

  error=no

  ls -1 lisp/[a-zA-Z]*.el lisp/[a-z]*/[a-zA-Z0-9]*.el \
	lisp/[a-z]*/[a-z]*/[a-zA-Z0-9]*.el \
	lisp/[a-z]*/[a-z]*/[a-z]*/[a-zA-Z0-9]*.el > /tmp/el

  ls -1 lisp/[a-zA-Z]*.elc lisp/[a-z]*/[a-zA-Z0-9]*.elc \
	lisp/[a-z]*/[a-z]*/[a-zA-Z0-9]*.elc \
	lisp/[a-z]*/[a-z]*/[a-z]*/[a-zA-Z0-9]*.elc > /tmp/elc

  ## Check for .elc files with no corresponding .el file.
  sed 's/\.el$/.elc/' /tmp/el > /tmp/elelc

  bogosities=`comm -13 /tmp/elelc /tmp/elc`
  if [ x"${bogosities}" != x"" ]; then
    error=yes
    echo "The following .elc files have no corresponding .el files:"
    echo "${bogosities}"
  fi

  ### Check for .el files with no corresponding .elc file.
  sed 's/\.elc$/.el/' /tmp/elc > /tmp/elcel
  losers=`comm -23 /tmp/el /tmp/elcel`

  bogosities=
  while read elc; do
      el=`echo $elc | sed 's/c$//'`
      [ -r $el ] || continue
      [ $elc -nt $el ] || bogosities="$bogosities $elc"
  done < /tmp/elc

  if [ x"${bogosities}" != x"" ]; then
      error=yes
      echo "The following .elc files are older than their .el files:"
      echo "${bogosities}"
  fi

  rm -f /tmp/el /tmp/elc /tmp/elcel /tmp/elelc

  bogosities=
  for file in $losers; do
    grep -q "no-byte-compile: t" $file && continue
    case $file in
      site-init.el | site-load.el | site-start.el | default.el) continue ;;
    esac

    bogosities="$file $bogosities"

  done
  if [ x"${bogosities}" != x"" ]; then
    error=yes
    echo "The following .el files have no corresponding .elc files:"
    echo "${bogosities}"
  fi


  ## This is only a crude check, eg it does not handle .info
  ## files with multiple .texi source files.
  find doc -name '*.texi' > /tmp/el

  bogosities=
  while read texi; do
      info=`sed -n 's/^@setfilename //p' $texi | sed 's|.*info/||'`
      [ x"${info}" != x"" ] || continue
      info=info/$info
      [ -r $info ] || continue
      [ $info -nt $texi ] || bogosities="$bogosities $info"
  done < /tmp/el

  rm -f /tmp/el

  if [ x"${bogosities}" != x"" ]; then
    error=yes
    echo "The following .info files are older than their .texi files:"
    echo "${bogosities}"
  fi

  ## This exits with non-zero status if any .info files need
  ## rebuilding.
  if [ -r Makefile ]; then
      echo "Checking to see if info files are up-to-date..."
      make --question info || error=yes
  fi

  ## Is this a release?
  case $version in
      [1-9][0-9].[0-9])
          if [ -r ChangeLog ]; then
              if ! grep -q "Version $version released" ChangeLog; then
                  echo "No release notice in ChangeLog"
                  error=yes
              fi
          else
              echo "A release must have a ChangeLog"
              error=yes
          fi
      ;;
  esac

  if [ $error = yes ]; then
      echo "Failed checks" >&2
      exit 1
  fi

fi

if [ $update = yes ]; then

  ## Make sure configure is newer than configure.ac, etc.
  ## It is better to let autoreconf do what is needed than
  ## for us to try and duplicate all its checks.
  echo "Running autoreconf"
  autoreconf -i -I m4 || { x=$?; echo Autoreconf FAILED! >&2; exit $x; }

  ## Make sure src/stamp-h.in is newer than configure.ac.
  rm -f src/stamp-h.in
  echo timestamp > src/stamp-h.in

  echo "Updating Info files"
  make info

  echo "Updating finder, custom and autoload data"
  (cd lisp && make updates EMACS="$EMACS")

  echo "Updating leim-list.el"
  (cd leim && make leim-list.el EMACS="$EMACS")

  echo "Recompiling Lisp files"
  $EMACS -batch -f batch-byte-recompile-directory lisp
fi                              # $update = yes

echo "Creating staging directory: '${tempparent}'"

mkdir ${tempparent} || exit
tempdir="${tempparent}/${emacsname}"

### This trap ensures that the staging directory will be cleaned up even
### when the script is interrupted in mid-career.
if [ "${clean_up}" = yes ]; then
  trap "echo 'Cleaning up the staging directory'; rm -rf ${tempparent}" EXIT
fi

echo "Creating top directory: '${tempdir}'"
mkdir ${tempdir} || exit

top_level_ChangeLog=
if [ "$changelog" = yes ]; then
  if test -r .git; then
    ## When making a release or pretest the ChangeLog should already
    ## have been created and edited as needed.  Don't ignore it.
    if test -r ChangeLog; then
      echo "Using existing top-level ChangeLog"
      top_level_ChangeLog=ChangeLog
    else
      echo "Making top-level ChangeLog"
      make ChangeLog CHANGELOG=${tempdir}/ChangeLog || \
        { x=$?; echo "make ChangeLog FAILED (try --no-changelog?)" >&2; exit $x; }
    fi
  else
    echo "No repository, so omitting top-level ChangeLog"
  fi
fi

### We copy in the top-level files before creating the subdirectories in
### hopes that this will make the top-level files appear first in the
### tar file; this means that people can start reading the INSTALL and
### README while the rest of the tar file is still unpacking.  Whoopee.
echo "Making links to top-level files"
top_level='
  INSTALL README BUGS
  ChangeLog.*[0-9] Makefile.in autogen.sh configure configure.ac
  config.bat make-dist .dir-locals.el
  aclocal.m4 CONTRIBUTE
'
ln $top_level $top_level_ChangeLog $tempdir || exit

echo "Creating subdirectories"
for subdir in site-lisp \
	      leim leim/CXTERM-DIC leim/MISC-DIC leim/SKK-DIC \
	      build-aux \
	      src src/bitmaps lib lib-src oldXMenu lwlib \
	      nt nt/inc nt/inc/sys nt/inc/arpa nt/inc/netinet nt/icons \
	      `find etc lisp admin test -type d` \
	      doc doc/emacs doc/misc doc/man doc/lispref doc/lispintro \
	      info m4 modules msdos \
	      nextstep nextstep/templates \
	      nextstep/Cocoa nextstep/Cocoa/Emacs.base \
	      nextstep/Cocoa/Emacs.base/Contents \
	      nextstep/Cocoa/Emacs.base/Contents/Resources \
	      nextstep/GNUstep \
	      nextstep/GNUstep/Emacs.base \
	      nextstep/GNUstep/Emacs.base/Resources
do

  if [ "$with_tests" != "yes" ]; then
    case $subdir in
        test*) continue ;;
    esac
  fi

  ## site-lisp for in-place installs (?).
  [ "$subdir" = "site-lisp" ] || [ -d "$subdir" ] || \
      echo "WARNING: $subdir not found, making anyway"
  [ "$verbose" = "yes" ] && echo "  ${tempdir}/${subdir}"
  mkdir ${tempdir}/${subdir} || exit
done

echo "Making links to 'lisp' and its subdirectories"
files=`find lisp \( -name '*.el' -o -name '*.elc' -o -name 'ChangeLog*' \
 -o -name 'README' \)` || exit

### Don't distribute site-init.el, site-load.el, or default.el.
for file in lisp/Makefile.in $files; do
  case $file in
    */site-init*|*/site-load*|*/default*) continue ;;
  esac
  ln $file $tempdir/$file || exit
done

echo "Making links to 'leim' and its subdirectories"
(cd leim &&
 ln ChangeLog.*[0-9] README ../${tempdir}/leim &&
 ln CXTERM-DIC/README CXTERM-DIC/*.tit ../${tempdir}/leim/CXTERM-DIC &&
 ln SKK-DIC/README SKK-DIC/SKK-JISYO.L ../${tempdir}/leim/SKK-DIC &&
 ln MISC-DIC/README MISC-DIC/*.* ../${tempdir}/leim/MISC-DIC &&
 ln Makefile.in ../${tempdir}/leim/Makefile.in &&
 ln leim-ext.el ../${tempdir}/leim/leim-ext.el &&
:) || exit

## FIXME Can we not just use the "find -type f" method for this one?
echo "Making links to 'build-aux'"
(cd build-aux &&
 ln config.guess config.sub msys-to-w32 ../${tempdir}/build-aux &&
 ln gitlog-to-changelog gitlog-to-emacslog ../${tempdir}/build-aux &&
 ln install-sh move-if-change ../${tempdir}/build-aux &&
 ln update-copyright update-subdirs ../${tempdir}/build-aux &&
 ln dir_top make-info-dir ../${tempdir}/build-aux &&
:) || exit

echo "Making links to 'src'"
### Don't distribute the configured versions of
### config.in, paths.in, buildobj.h, or Makefile.in.
(cd src &&
 ln [a-zA-Z]*.[chm] ../${tempdir}/src &&
 ln [a-zA-Z]*.in ../${tempdir}/src &&
 ln deps.mk ../${tempdir}/src &&
 ln README ChangeLog.*[0-9] ../${tempdir}/src &&
 ln .gdbinit .dbxinit ../${tempdir}/src &&
 cd ../${tempdir}/src &&
 rm -f globals.h config.h epaths.h Makefile buildobj.h &&
:) || exit

echo "Making links to 'src/bitmaps'"
(cd src/bitmaps &&
 ln README *.xbm ../../${tempdir}/src/bitmaps &&
:) || exit

echo "Making links to 'lib'"
(cd lib &&
 ln [a-zA-Z_]*.[ch] ../${tempdir}/lib &&
 ln gnulib.mk.in Makefile.in ../${tempdir}/lib &&
 cd ../${tempdir}/lib &&
 script='/[*]/d; s/\.in\.h$/.h/' &&
 rm -f `ls *.in.h | sed "$script"` &&
:) || exit

echo "Making links to 'lib-src'"
(cd lib-src &&
 ln [a-zA-Z]*.[ch] ../${tempdir}/lib-src &&
 ln ChangeLog.*[0-9] Makefile.in README ../${tempdir}/lib-src &&
 ln rcs2log ../${tempdir}/lib-src &&
:) || exit

echo "Making links to 'm4'"
(cd m4 &&
 ln *.m4 ../${tempdir}/m4 &&
:) || exit

echo "Making links to 'modules'"
(cd modules &&
 ln *.py ../${tempdir}/modules &&
:) || exit

echo "Making links to 'nt'"
(cd nt &&
 ln emacs-x86.manifest emacs-x64.manifest ../${tempdir}/nt &&
 ln [a-z]*.bat [a-z]*.[ch] ../${tempdir}/nt &&
 ln *.in gnulib-cfg.mk ../${tempdir}/nt &&
 ln mingw-cfg.site epaths.nt INSTALL.W64 ../${tempdir}/nt &&
 ln ChangeLog.*[0-9] INSTALL README README.W32 ../${tempdir}/nt &&
:) || exit

echo "Making links to 'nt/inc' and its subdirectories"
for f in `find nt/inc -type f -name '[a-z]*.h'`; do
  ln $f $tempdir/$f || exit
done

echo "Making links to 'nt/icons'"
(cd nt/icons &&
 ln README [a-z]*.ico ../../${tempdir}/nt/icons &&
 ln [a-z]*.cur ../../${tempdir}/nt/icons &&
:) || exit

echo "Making links to 'msdos'"
(cd msdos &&
 ln ChangeLog.*[0-9] INSTALL README emacs.ico emacs.pif ../${tempdir}/msdos &&
 ln depfiles.bat inttypes.h ../${tempdir}/msdos &&
 ln mainmake.v2 sed*.inp ../${tempdir}/msdos &&
:) || exit

echo "Making links to 'nextstep'"
(cd nextstep &&
 ln ChangeLog.*[0-9] README INSTALL Makefile.in ../${tempdir}/nextstep &&
:) || exit

echo "Making links to 'nextstep/templates'"
(cd nextstep/templates &&
 ln Emacs.desktop.in Info-gnustep.plist.in Info.plist.in InfoPlist.strings.in \
   ../../${tempdir}/nextstep/templates &&
:) || exit

echo "Making links to 'nextstep/Cocoa/Emacs.base/Contents'"
(cd nextstep/Cocoa/Emacs.base/Contents &&
 ln PkgInfo ../../../../${tempdir}/nextstep/Cocoa/Emacs.base/Contents &&
:) || exit

echo "Making links to 'nextstep/Cocoa/Emacs.base/Contents/Resources'"
(cd nextstep/Cocoa/Emacs.base/Contents/Resources &&
 ln Credits.html *.icns \
   ../../../../../${tempdir}/nextstep/Cocoa/Emacs.base/Contents/Resources &&
:) || exit

echo "Making links to 'nextstep/GNUstep/Emacs.base/Resources'"
(cd nextstep/GNUstep/Emacs.base/Resources &&
 ln README emacs.tiff \
   ../../../../${tempdir}/nextstep/GNUstep/Emacs.base/Resources &&
:) || exit

echo "Making links to 'oldXMenu'"
(cd oldXMenu &&
 ln *.[ch] *.in *.mk ../${tempdir}/oldXMenu &&
 ln README ChangeLog.*[0-9] ../${tempdir}/oldXMenu &&
:) || exit

echo "Making links to 'lwlib'"
(cd lwlib &&
 ln *.[ch] *.in *.mk ../${tempdir}/lwlib &&
 ln README ChangeLog.*[0-9] ../${tempdir}/lwlib &&
:) || exit

## It is important to distribute admin/ because it contains sources
## for generated lisp/international/uni-*.el files.
echo "Making links to 'admin' and its subdirectories"
for f in `find admin -type f`; do
  case $f in
    */Makefile) [ -f $f.in ] && continue ;;
  esac
  ln $f $tempdir/$f || exit
done

if [ "$with_tests" = "yes" ]; then
  echo "Making links to 'test' and its subdirectories"
  for f in `find test -type f ! -name '*.log' ! -name a.out \
	      ! -name '*.so' ! -name '*.dll' ! -name '*.o'
  `; do
    case $f in
        */Makefile) [ -f $f.in ] && continue ;;
    esac
    ln $f $tempdir/$f || exit
  done
fi

echo "Making links to 'etc' and its subdirectories"
for f in `find etc -type f`; do
  case $f in
    etc/DOC*|etc/*.pyc) continue ;;
    ## Arguably we should not exclude *.ps.
    etc/refcards/*.aux|etc/refcards/*.dvi|etc/refcards/*.log|etc/refcards/*.ps)
      continue ;;
  esac
  ln $f $tempdir/$f || exit
done

echo "Making links to 'info'"
ln `find info -type f -print` ${tempdir}/info || exit

echo "Making links to 'doc/emacs'"
(cd doc/emacs &&
 ln *.texi *.in ChangeLog.*[0-9] ../../${tempdir}/doc/emacs &&
:) || exit

echo "Making links to 'doc/misc'"
(cd doc/misc &&
 ln *.texi *.tex *.in gnus-news.el ChangeLog.*[0-9] \
   ../../${tempdir}/doc/misc &&
:) || exit

echo "Making links to 'doc/lispref'"
(cd doc/lispref &&
 ln *.texi *.in README ChangeLog.*[0-9] ../../${tempdir}/doc/lispref &&
 ln spellfile ../../${tempdir}/doc/lispref &&
 ln two-volume.make two-volume-cross-refs.txt ../../${tempdir}/doc/lispref &&
:) || exit

echo "Making links to 'doc/lispintro'"
(cd doc/lispintro &&
 ln *.texi *.in *.eps *.pdf ../../${tempdir}/doc/lispintro &&
 ln README ChangeLog.*[0-9] ../../${tempdir}/doc/lispintro &&
 cd ../../${tempdir}/doc/lispintro &&
:) || exit

echo "Making links to 'doc/man'"
(cd doc/man &&
 ln *.*[0-9] *.in ../../${tempdir}/doc/man &&
 cd ../../${tempdir}/doc/man &&
 rm -f emacs.1 &&
:) || exit

### It would be nice if they could all be symlinks to top-level copy, but
### you're not supposed to have any symlinks in distribution tar files.
echo "Making sure copying notices are all copies of 'COPYING'"
for subdir in . etc leim lib lib-src lisp lwlib msdos nt src; do
  rm -f ${tempdir}/${subdir}/COPYING || exit
  cp COPYING ${tempdir}/${subdir} || exit
done

if [ "${newer}" ]; then
  printf '%s\n' "Removing files older than $newer"
  ## We remove .elc files unconditionally, on the theory that anyone picking
  ## up an incremental distribution already has a running Emacs to byte-compile
  ## them with.
  find ${tempparent} \( -name '*.elc' -o ! -newer ${newer} \) \
    -exec rm -f {} \; || exit
fi

## Don't distribute backups, autosaves, etc.
echo "Removing unwanted files"
find ${tempparent} \( -name '*~' -o -name '#*#' -o -name '.*ignore' -o -name '=*' -o -name 'TAGS' \) -exec rm -f {} \; || exit

if [ "${make_tar}" = yes ]; then
  echo "Looking for $default_gzip"
  found=0
  temppath=`printf '%s\n' "$PATH" |
    sed -e 's/^:/.:/' -e 's/::/:.:/g' -e 's/:$/:./' -e 's/:/ /g'
  `
  for dir in ${temppath}; do
    [ -x ${dir}/$default_gzip ] || continue
    found=1; break
  done
  if [ "$found" = "0" ]; then
    echo "WARNING: '$default_gzip' not found, will not compress" >&2
    default_gzip=cat
  fi
  case "${default_gzip}" in
    bzip2) gzip_extension=.bz2 ;;
    xz)  gzip_extension=.xz ;;
    gzip)  gzip_extension=.gz ; default_gzip="gzip --best --no-name";;
       *)  gzip_extension= ;;
  esac
  echo "Creating tar file"
  taropt='--numeric-owner --owner=0 --group=0 --mode=go+u,go-w'
  tar --sort=name -cf /dev/null $tempparent/$emacsname/src/lisp.h 2>/tmp/out &&
    taropt="$taropt --sort=name"
  [ "$verbose" = "yes" ] && taropt="$taropt --verbose"

  (cd $tempparent &&
   case $default_gzip in
     cat) tar $taropt -cf - $emacsname;;
     *) if tar $taropt -cf /dev/null --use-compress-program="$default_gzip" \
	       $emacsname/src/lisp.h
	then
	  tar $taropt -cf - --use-compress-program="$default_gzip" $emacsname
	else
	  tar $taropt -cf $emacsname.tar $emacsname &&
	  $default_gzip <$emacsname.tar
        fi;;
   esac
  ) >$emacsname.tar$gzip_extension || exit
fi

if [ "${clean_up}" != yes ]; then
  (cd ${tempparent} && mv ${emacsname} ..) &&
  rm -rf ${tempparent}
fi

### make-dist ends here