[RFC] src-release.sh: Convert src-release to a shell script

[RFC] src-release.sh: Convert src-release to a shell script

[Will Newton]

This began as an attempt to make a few small changes to src-release
but heeding the comments at the top of the file it seemed it might be
wise to convert it to a shell script instead which should hopefully
be more maintainable.

This shell script contains most of the functionality of the Makefile
version. It can be run like this:

  # This will build gdb and compress with bzip2, gzip and xz
  ./src-release.sh -bgx gdb

The functionality that has been removed is building tarballs for insight
and gnats, which were broken in the current repository layout and
gas+binutils which amounted to basically the same thing as binutils so
didn't seem worth keeping. The script always builds a tar file,
compression is optional, so no need for specific commands to build tar
files rather than compressed tar files.

The build of gas releases has been fixed, as has extraction of version
numbers which had been broken since the switch to configure.ac files
and AC_INIT.

I also removed the code for distcleaning the intl sub-directory as it
seems like that issue has been fixed.

The script is capable of compressing with bzip2, gzip and xz which
should cover all formats available on gnu.org.

I tested it by producing release tarballs which are substantially
identical to the ones produced by the src-release script.

ChangeLog:

2014-08-07  Will Newton  <will.newton@linaro.org>

	* src-release.sh: New file.
---
 src-release.sh | 353 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 353 insertions(+)
 create mode 100755 src-release.sh

diff --git a/src-release.sh b/src-release.sh
new file mode 100755
index 0000000..3d41269
--- /dev/null
+++ b/src-release.sh
@@ -0,0 +1,353 @@
+#!/bin/bash
+#   Copyright (C) 1990-2014 Free Software Foundation
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+
+# This script creates release packages for gdb, binutils, and other
+# packages which live in src.  It used to be implemented as the src-release
+# Makefile and prior to that was part of the top level Makefile, but that
+# turned out to be very messy and hard to maintain.
+
+set -e
+
+BZIPPROG=bzip2
+GZIPPROG=gzip
+XZPROG=xz
+MD5PROG=md5sum
+MAKE=make
+CC=gcc
+CXX=g++
+
+# Default to avoid splitting info files by setting the threshold high.
+MAKEINFOFLAGS=--split-size=5000000
+
+#
+# Support for building net releases
+
+# Files in devo used in any net release.
+DEVO_SUPPORT="README Makefile.in configure configure.ac \
+	config.guess config.sub config move-if-change \
+	COPYING COPYING.LIB install-sh config-ml.in symlink-tree \
+	mkinstalldirs ltmain.sh missing ylwrap \
+	libtool.m4 ltsugar.m4 ltversion.m4 ltoptions.m4 \
+	Makefile.def Makefile.tpl src-release config.rpath \
+	ChangeLog MAINTAINERS README-maintainer-mode \
+	lt~obsolete.m4 ltgcc.m4 depcomp mkdep compile \
+	COPYING3 COPYING3.LIB"
+
+# Files in devo/etc used in any net release.
+ETC_SUPPORT="Makefile.in configure configure.in standards.texi \
+	make-stds.texi standards.info* configure.texi configure.info* \
+	ChangeLog configbuild.* configdev.* fdl.texi texi2pod.pl gnu-oids.texi"
+
+# Get the version number of a given tool
+getver()
+{
+    tool=$1
+    if grep 'AC_INIT.*BFD_VERSION' $tool/configure.ac >/dev/null 2>&1; then
+	bfd/configure --version | sed -n -e '1s,.* ,,p'
+    elif test -f $tool/common/create-version.sh; then
+	$tool/common/create-version.sh $tool 'dummy-host' 'dummy-target' VER.tmp
+	cat VER.tmp | grep 'version\[\]' | sed 's/.*"\([^"]*\)".*/\1/' | sed 's/-cvs$//'
+        rm -f VER.tmp
+    elif test -f $tool/version.in; then
+	head -1 $tool/version.in
+    else
+	echo VERSION
+    fi
+}
+
+# Setup build directory for building release tarball
+do_proto_toplev()
+{
+    package=$1
+    ver=$2
+    tool=$3
+    support_files=$4
+    echo "==> Making $package-$ver/"
+    # Take out texinfo from a few places.
+    sed -e '/^all\.normal: /s/\all-texinfo //' \
+	-e '/^	install-texinfo /d' \
+	<Makefile.in >tmp
+    mv -f tmp Makefile.in
+    #
+    ./configure i686-pc-linux-gnu
+    $MAKE configure-host configure-target \
+	ALL_GCC="" ALL_GCC_C="" ALL_GCC_CXX="" \
+	CC_FOR_TARGET="$CC" CXX_FOR_TARGET="$CXX"
+    # Make links, and run "make diststuff" or "make info" when needed.
+    rm -rf proto-toplev
+    mkdir proto-toplev
+    dirs="$DEVO_SUPPORT $support_files $tool"
+    for d in $dirs ; do
+	if [ -d $d ]; then
+	    if [ ! -f $d/Makefile ] ; then
+		true
+	    elif grep '^diststuff:' $d/Makefile >/dev/null ; then
+		(cd $d ; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" diststuff) \
+		    || exit 1
+	    elif grep '^info:' $d/Makefile >/dev/null ; then
+		(cd $d ; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" info) \
+		    || exit 1
+	    fi
+	    if [ -d $d/proto-$d.dir ]; then
+		ln -s ../$d/proto-$d.dir proto-toplev/$d
+	    else
+		ln -s ../$d proto-toplev/$d
+	    fi
+	else
+	    if (echo x$d | grep / >/dev/null); then
+	      mkdir -p proto-toplev/`dirname $d`
+	      x=`dirname $d`
+	      ln -s ../`echo $x/ | sed -e 's,[^/]*/,../,g'`$d proto-toplev/$d
+	    else
+	      ln -s ../$d proto-toplev/$d
+	    fi
+	  fi
+	done
+	(cd etc; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" info)
+	$MAKE distclean
+	mkdir proto-toplev/etc
+	(cd proto-toplev/etc;
+	    for i in $ETC_SUPPORT; do
+		ln -s ../../etc/$i .
+		done)
+	#
+	# Take out texinfo from configurable dirs
+	rm proto-toplev/configure.ac
+	sed -e '/^host_tools=/s/texinfo //' \
+	    <configure.ac >proto-toplev/configure.ac
+	#
+	mkdir proto-toplev/texinfo
+	ln -s ../../texinfo/texinfo.tex	proto-toplev/texinfo/
+	if test -r texinfo/util/tex3patch ; then
+	    mkdir proto-toplev/texinfo/util && \
+		ln -s ../../../texinfo/util/tex3patch proto-toplev/texinfo/util
+	else
+	    true
+	fi
+	chmod -R og=u . || chmod og=u `find . -print`
+	#
+	# Create .gmo files from .po files.
+	for f in `find . -name '*.po' -type f -print`; do
+	    msgfmt -o `echo $f | sed -e 's/\.po$/.gmo/'` $f
+	done
+	#
+	rm -f $package-$ver
+	ln -s proto-toplev $package-$ver
+}
+
+CVS_NAMES='-name CVS -o -name .cvsignore'
+
+# Add an md5sum to the built tarball
+do_md5sum()
+{
+    echo "==> Adding md5 checksum to top-level directory"
+    (cd proto-toplev && find * -follow \( $CVS_NAMES \) -prune \
+	-o -type f -print \
+	| xargs $MD5PROG > ../md5.new)
+    rm -f proto-toplev/md5.sum
+    mv md5.new proto-toplev/md5.sum
+}
+
+# Build the release tarball
+do_tar()
+{
+    package=$1
+    ver=$2
+    echo "==> Making $package-$ver.tar"
+    rm -f $package-$ver.tar
+    find $package-$ver -follow \( $CVS_NAMES \) -prune \
+	-o -type f -print \
+	| tar cTfh - $package-$ver.tar
+}
+
+# Compress the output with bzip2
+do_bz2()
+{
+    package=$1
+    ver=$2
+    echo "==> Bzipping $package-$ver.tar.bz2"
+    rm -f $package-$ver.tar.bz2
+    $BZIPPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with gzip
+do_gz()
+{
+    package=$1
+    ver=$2
+    echo "==> Gzipping $package-$ver.tar.gz"
+    rm -f $package-$ver.tar.gz
+    $GZIPPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with xz
+do_xz()
+{
+    package=$1
+    ver=$2
+    echo "==> Xzipping $package-$ver.tar.xz"
+    rm -f $package-$ver.tar.xz
+    $XZPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with all selected compresion methods
+do_compress()
+{
+    package=$1
+    ver=$2
+    compressors=$3
+    for comp in $compressors; do
+	case $comp in
+	    bz2)
+		do_bz2 $package $ver;;
+	    gz)
+		do_gz $package $ver;;
+	    xz)
+		do_xz $package $ver;;
+	    *)
+		echo "Unknown compression method: $comp" && exit 1;;
+	esac
+    done
+}
+
+# Add djunpack.bat the tarball
+do_djunpack()
+{
+    package=$1
+    ver=$2
+    echo "==> Adding updated djunpack.bat to top-level directory"
+    echo - 's /gdb-[0-9\.]*/$package-'"$ver"'/'
+    sed < djunpack.bat > djunpack.new \
+	-e 's/gdb-[0-9][0-9\.]*/$package-'"$ver"'/'
+    rm -f proto-toplev/djunpack.bat
+    mv djunpack.new proto-toplev/djunpack.bat
+}
+
+# Create a release package, tar it and compress it
+tar_compress()
+{
+    package=$1
+    tool=$2
+    support_files=$3
+    compressors=$4
+    ver=$(getver $tool)
+    do_proto_toplev $package $ver $tool "$support_files"
+    do_md5sum
+    do_tar $package $ver
+    do_compress $package $ver "$compressors"
+}
+
+# Create a gdb release package, tar it and compress it
+gdb_tar_compress()
+{
+    package=$1
+    tool=$2
+    support_files=$3
+    compressors=$4
+    ver=$(getver $tool)
+    do_proto_toplev $package $ver $tool "$support_files"
+    do_md5sum
+    do_djunpack $package $ver
+    do_tar $package $ver
+    do_compress $package $ver "$compressors"
+}
+
+# The FSF "binutils" release includes gprof and ld.
+BINUTILS_SUPPORT_DIRS="bfd gas include libiberty opcodes ld elfcpp gold gprof intl setup.com makefile.vms cpu"
+binutils_release()
+{
+    compressors=$1
+    package=binutils
+    tool=binutils
+    tar_compress $package $tool "$BINUTILS_SUPPORT_DIRS" "$compressors"
+}
+
+GAS_SUPPORT_DIRS="bfd include libiberty opcodes intl setup.com makefile.vms"
+gas_release()
+{
+    compressors=$1
+    package=gas
+    tool=gas
+    tar_compress $package $tool "$GAS_SUPPORT_DIRS" "$compressors"
+}
+
+GDB_SUPPORT_DIRS="bfd include libiberty opcodes readline sim intl libdecnumber cpu"
+gdb_release()
+{
+    compressors=$1
+    package=gdb
+    tool=gdb
+    gdb_tar_compress $package $tool "$GDB_SUPPORT_DIRS" "$compressors"
+}
+
+# Corresponding to the CVS "sim" module.
+SIM_SUPPORT_DIRS="bfd opcodes libiberty include intl gdb/version.in makefile.vms"
+sim_release()
+{
+    compressors=$1
+    package=sim
+    tool=sim
+    tar_compress $package $tool "$SIM_SUPPORT_DIRS" "$compressors"
+}
+
+usage()
+{
+    echo "src-release.sh <options> <release>"
+    echo "options:"
+    echo "  -b: Compress with bzip2"
+    echo "  -g: Compress with gzip"
+    echo "  -x: Compress with xz"
+    exit 1
+}
+
+build_release()
+{
+    release=$1
+    compressors=$2
+    case $release in
+	binutils)
+	    binutils_release "$compressors";;
+	gas)
+	    gas_release "$compressors";;
+	gdb)
+	    gdb_release "$compressors";;
+	sim)
+	    sim_release "$compressors";;
+	*)
+	    echo "Unknown release name: $release" && usage;;
+    esac
+}
+
+compressors=""
+
+while getopts ":gbx" opt; do
+    case $opt in
+	b)
+	    compressors="$compressors bz2";;
+	g)
+	    compressors="$compressors gz";;
+	x)
+	    compressors="$compressors xz";;
+	\?)
+	    echo "Invalid option: -$OPTARG" && usage;;
+  esac
+done
+shift $((OPTIND -1))
+release=$1
+
+build_release $release "$compressors"
-- 
1.9.3

Re: [RFC] src-release.sh: Convert src-release to a shell script

[Will Newton]

On 27 August 2014 18:13, Hans-Peter Nilsson <hans-peter.nilsson@axis.com> wrote:
>> From: Will Newton <will.newton@linaro.org>
>> Date: Wed, 27 Aug 2014 14:52:40 +0200
>
>> I made that change and pushed. Let me know if anyone finds any
>> functionality missing.
>
> I wished you had not also pushed the deletion of the old
> src-release Makefile at the same time; your posted ChangeLog
> and the discussion didn't indicate that...  (heads-up to others)

It wasn't part of the initial submission but I added the deletion of
src-release to the changelog in response to review. I guess I should
have been more explicit about that, apologies.

> Anyway, I'ved moved over my (sim, binutils) autotesters to
> src-release.sh and so far so good.

But on the plus side if that means you test it that's great. ;-)

Thanks for letting me know.

-- 
Will Newton
Toolchain Working Group, Linaro

Re: [RFC] src-release.sh: Convert src-release to a shell script

[Hans-Peter Nilsson]

> From: Will Newton <will.newton@linaro.org>
> Date: Wed, 27 Aug 2014 14:52:40 +0200

> I made that change and pushed. Let me know if anyone finds any
> functionality missing.

I wished you had not also pushed the deletion of the old
src-release Makefile at the same time; your posted ChangeLog
and the discussion didn't indicate that...  (heads-up to others)

Anyway, I'ved moved over my (sim, binutils) autotesters to
src-release.sh and so far so good.

(Incidentally, they had no problems with the tarballs being
previously misnamed, e.g. "binutils-@VERSION@.tar.bz2". :-)

brgds, H-P

Re: [RFC] src-release.sh: Convert src-release to a shell script

[Will Newton]

On 27 August 2014 07:06, Joel Brobecker <brobecker@adacore.com> wrote:
> Hi Will,
>
>> +#   Copyright (C) 1990-2014 Free Software Foundation
>> +#
>> +# This file is free software; you can redistribute it and/or modify
>> +# it under the terms of the GNU General Public License as published by
>> +# the Free Software Foundation; either version 3 of the License, or
>> +# (at your option) any later version.
>> +#
>> +# This program is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
>> +#
>
> Can you also replace the last paragraph with...
>
>     # You should have received a copy of the GNU General Public License
>     # along with this program.  If not, see <http://www.gnu.org/licenses/>.
>
> ?

I made that change and pushed. Let me know if anyone finds any
functionality missing.

-- 
Will Newton
Toolchain Working Group, Linaro

Re: [RFC] src-release.sh: Convert src-release to a shell script

[Joel Brobecker]

Hi Will,

> +#   Copyright (C) 1990-2014 Free Software Foundation
> +#
> +# This file is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 3 of the License, or
> +# (at your option) any later version.
> +# 
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +# 
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
> +#

Can you also replace the last paragraph with...

    # You should have received a copy of the GNU General Public License
    # along with this program.  If not, see <http://www.gnu.org/licenses/>.

?

Thank you!
-- 
Joel

Re: [RFC] src-release.sh: Convert src-release to a shell script

[Will Newton]

[-- Attachment #1: Type: text/plain, Size: 2206 bytes --]

On 15 August 2014 13:49, Joel Brobecker <brobecker@adacore.com> wrote:
>> 2014-08-07  Will Newton  <will.newton@linaro.org>
>>
>>       * src-release.sh: New file.
>
> Generally speaking, I tend to agree that a makefile is slightly
> overkill for this task. The contents of the makefile is fairly
> straightforward though, and outside of variable VEC in the makefile,
> you script doesn't look any simpler than the makefile... What
> I appreciate, however, is the lack of make recursions (with
> the associated multiple "make" processes).
>
> Give it another week for people to comment on, and then let's get
> this in (I'll take care of talking to Tristan when he comes back).
> Also, let's delete the old src-release script.
>
> Some relatively trivial comments below:
>
>> diff --git a/src-release.sh b/src-release.sh
>> new file mode 100755
>> index 0000000..3d41269
>> --- /dev/null
>> +++ b/src-release.sh
>> @@ -0,0 +1,353 @@
>> +#!/bin/bash
>
> Can you use /usr/bin/env? People trying to create tarballs for their
> own might not be doing it on a GNU/Linux machine, and so bash might
> be installed elsewhere.
>
>> +#   Copyright (C) 1990-2014 Free Software Foundation
>> +#
>> +# This file is free software; you can redistribute it and/or modify
>> +# it under the terms of the GNU General Public License as published by
>> +# the Free Software Foundation; either version 2 of the License, or
>> +# (at your option) any later version.
>> +#
>> +# This program is distributed in the hope that it will be useful,
>> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
>> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> +# GNU General Public License for more details.
>> +#
>> +# You should have received a copy of the GNU General Public License
>> +# along with this program; if not, write to the Free Software
>> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
>
> The copyright year range looks correct to me, but the license
> should be GPLv3.

Thanks for the review. I made the fixes you suggested and will apply
the attached patch in the next 24 hours unless somebody objects.

-- 
Will Newton
Toolchain Working Group, Linaro

[-- Attachment #2: 0001-src-release.sh-Convert-src-release-to-a-shell-script.patch --]
[-- Type: text/x-patch, Size: 11448 bytes --]

From 61bd489a8a5ab27d861111726b8a6ceb4ca2eada Mon Sep 17 00:00:00 2001
From: Will Newton <will.newton@linaro.org>
Date: Thu, 7 Aug 2014 14:38:55 +0100
Subject: [PATCH] src-release.sh: Convert src-release to a shell script

This began as an attempt to make a few small changes to src-release
but heeding the comments at the top of the file it seemed it might be
wise to convert it to a shell script instead which should hopefully
be more maintainable.

This shell script contains most of the functionality of the Makefile
version. It can be run like this:

  # This will build gdb and compress with bzip2, gzip and xz
  ./src-release.sh -bgx gdb

The functionality that has been removed is building tarballs for insight
and gnats, which were broken in the current repository layout and
gas+binutils which amounted to basically the same thing as binutils so
didn't seem worth keeping. The script always builds a tar file,
compression is optional, so no need for specific commands to build tar
files rather than compressed tar files.

The build of gas releases has been fixed, as has extraction of version
numbers which had been broken since the switch to configure.ac files
and AC_INIT.

I also removed the code for distcleaning the intl sub-directory as it
seems like that issue has been fixed.

The script is capable of compressing with bzip2, gzip and xz which
should cover all formats available on gnu.org.

I tested it by producing release tarballs which are substantially
identical to the ones produced by the src-release script.

ChangeLog:

2014-08-26  Will Newton  <will.newton@linaro.org>

	* src-release.sh: New file.
	* src-release: Remove file.
---
 src-release.sh | 353 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 353 insertions(+)
 create mode 100755 src-release.sh

diff --git a/src-release.sh b/src-release.sh
new file mode 100755
index 0000000..6c5c390
--- /dev/null
+++ b/src-release.sh
@@ -0,0 +1,353 @@
+#!/usr/bin/env bash
+#   Copyright (C) 1990-2014 Free Software Foundation
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+
+# This script creates release packages for gdb, binutils, and other
+# packages which live in src.  It used to be implemented as the src-release
+# Makefile and prior to that was part of the top level Makefile, but that
+# turned out to be very messy and hard to maintain.
+
+set -e
+
+BZIPPROG=bzip2
+GZIPPROG=gzip
+XZPROG=xz
+MD5PROG=md5sum
+MAKE=make
+CC=gcc
+CXX=g++
+
+# Default to avoid splitting info files by setting the threshold high.
+MAKEINFOFLAGS=--split-size=5000000
+
+#
+# Support for building net releases
+
+# Files in devo used in any net release.
+DEVO_SUPPORT="README Makefile.in configure configure.ac \
+	config.guess config.sub config move-if-change \
+	COPYING COPYING.LIB install-sh config-ml.in symlink-tree \
+	mkinstalldirs ltmain.sh missing ylwrap \
+	libtool.m4 ltsugar.m4 ltversion.m4 ltoptions.m4 \
+	Makefile.def Makefile.tpl src-release config.rpath \
+	ChangeLog MAINTAINERS README-maintainer-mode \
+	lt~obsolete.m4 ltgcc.m4 depcomp mkdep compile \
+	COPYING3 COPYING3.LIB"
+
+# Files in devo/etc used in any net release.
+ETC_SUPPORT="Makefile.in configure configure.in standards.texi \
+	make-stds.texi standards.info* configure.texi configure.info* \
+	ChangeLog configbuild.* configdev.* fdl.texi texi2pod.pl gnu-oids.texi"
+
+# Get the version number of a given tool
+getver()
+{
+    tool=$1
+    if grep 'AC_INIT.*BFD_VERSION' $tool/configure.ac >/dev/null 2>&1; then
+	bfd/configure --version | sed -n -e '1s,.* ,,p'
+    elif test -f $tool/common/create-version.sh; then
+	$tool/common/create-version.sh $tool 'dummy-host' 'dummy-target' VER.tmp
+	cat VER.tmp | grep 'version\[\]' | sed 's/.*"\([^"]*\)".*/\1/' | sed 's/-cvs$//'
+        rm -f VER.tmp
+    elif test -f $tool/version.in; then
+	head -1 $tool/version.in
+    else
+	echo VERSION
+    fi
+}
+
+# Setup build directory for building release tarball
+do_proto_toplev()
+{
+    package=$1
+    ver=$2
+    tool=$3
+    support_files=$4
+    echo "==> Making $package-$ver/"
+    # Take out texinfo from a few places.
+    sed -e '/^all\.normal: /s/\all-texinfo //' \
+	-e '/^	install-texinfo /d' \
+	<Makefile.in >tmp
+    mv -f tmp Makefile.in
+    #
+    ./configure i686-pc-linux-gnu
+    $MAKE configure-host configure-target \
+	ALL_GCC="" ALL_GCC_C="" ALL_GCC_CXX="" \
+	CC_FOR_TARGET="$CC" CXX_FOR_TARGET="$CXX"
+    # Make links, and run "make diststuff" or "make info" when needed.
+    rm -rf proto-toplev
+    mkdir proto-toplev
+    dirs="$DEVO_SUPPORT $support_files $tool"
+    for d in $dirs ; do
+	if [ -d $d ]; then
+	    if [ ! -f $d/Makefile ] ; then
+		true
+	    elif grep '^diststuff:' $d/Makefile >/dev/null ; then
+		(cd $d ; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" diststuff) \
+		    || exit 1
+	    elif grep '^info:' $d/Makefile >/dev/null ; then
+		(cd $d ; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" info) \
+		    || exit 1
+	    fi
+	    if [ -d $d/proto-$d.dir ]; then
+		ln -s ../$d/proto-$d.dir proto-toplev/$d
+	    else
+		ln -s ../$d proto-toplev/$d
+	    fi
+	else
+	    if (echo x$d | grep / >/dev/null); then
+	      mkdir -p proto-toplev/`dirname $d`
+	      x=`dirname $d`
+	      ln -s ../`echo $x/ | sed -e 's,[^/]*/,../,g'`$d proto-toplev/$d
+	    else
+	      ln -s ../$d proto-toplev/$d
+	    fi
+	  fi
+	done
+	(cd etc; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" info)
+	$MAKE distclean
+	mkdir proto-toplev/etc
+	(cd proto-toplev/etc;
+	    for i in $ETC_SUPPORT; do
+		ln -s ../../etc/$i .
+		done)
+	#
+	# Take out texinfo from configurable dirs
+	rm proto-toplev/configure.ac
+	sed -e '/^host_tools=/s/texinfo //' \
+	    <configure.ac >proto-toplev/configure.ac
+	#
+	mkdir proto-toplev/texinfo
+	ln -s ../../texinfo/texinfo.tex	proto-toplev/texinfo/
+	if test -r texinfo/util/tex3patch ; then
+	    mkdir proto-toplev/texinfo/util && \
+		ln -s ../../../texinfo/util/tex3patch proto-toplev/texinfo/util
+	else
+	    true
+	fi
+	chmod -R og=u . || chmod og=u `find . -print`
+	#
+	# Create .gmo files from .po files.
+	for f in `find . -name '*.po' -type f -print`; do
+	    msgfmt -o `echo $f | sed -e 's/\.po$/.gmo/'` $f
+	done
+	#
+	rm -f $package-$ver
+	ln -s proto-toplev $package-$ver
+}
+
+CVS_NAMES='-name CVS -o -name .cvsignore'
+
+# Add an md5sum to the built tarball
+do_md5sum()
+{
+    echo "==> Adding md5 checksum to top-level directory"
+    (cd proto-toplev && find * -follow \( $CVS_NAMES \) -prune \
+	-o -type f -print \
+	| xargs $MD5PROG > ../md5.new)
+    rm -f proto-toplev/md5.sum
+    mv md5.new proto-toplev/md5.sum
+}
+
+# Build the release tarball
+do_tar()
+{
+    package=$1
+    ver=$2
+    echo "==> Making $package-$ver.tar"
+    rm -f $package-$ver.tar
+    find $package-$ver -follow \( $CVS_NAMES \) -prune \
+	-o -type f -print \
+	| tar cTfh - $package-$ver.tar
+}
+
+# Compress the output with bzip2
+do_bz2()
+{
+    package=$1
+    ver=$2
+    echo "==> Bzipping $package-$ver.tar.bz2"
+    rm -f $package-$ver.tar.bz2
+    $BZIPPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with gzip
+do_gz()
+{
+    package=$1
+    ver=$2
+    echo "==> Gzipping $package-$ver.tar.gz"
+    rm -f $package-$ver.tar.gz
+    $GZIPPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with xz
+do_xz()
+{
+    package=$1
+    ver=$2
+    echo "==> Xzipping $package-$ver.tar.xz"
+    rm -f $package-$ver.tar.xz
+    $XZPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with all selected compresion methods
+do_compress()
+{
+    package=$1
+    ver=$2
+    compressors=$3
+    for comp in $compressors; do
+	case $comp in
+	    bz2)
+		do_bz2 $package $ver;;
+	    gz)
+		do_gz $package $ver;;
+	    xz)
+		do_xz $package $ver;;
+	    *)
+		echo "Unknown compression method: $comp" && exit 1;;
+	esac
+    done
+}
+
+# Add djunpack.bat the tarball
+do_djunpack()
+{
+    package=$1
+    ver=$2
+    echo "==> Adding updated djunpack.bat to top-level directory"
+    echo - 's /gdb-[0-9\.]*/$package-'"$ver"'/'
+    sed < djunpack.bat > djunpack.new \
+	-e 's/gdb-[0-9][0-9\.]*/$package-'"$ver"'/'
+    rm -f proto-toplev/djunpack.bat
+    mv djunpack.new proto-toplev/djunpack.bat
+}
+
+# Create a release package, tar it and compress it
+tar_compress()
+{
+    package=$1
+    tool=$2
+    support_files=$3
+    compressors=$4
+    ver=$(getver $tool)
+    do_proto_toplev $package $ver $tool "$support_files"
+    do_md5sum
+    do_tar $package $ver
+    do_compress $package $ver "$compressors"
+}
+
+# Create a gdb release package, tar it and compress it
+gdb_tar_compress()
+{
+    package=$1
+    tool=$2
+    support_files=$3
+    compressors=$4
+    ver=$(getver $tool)
+    do_proto_toplev $package $ver $tool "$support_files"
+    do_md5sum
+    do_djunpack $package $ver
+    do_tar $package $ver
+    do_compress $package $ver "$compressors"
+}
+
+# The FSF "binutils" release includes gprof and ld.
+BINUTILS_SUPPORT_DIRS="bfd gas include libiberty opcodes ld elfcpp gold gprof intl setup.com makefile.vms cpu"
+binutils_release()
+{
+    compressors=$1
+    package=binutils
+    tool=binutils
+    tar_compress $package $tool "$BINUTILS_SUPPORT_DIRS" "$compressors"
+}
+
+GAS_SUPPORT_DIRS="bfd include libiberty opcodes intl setup.com makefile.vms"
+gas_release()
+{
+    compressors=$1
+    package=gas
+    tool=gas
+    tar_compress $package $tool "$GAS_SUPPORT_DIRS" "$compressors"
+}
+
+GDB_SUPPORT_DIRS="bfd include libiberty opcodes readline sim intl libdecnumber cpu"
+gdb_release()
+{
+    compressors=$1
+    package=gdb
+    tool=gdb
+    gdb_tar_compress $package $tool "$GDB_SUPPORT_DIRS" "$compressors"
+}
+
+# Corresponding to the CVS "sim" module.
+SIM_SUPPORT_DIRS="bfd opcodes libiberty include intl gdb/version.in makefile.vms"
+sim_release()
+{
+    compressors=$1
+    package=sim
+    tool=sim
+    tar_compress $package $tool "$SIM_SUPPORT_DIRS" "$compressors"
+}
+
+usage()
+{
+    echo "src-release.sh <options> <release>"
+    echo "options:"
+    echo "  -b: Compress with bzip2"
+    echo "  -g: Compress with gzip"
+    echo "  -x: Compress with xz"
+    exit 1
+}
+
+build_release()
+{
+    release=$1
+    compressors=$2
+    case $release in
+	binutils)
+	    binutils_release "$compressors";;
+	gas)
+	    gas_release "$compressors";;
+	gdb)
+	    gdb_release "$compressors";;
+	sim)
+	    sim_release "$compressors";;
+	*)
+	    echo "Unknown release name: $release" && usage;;
+    esac
+}
+
+compressors=""
+
+while getopts ":gbx" opt; do
+    case $opt in
+	b)
+	    compressors="$compressors bz2";;
+	g)
+	    compressors="$compressors gz";;
+	x)
+	    compressors="$compressors xz";;
+	\?)
+	    echo "Invalid option: -$OPTARG" && usage;;
+  esac
+done
+shift $((OPTIND -1))
+release=$1
+
+build_release $release "$compressors"
-- 
1.9.3

Re: [RFC] src-release.sh: Convert src-release to a shell script

[Joel Brobecker]

> 2014-08-07  Will Newton  <will.newton@linaro.org>
> 
> 	* src-release.sh: New file.

Generally speaking, I tend to agree that a makefile is slightly
overkill for this task. The contents of the makefile is fairly
straightforward though, and outside of variable VEC in the makefile,
you script doesn't look any simpler than the makefile... What
I appreciate, however, is the lack of make recursions (with
the associated multiple "make" processes).

Give it another week for people to comment on, and then let's get
this in (I'll take care of talking to Tristan when he comes back).
Also, let's delete the old src-release script.

Some relatively trivial comments below:

> diff --git a/src-release.sh b/src-release.sh
> new file mode 100755
> index 0000000..3d41269
> --- /dev/null
> +++ b/src-release.sh
> @@ -0,0 +1,353 @@
> +#!/bin/bash

Can you use /usr/bin/env? People trying to create tarballs for their
own might not be doing it on a GNU/Linux machine, and so bash might
be installed elsewhere.

> +#   Copyright (C) 1990-2014 Free Software Foundation
> +#
> +# This file is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +# 
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +# 
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.

The copyright year range looks correct to me, but the license
should be GPLv3.

-- 
Joel

[RFC] src-release.sh: Convert src-release to a shell script

[Will Newton]

This began as an attempt to make a few small changes to src-release
but heeding the comments at the top of the file it seemed it might be
wise to convert it to a shell script instead which should hopefully
be more maintainable.

This shell script contains most of the functionality of the Makefile
version. It can be run like this:

  # This will build gdb and compress with bzip2, gzip and xz
  ./src-release.sh -bgx gdb

The functionality that has been removed is building tarballs for insight
and gnats, which were broken in the current repository layout and
gas+binutils which amounted to basically the same thing as binutils so
didn't seem worth keeping. The script always builds a tar file,
compression is optional, so no need for specific commands to build tar
files rather than compressed tar files.

The build of gas releases has been fixed, as has extraction of version
numbers which had been broken since the switch to configure.ac files
and AC_INIT.

I also removed the code for distcleaning the intl sub-directory as it
seems like that issue has been fixed.

The script is capable of compressing with bzip2, gzip and xz which
should cover all formats available on gnu.org.

I tested it by producing release tarballs which are substantially
identical to the ones produced by the src-release script.

ChangeLog:

2014-08-07  Will Newton  <will.newton@linaro.org>

	* src-release.sh: New file.
---
 src-release.sh | 353 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 353 insertions(+)
 create mode 100755 src-release.sh

diff --git a/src-release.sh b/src-release.sh
new file mode 100755
index 0000000..3d41269
--- /dev/null
+++ b/src-release.sh
@@ -0,0 +1,353 @@
+#!/bin/bash
+#   Copyright (C) 1990-2014 Free Software Foundation
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+#
+
+# This script creates release packages for gdb, binutils, and other
+# packages which live in src.  It used to be implemented as the src-release
+# Makefile and prior to that was part of the top level Makefile, but that
+# turned out to be very messy and hard to maintain.
+
+set -e
+
+BZIPPROG=bzip2
+GZIPPROG=gzip
+XZPROG=xz
+MD5PROG=md5sum
+MAKE=make
+CC=gcc
+CXX=g++
+
+# Default to avoid splitting info files by setting the threshold high.
+MAKEINFOFLAGS=--split-size=5000000
+
+#
+# Support for building net releases
+
+# Files in devo used in any net release.
+DEVO_SUPPORT="README Makefile.in configure configure.ac \
+	config.guess config.sub config move-if-change \
+	COPYING COPYING.LIB install-sh config-ml.in symlink-tree \
+	mkinstalldirs ltmain.sh missing ylwrap \
+	libtool.m4 ltsugar.m4 ltversion.m4 ltoptions.m4 \
+	Makefile.def Makefile.tpl src-release config.rpath \
+	ChangeLog MAINTAINERS README-maintainer-mode \
+	lt~obsolete.m4 ltgcc.m4 depcomp mkdep compile \
+	COPYING3 COPYING3.LIB"
+
+# Files in devo/etc used in any net release.
+ETC_SUPPORT="Makefile.in configure configure.in standards.texi \
+	make-stds.texi standards.info* configure.texi configure.info* \
+	ChangeLog configbuild.* configdev.* fdl.texi texi2pod.pl gnu-oids.texi"
+
+# Get the version number of a given tool
+getver()
+{
+    tool=$1
+    if grep 'AC_INIT.*BFD_VERSION' $tool/configure.ac >/dev/null 2>&1; then
+	bfd/configure --version | sed -n -e '1s,.* ,,p'
+    elif test -f $tool/common/create-version.sh; then
+	$tool/common/create-version.sh $tool 'dummy-host' 'dummy-target' VER.tmp
+	cat VER.tmp | grep 'version\[\]' | sed 's/.*"\([^"]*\)".*/\1/' | sed 's/-cvs$//'
+        rm -f VER.tmp
+    elif test -f $tool/version.in; then
+	head -1 $tool/version.in
+    else
+	echo VERSION
+    fi
+}
+
+# Setup build directory for building release tarball
+do_proto_toplev()
+{
+    package=$1
+    ver=$2
+    tool=$3
+    support_files=$4
+    echo "==> Making $package-$ver/"
+    # Take out texinfo from a few places.
+    sed -e '/^all\.normal: /s/\all-texinfo //' \
+	-e '/^	install-texinfo /d' \
+	<Makefile.in >tmp
+    mv -f tmp Makefile.in
+    #
+    ./configure i686-pc-linux-gnu
+    $MAKE configure-host configure-target \
+	ALL_GCC="" ALL_GCC_C="" ALL_GCC_CXX="" \
+	CC_FOR_TARGET="$CC" CXX_FOR_TARGET="$CXX"
+    # Make links, and run "make diststuff" or "make info" when needed.
+    rm -rf proto-toplev
+    mkdir proto-toplev
+    dirs="$DEVO_SUPPORT $support_files $tool"
+    for d in $dirs ; do
+	if [ -d $d ]; then
+	    if [ ! -f $d/Makefile ] ; then
+		true
+	    elif grep '^diststuff:' $d/Makefile >/dev/null ; then
+		(cd $d ; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" diststuff) \
+		    || exit 1
+	    elif grep '^info:' $d/Makefile >/dev/null ; then
+		(cd $d ; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" info) \
+		    || exit 1
+	    fi
+	    if [ -d $d/proto-$d.dir ]; then
+		ln -s ../$d/proto-$d.dir proto-toplev/$d
+	    else
+		ln -s ../$d proto-toplev/$d
+	    fi
+	else
+	    if (echo x$d | grep / >/dev/null); then
+	      mkdir -p proto-toplev/`dirname $d`
+	      x=`dirname $d`
+	      ln -s ../`echo $x/ | sed -e 's,[^/]*/,../,g'`$d proto-toplev/$d
+	    else
+	      ln -s ../$d proto-toplev/$d
+	    fi
+	  fi
+	done
+	(cd etc; $MAKE MAKEINFOFLAGS="$MAKEINFOFLAGS" info)
+	$MAKE distclean
+	mkdir proto-toplev/etc
+	(cd proto-toplev/etc;
+	    for i in $ETC_SUPPORT; do
+		ln -s ../../etc/$i .
+		done)
+	#
+	# Take out texinfo from configurable dirs
+	rm proto-toplev/configure.ac
+	sed -e '/^host_tools=/s/texinfo //' \
+	    <configure.ac >proto-toplev/configure.ac
+	#
+	mkdir proto-toplev/texinfo
+	ln -s ../../texinfo/texinfo.tex	proto-toplev/texinfo/
+	if test -r texinfo/util/tex3patch ; then
+	    mkdir proto-toplev/texinfo/util && \
+		ln -s ../../../texinfo/util/tex3patch proto-toplev/texinfo/util
+	else
+	    true
+	fi
+	chmod -R og=u . || chmod og=u `find . -print`
+	#
+	# Create .gmo files from .po files.
+	for f in `find . -name '*.po' -type f -print`; do
+	    msgfmt -o `echo $f | sed -e 's/\.po$/.gmo/'` $f
+	done
+	#
+	rm -f $package-$ver
+	ln -s proto-toplev $package-$ver
+}
+
+CVS_NAMES='-name CVS -o -name .cvsignore'
+
+# Add an md5sum to the built tarball
+do_md5sum()
+{
+    echo "==> Adding md5 checksum to top-level directory"
+    (cd proto-toplev && find * -follow \( $CVS_NAMES \) -prune \
+	-o -type f -print \
+	| xargs $MD5PROG > ../md5.new)
+    rm -f proto-toplev/md5.sum
+    mv md5.new proto-toplev/md5.sum
+}
+
+# Build the release tarball
+do_tar()
+{
+    package=$1
+    ver=$2
+    echo "==> Making $package-$ver.tar"
+    rm -f $package-$ver.tar
+    find $package-$ver -follow \( $CVS_NAMES \) -prune \
+	-o -type f -print \
+	| tar cTfh - $package-$ver.tar
+}
+
+# Compress the output with bzip2
+do_bz2()
+{
+    package=$1
+    ver=$2
+    echo "==> Bzipping $package-$ver.tar.bz2"
+    rm -f $package-$ver.tar.bz2
+    $BZIPPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with gzip
+do_gz()
+{
+    package=$1
+    ver=$2
+    echo "==> Gzipping $package-$ver.tar.gz"
+    rm -f $package-$ver.tar.gz
+    $GZIPPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with xz
+do_xz()
+{
+    package=$1
+    ver=$2
+    echo "==> Xzipping $package-$ver.tar.xz"
+    rm -f $package-$ver.tar.xz
+    $XZPROG -k -v -9 $package-$ver.tar
+}
+
+# Compress the output with all selected compresion methods
+do_compress()
+{
+    package=$1
+    ver=$2
+    compressors=$3
+    for comp in $compressors; do
+	case $comp in
+	    bz2)
+		do_bz2 $package $ver;;
+	    gz)
+		do_gz $package $ver;;
+	    xz)
+		do_xz $package $ver;;
+	    *)
+		echo "Unknown compression method: $comp" && exit 1;;
+	esac
+    done
+}
+
+# Add djunpack.bat the tarball
+do_djunpack()
+{
+    package=$1
+    ver=$2
+    echo "==> Adding updated djunpack.bat to top-level directory"
+    echo - 's /gdb-[0-9\.]*/$package-'"$ver"'/'
+    sed < djunpack.bat > djunpack.new \
+	-e 's/gdb-[0-9][0-9\.]*/$package-'"$ver"'/'
+    rm -f proto-toplev/djunpack.bat
+    mv djunpack.new proto-toplev/djunpack.bat
+}
+
+# Create a release package, tar it and compress it
+tar_compress()
+{
+    package=$1
+    tool=$2
+    support_files=$3
+    compressors=$4
+    ver=$(getver $tool)
+    do_proto_toplev $package $ver $tool "$support_files"
+    do_md5sum
+    do_tar $package $ver
+    do_compress $package $ver "$compressors"
+}
+
+# Create a gdb release package, tar it and compress it
+gdb_tar_compress()
+{
+    package=$1
+    tool=$2
+    support_files=$3
+    compressors=$4
+    ver=$(getver $tool)
+    do_proto_toplev $package $ver $tool "$support_files"
+    do_md5sum
+    do_djunpack $package $ver
+    do_tar $package $ver
+    do_compress $package $ver "$compressors"
+}
+
+# The FSF "binutils" release includes gprof and ld.
+BINUTILS_SUPPORT_DIRS="bfd gas include libiberty opcodes ld elfcpp gold gprof intl setup.com makefile.vms cpu"
+binutils_release()
+{
+    compressors=$1
+    package=binutils
+    tool=binutils
+    tar_compress $package $tool "$BINUTILS_SUPPORT_DIRS" "$compressors"
+}
+
+GAS_SUPPORT_DIRS="bfd include libiberty opcodes intl setup.com makefile.vms"
+gas_release()
+{
+    compressors=$1
+    package=gas
+    tool=gas
+    tar_compress $package $tool "$GAS_SUPPORT_DIRS" "$compressors"
+}
+
+GDB_SUPPORT_DIRS="bfd include libiberty opcodes readline sim intl libdecnumber cpu"
+gdb_release()
+{
+    compressors=$1
+    package=gdb
+    tool=gdb
+    gdb_tar_compress $package $tool "$GDB_SUPPORT_DIRS" "$compressors"
+}
+
+# Corresponding to the CVS "sim" module.
+SIM_SUPPORT_DIRS="bfd opcodes libiberty include intl gdb/version.in makefile.vms"
+sim_release()
+{
+    compressors=$1
+    package=sim
+    tool=sim
+    tar_compress $package $tool "$SIM_SUPPORT_DIRS" "$compressors"
+}
+
+usage()
+{
+    echo "src-release.sh <options> <release>"
+    echo "options:"
+    echo "  -b: Compress with bzip2"
+    echo "  -g: Compress with gzip"
+    echo "  -x: Compress with xz"
+    exit 1
+}
+
+build_release()
+{
+    release=$1
+    compressors=$2
+    case $release in
+	binutils)
+	    binutils_release "$compressors";;
+	gas)
+	    gas_release "$compressors";;
+	gdb)
+	    gdb_release "$compressors";;
+	sim)
+	    sim_release "$compressors";;
+	*)
+	    echo "Unknown release name: $release" && usage;;
+    esac
+}
+
+compressors=""
+
+while getopts ":gbx" opt; do
+    case $opt in
+	b)
+	    compressors="$compressors bz2";;
+	g)
+	    compressors="$compressors gz";;
+	x)
+	    compressors="$compressors xz";;
+	\?)
+	    echo "Invalid option: -$OPTARG" && usage;;
+  esac
+done
+shift $((OPTIND -1))
+release=$1
+
+build_release $release "$compressors"
-- 
1.9.3