Tweak build script and docker setup

- Output deb files to debs/ folder by default. - Use $HOME/.termux-build for build folder by default (changed from previous $HOME/termux). - Make scripts/run-docker.sh re-use a single container instead of creating new ones for each run. - Avoid mounting the whole Termux build folder in docker. This fixes #294, mmap not supported host folders under OS X. - Add a scripts/update-docker.sh utility for checking for an updated docker image.. - Make build-package.sh support the [-a ARCH] flag to specify arch. - Make aarch64 the default arch (from previous arm). - Try to improve the README a bit.
2016-06-03 18:41:30 -04:00 · 2016-06-03 18:41:30 -04:00 · a005794323
parent 441fa39553
commit a005794323
7 changed files with 166 additions and 81 deletions
--- a/.gitignore
+++ b/.gitignore
@ -5,3 +5,4 @@
 Session.vim
 .netrwhist
 *~
+/debs/
--- a/README.md
+++ b/README.md
@ -27,16 +27,22 @@ adapted from the below setup for Ubuntu:

 Build environment using Docker
 ==============================
-A Docker container configured for building images can be downloaded an run with:
+A Docker container configured for building images can be downloaded and run with:

    ./scripts/run-docker.sh

 This will set you up with a interactive prompt in a container, where this source folder
 is mounted as the /root/termux-packages data volume, so changes are kept in sync between
-the host and the container when trying things out before committing.
+the host and the container when trying things out before committing, and built deb files
+will be available on the host in the `debs/` directory just as when building on the host.

-The build output folder is mounted to $HOME/termux, so deb files can be found in
-$HOME/termux/_deb on the host for trying them out on a device or emulator.
+Build commands can be given to be executed in the docker container directly:
+
+    ./scripts/run-docker.sh ./build-package.sh libandroid-support
+
+will launch the docker container, execute the `./build-package.sh libandroid-support`
+command inside it and afterwards return you to the host prompt, with the newly built
+deb in `debs/` to try out.


 Building a package
@ -45,13 +51,19 @@ The basic build operation is to run `./build-package.sh $PKG`, which:

 * Sets up a patched stand-alone Android NDK toolchain if necessary.

-* Reads `packages/$PKG/build.s`h to find out where to find the source code of the  package and how to build it.
+* Reads `packages/$PKG/build.sh` to find out where to find the source code of the package and how to build it.
+
+* Extracts the source in `$HOME/.termux-build/$PKG/src`.

 * Applies all patches in packages/$PKG/\*.patch.

-* Builds the package and installs it to `$PREFIX`.
+* Builds the package under `$HOME/.termux-build/$PKG/` (either in the build/ directory there or in the
+  src/ directory if the package is specified to build in the src dir) and installs it to `$PREFIX`.

-* Creates a dpkg package file for distribution in `$HOME/termux/_deb`.
+* Extracts modified files in `$PREFIX` into `$HOME/.termux-build/$PKG/massage` and massages the
+  files there for distribution (removes some files, splits it up in sub-packages, modifies elf files).
+
+* Creates a deb package file for distribution in `debs/`.

 Reading `build-package.sh` is the best way to understand what is going on.

--- a/build-all.sh
+++ b/build-all.sh
@ -3,24 +3,27 @@

 set -e -u -o pipefail

+# Read settings from .termuxrc if existing
+test -f $HOME/.termuxrc && . $HOME/.termuxrc
+: ${TERMUX_TOPDIR:="$HOME/.termux-build"}
+
 BUILDSCRIPT=`dirname $0`/build-package.sh
-BUILDORDER_FILE=$HOME/termux/_buildall/buildorder.txt
-BUILDSTATUS_FILE=$HOME/termux/_buildall/buildstatus.txt
+BUILDORDER_FILE=$TERMUX_TOPDIR/_buildall/buildorder.txt
+BUILDSTATUS_FILE=$TERMUX_TOPDIR/_buildall/buildstatus.txt

 if [ -e $BUILDORDER_FILE ]; then
 	echo "Using existing buildorder file: $BUILDORDER_FILE"
 else
-	rm -Rf $HOME/termux/* /data/data $HOME/termux/_buildall
-	mkdir -p $HOME/termux/_buildall
+	mkdir -p $TERMUX_TOPDIR/_buildall
 	./scripts/buildorder.py > $BUILDORDER_FILE
 fi
 if [ -e $BUILDSTATUS_FILE ]; then
 	echo "Continuing build-all from: $BUILDSTATUS_FILE"
 fi

-exec >  >(tee -a $HOME/termux/_buildall/ALL.out)
-exec 2> >(tee -a $HOME/termux/_buildall/ALL.err >&2)
-trap 'echo failure; echo See: $HOME/termux/_buildall/${package}.err' ERR
+exec >  >(tee -a $TERMUX_TOPDIR/_buildall/ALL.out)
+exec 2> >(tee -a $TERMUX_TOPDIR/_buildall/ALL.err >&2)
+trap 'echo failure; echo See: $TERMUX_TOPDIR/_buildall/${package}.err' ERR

 for package in `cat $BUILDORDER_FILE`; do
 	# Check build status (grepping is a bit crude, but it works)
@ -32,7 +35,7 @@ for package in `cat $BUILDORDER_FILE`; do

 	echo -n "Building $package... "
 	BUILD_START=`date "+%s"`
-	bash -x $BUILDSCRIPT $package > $HOME/termux/_buildall/${package}.out 2> $HOME/termux/_buildall/${package}.err
+	bash -x $BUILDSCRIPT $package > $TERMUX_TOPDIR/_buildall/${package}.out 2> $TERMUX_TOPDIR/_buildall/${package}.err
 	BUILD_END=`date "+%s"`
 	BUILD_SECONDS=$(( $BUILD_END - $BUILD_START ))
 	echo "done in $BUILD_SECONDS"
--- a/build-package.sh
+++ b/build-package.sh
@ -2,10 +2,72 @@

 set -e -o pipefail -u

-if [ "$#" -ne 1 ]; then echo "ERROR: Specify one argument (name of or path to package)"; exit 1; fi
-export TERMUX_PKG_NAME=`basename $1`
-export TERMUX_SCRIPTDIR=`cd $(dirname $0); pwd`
+# Read settings from .termuxrc if existing:
+test -f $HOME/.termuxrc && . $HOME/.termuxrc

+# Configurable settings:
+: ${ANDROID_HOME:="${HOME}/lib/android-sdk"}
+: ${NDK:="${HOME}/lib/android-ndk"}
+: ${TERMUX_MAKE_PROCESSES:='4'}
+: ${TERMUX_TOPDIR:="$HOME/.termux-build"}
+: ${TERMUX_ARCH:="aarch64"} # (arm|aarch64|i686|x86_64) - the x86_64 arch is not yet used or tested.
+: ${TERMUX_CLANG:=""} # Set to non-empty to use clang.
+: ${TERMUX_PREFIX:='/data/data/com.termux/files/usr'}
+: ${TERMUX_ANDROID_HOME:='/data/data/com.termux/files/home'}
+: ${TERMUX_DEBUG:=""}
+: ${TERMUX_PROCESS_DEB:=""}
+: ${TERMUX_API_LEVEL:="21"}
+: ${TERMUX_ANDROID_BUILD_TOOLS_VERSION:="23.0.3"}
+: ${TERMUX_NDK_VERSION:="11"}
+
+# Handle command-line arguments:
+show_usage () {
+    echo "Usage: ./build-package.sh [-a ARCH] PACKAGE"
+    echo "Build a package."
+    echo ""
+    exit 1
+}
+while getopts :a:h option
+do
+    case "$option" in
+        a) TERMUX_ARCH="$OPTARG";;
+        h) show_usage;;
+        ?) echo "./build-package.sh: illegal option -$OPTARG"; exit 1;;
+    esac
+done
+shift $(($OPTIND-1))
+if [ "$#" -ne 1 ]; then show_usage; fi
+
+# Check the NDK:
+if [ ! -d "$NDK" ]; then
+	echo 'ERROR: $NDK not defined as pointing at a directory - define it pointing at a android NDK installation!'
+	exit 1
+fi
+if grep -s -q "Pkg.Revision = $TERMUX_NDK_VERSION" $NDK/source.properties; then
+	:
+else
+	echo "Wrong NDK version - we need $TERMUX_NDK_VERSION"
+	exit 1
+fi
+
+# Compute standalone toolchain dir, bitness of arch and name of host platform:
+TERMUX_STANDALONE_TOOLCHAIN="$TERMUX_TOPDIR/_lib/android-standalone-toolchain-${TERMUX_ARCH}-api${TERMUX_API_LEVEL}-"
+if [ "$TERMUX_CLANG" = "" ]; then
+	TERMUX_STANDALONE_TOOLCHAIN+="gcc4.9"
+else
+	TERMUX_STANDALONE_TOOLCHAIN+="clang38"
+fi
+if [ "x86_64" = $TERMUX_ARCH -o "aarch64" = $TERMUX_ARCH ]; then
+	TERMUX_ARCH_BITS=64
+else
+	TERMUX_ARCH_BITS=32
+fi
+TERMUX_HOST_PLATFORM="${TERMUX_ARCH}-linux-android"
+if [ $TERMUX_ARCH = "arm" ]; then TERMUX_HOST_PLATFORM="${TERMUX_HOST_PLATFORM}eabi"; fi
+
+# Check the package to build:
+TERMUX_PKG_NAME=`basename $1`
+export TERMUX_SCRIPTDIR=`cd $(dirname $0); pwd`
 if [[ $1 == *"/"* ]]; then
  # Path to directory which may be outside this repo:
  if [ ! -d $1 ]; then echo "ERROR: '$1' seems to be a path but is not a directory"; exit 1; fi
@ -14,40 +76,23 @@ else
  # Package name:
  export TERMUX_PKG_BUILDER_DIR=$TERMUX_SCRIPTDIR/packages/$TERMUX_PKG_NAME
 fi
-export TERMUX_PKG_BUILDER_SCRIPT=$TERMUX_PKG_BUILDER_DIR/build.sh
-if test ! -f $TERMUX_PKG_BUILDER_SCRIPT; then echo "ERROR: No build.sh script at supposed package dir $TERMUX_PKG_BUILDER_DIR!"; exit 1; fi
+TERMUX_PKG_BUILDER_SCRIPT=$TERMUX_PKG_BUILDER_DIR/build.sh
+if test ! -f $TERMUX_PKG_BUILDER_SCRIPT; then
+	echo "ERROR: No build.sh script at supposed package dir $TERMUX_PKG_BUILDER_DIR!"
+	exit 1
+fi

-echo "termux - building $1..."
+# Handle 'all' arch:
+if [ $TERMUX_ARCH = 'all' ]; then
+	for arch in 'arm' 'i686' 'aarch64'; do
+		./build-package.sh -a $arch $1
+	done
+	exit
+fi
+
+echo "termux - building $1 for arch $TERMUX_ARCH..."
 test -t 1 && printf "\033]0;$1...\007"

-# Read settings from .termuxrc if existing
-test -f $HOME/.termuxrc && . $HOME/.termuxrc
-
-# Configurable settings
-: ${NDK:="${HOME}/lib/android-ndk"}
-: ${ANDROID_HOME:="${HOME}/lib/android-sdk"}
-if [ ! -d "$NDK" ]; then echo 'ERROR: $NDK not defined as pointing at a directory - define it pointing at a android NDK installation!'; exit 1; fi
-: ${TERMUX_MAKE_PROCESSES:='4'}
-: ${TERMUX_TOPDIR:="$HOME/termux"}
-: ${TERMUX_ARCH:="arm"} # (arm|aarch64|i686|x86_64) - the x86_64 arch is not yet used or tested.
-TERMUX_ARCH_BITS="32"
-if [ "x86_64" = $TERMUX_ARCH -o "aarch64" = $TERMUX_ARCH ]; then
-	TERMUX_ARCH_BITS="64"
-fi
-: ${TERMUX_CLANG:=""} # Set to non-empty to use clang.
-: ${TERMUX_HOST_PLATFORM:="${TERMUX_ARCH}-linux-android"}
-if [ $TERMUX_ARCH = "arm" ]; then TERMUX_HOST_PLATFORM="${TERMUX_HOST_PLATFORM}eabi"; fi
-: ${TERMUX_PREFIX:='/data/data/com.termux/files/usr'}
-: ${TERMUX_ANDROID_HOME:='/data/data/com.termux/files/home'}
-: ${TERMUX_DEBUG:=""}
-: ${TERMUX_PROCESS_DEB:=""}
-: ${TERMUX_API_LEVEL:="21"}
-if [ "$TERMUX_CLANG" = "" ]; then
-	: ${TERMUX_STANDALONE_TOOLCHAIN:="$HOME/lib/android-standalone-toolchain-${TERMUX_ARCH}-api${TERMUX_API_LEVEL}-gcc4.9"}
-else
-	: ${TERMUX_STANDALONE_TOOLCHAIN:="$HOME/lib/android-standalone-toolchain-${TERMUX_ARCH}-api${TERMUX_API_LEVEL}-clang38"}
-fi
-: ${TERMUX_ANDROID_BUILD_TOOLS_VERSION:="23.0.3"}
 # We do not put all of build-tools/$TERMUX_ANDROID_BUILD_TOOLS_VERSION/ into PATH
 # to avoid stuff like arm-linux-androideabi-ld there to conflict with ones from
 # the standalone toolchain.
@ -62,15 +107,6 @@ test `uname` = "Darwin" && TERMUX_TAR=gnutar
 export TERMUX_TOUCH="touch"
 test `uname` = "Darwin" && TERMUX_TOUCH=gtouch

-# Compute NDK version. We remove the first character (the r in e.g. r9d) to get a version number which can be used in packages):
-export TERMUX_NDK_VERSION=11
-if grep -s -q "Pkg.Revision = $TERMUX_NDK_VERSION" $NDK/source.properties; then
-	:
-else
-	echo "Wrong NDK version - we need $TERMUX_NDK_VERSION"
-	exit 1
-fi
-
 export prefix=${TERMUX_PREFIX} # prefix is used by some makefiles
 #export ACLOCAL="aclocal -I $TERMUX_PREFIX/share/aclocal"
 export AR=$TERMUX_HOST_PLATFORM-ar
@ -113,6 +149,11 @@ elif [ $TERMUX_ARCH = "i686" ]; then
 elif [ $TERMUX_ARCH = "aarch64" ]; then
 	LDFLAGS+=" -Wl,-rpath-link,$TERMUX_PREFIX/lib"
 	LDFLAGS+=" -Wl,-rpath-link,$TERMUX_STANDALONE_TOOLCHAIN/sysroot/usr/lib"
+elif [ $TERMUX_ARCH = "x86_64" ]; then
+	continue
+else
+	echo "Error: Invalid arch '$TERMUX_ARCH' - support arches are 'arm', 'i686', 'aarch64', 'x86_64'"
+	exit 1
 fi

 if [ -n "$TERMUX_DEBUG" ]; then
@ -153,8 +194,7 @@ if [ ! -d $TERMUX_STANDALONE_TOOLCHAIN ]; then
 	for f in $TERMUX_SCRIPTDIR/ndk_patches/*.patch; do
 		sed "s%\@TERMUX_PREFIX\@%${TERMUX_PREFIX}%g" $f | \
 			sed "s%\@TERMUX_HOME\@%${TERMUX_ANDROID_HOME}%g" | \
-			patch -p1;
-		echo "PATCHING FILE $f done!"
+			patch --silent -p1;
 	done
        # elf.h is taken from glibc since the elf.h in the NDK is lacking.
 	# sysexits.h is header-only and used by a few programs.
@ -162,8 +202,8 @@ if [ ! -d $TERMUX_STANDALONE_TOOLCHAIN ]; then
 fi

 export TERMUX_COMMON_CACHEDIR="$TERMUX_TOPDIR/_cache"
-export TERMUX_COMMON_DEBDIR="$TERMUX_TOPDIR/_deb"
-mkdir -p $TERMUX_COMMON_CACHEDIR $TERMUX_COMMON_DEBDIR
+export TERMUX_DEBDIR="$TERMUX_SCRIPTDIR/debs"
+mkdir -p $TERMUX_COMMON_CACHEDIR $TERMUX_DEBDIR

 TERMUX_PKG_BUILDDIR=$TERMUX_TOPDIR/$TERMUX_PKG_NAME/build
 TERMUX_PKG_CACHEDIR=$TERMUX_TOPDIR/$TERMUX_PKG_NAME/cache
@ -206,8 +246,6 @@ if [ -f "${TERMUX_ARCH_FILE}" ]; then
                mkdir -p $TERMUX_DATA_BACKUPDIRS
                TERMUX_DATA_PREVIOUS_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_PREVIOUS_ARCH
                TERMUX_DATA_CURRENT_BACKUPDIR=$TERMUX_DATA_BACKUPDIRS/$TERMUX_ARCH
-                echo "NOTE: Different archs - building for $TERMUX_ARCH, but current $TERMUX_PREVIOUS_ARCH"
-                echo "      Saving current /data/data to $TERMUX_DATA_PREVIOUS_BACKUPDIR"
                # Save current /data (removing old backup if any)
 		if test -e $TERMUX_DATA_PREVIOUS_BACKUPDIR; then
 			echo "ERROR: Directory already exists"
@ -216,7 +254,6 @@ if [ -f "${TERMUX_ARCH_FILE}" ]; then
                mv /data/data $TERMUX_DATA_PREVIOUS_BACKUPDIR
                # Restore new one (if any)
                if [ -d $TERMUX_DATA_CURRENT_BACKUPDIR ]; then
-                        echo "      Restoring old backupdir from $TERMUX_DATA_CURRENT_BACKUPDIR"
                        mv $TERMUX_DATA_CURRENT_BACKUPDIR /data/data
                fi
        fi
@ -342,7 +379,7 @@ termux_step_patch_package () {
 	for patch in $TERMUX_PKG_BUILDER_DIR/*.patch{$TERMUX_ARCH_BITS,}; do
 		test -f $patch && sed "s%\@TERMUX_PREFIX\@%${TERMUX_PREFIX}%g" $patch | \
 			sed "s%\@TERMUX_HOME\@%${TERMUX_ANDROID_HOME}%g" | \
-			patch -p1
+			patch --silent -p1
 	done

 	find . -name config.sub -exec chmod u+w '{}' \; -exec cp $TERMUX_COMMON_CACHEDIR/config.sub '{}' \;
@ -384,9 +421,6 @@ termux_step_configure () {
 	for f in $TERMUX_PREFIX/bin/*config; do
 		test -f $f && cp $f $TERMUX_PKG_TMPDIR/config-scripts
 	done
-	set +e +o pipefail
-	find $TERMUX_PKG_TMPDIR/config-scripts | xargs file | grep -F " script" | cut -f 1 -d : | xargs sed -i -E "s@^#\!/system/bin/sh@#\!/bin/sh@"
-	set -e -o pipefail
 	export PATH=$TERMUX_PKG_TMPDIR/config-scripts:$PATH

 	# See http://wiki.buici.com/xwiki/bin/view/Programing+C+and+C%2B%2B/Autoconf+and+RPL_MALLOC
@ -529,7 +563,6 @@ termux_step_massage () {
 		SUB_PKG_NAME=`basename $subpackage .subpackage.sh`
                # Default value is same as main package, but sub package may override:
                TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT
-		echo "$SUB_PKG_NAME => $subpackage"
                SUB_PKG_DIR=$TERMUX_TOPDIR/$TERMUX_PKG_NAME/subpackages/$SUB_PKG_NAME
                TERMUX_SUBPKG_DEPENDS=""
 		TERMUX_SUBPKG_CONFLICTS=""
@ -572,7 +605,7 @@ HERE
 		$TERMUX_TAR -cJf $SUB_PKG_PACKAGE_DIR/control.tar.xz .

                # Create the actual .deb file:
-                TERMUX_SUBPKG_DEBFILE=$TERMUX_COMMON_DEBDIR/${SUB_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${SUB_PKG_ARCH}.deb
+                TERMUX_SUBPKG_DEBFILE=$TERMUX_DEBDIR/${SUB_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${SUB_PKG_ARCH}.deb
 		ar cr $TERMUX_SUBPKG_DEBFILE \
 				   $TERMUX_COMMON_CACHEDIR/debian-binary \
 				   $SUB_PKG_PACKAGE_DIR/control.tar.xz \
@ -652,7 +685,7 @@ termux_step_post_extract_package
 if [ "x$TERMUX_PKG_HOSTBUILD" != "x" ]; then
 	cd $TERMUX_PKG_SRCDIR
 	for patch in $TERMUX_PKG_BUILDER_DIR/*.patch.beforehostbuild; do
-		test -f $patch && sed "s%\@TERMUX_PREFIX\@%${TERMUX_PREFIX}%g" $patch | patch -p1
+		test -f $patch && sed "s%\@TERMUX_PREFIX\@%${TERMUX_PREFIX}%g" $patch | patch --silent -p1
 	done

        if [ -f "$TERMUX_PKG_HOSTBUILD_DIR/TERMUX_BUILT_FOR_$TERMUX_PKG_VERSION" ]; then
@ -770,7 +803,7 @@ termux_step_create_debscripts
 # Create control.tar.xz
 $TERMUX_TAR -cJf $TERMUX_PKG_PACKAGEDIR/control.tar.xz .
 # In the .deb ar file there should be a file "debian-binary" with "2.0" as the content:
-TERMUX_PKG_DEBFILE=$TERMUX_COMMON_DEBDIR/${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb
+TERMUX_PKG_DEBFILE=$TERMUX_DEBDIR/${TERMUX_PKG_NAME}_${TERMUX_PKG_FULLVERSION}_${TERMUX_ARCH}.deb
 # Create the actual .deb file:
 ar cr $TERMUX_PKG_DEBFILE \
                   $TERMUX_COMMON_CACHEDIR/debian-binary \
--- a/clean-rebuild-all.sh
+++ b/clean-rebuild-all.sh
@ -1,5 +1,9 @@
 #!/bin/sh
 # clean-rebuild-all.sh - clean everything and rebuild

-rm -Rf /data/* $HOME/termux/* $HOME/lib/android-standalone-toolchain-*
+# Read settings from .termuxrc if existing
+test -f $HOME/.termuxrc && . $HOME/.termuxrc
+: ${TERMUX_TOPDIR:="$HOME/.termux-build"}
+
+rm -Rf /data/* $TERMUX_TOPDIR
 bash -x build-all.sh
--- a/scripts/run-docker.sh
+++ b/scripts/run-docker.sh
@ -1,15 +1,28 @@
 #!/bin/sh
 set -e -u

-# Create build output folder if necessary:
-mkdir -p $HOME/termux
+# Read settings from .termuxrc if existing
+test -f $HOME/.termuxrc && . $HOME/.termuxrc
+: ${TERMUX_TOPDIR:="$HOME/.termux-build"}

 IMAGE_NAME=termux/package-builder
+CONTAINER_NAME=termux-package-builder

-echo "Running container from image '$IMAGE_NAME'..."
+echo "Running container '$CONTAINER_NAME' from image '$IMAGE_NAME'..."
+
+docker start $CONTAINER_NAME > /dev/null 2> /dev/null || {
+	echo "Creating new container..."
+	docker run \
+		-d \
+		--name $CONTAINER_NAME \
+		-v $PWD:/root/termux-packages \
+		-t $IMAGE_NAME
+}
+
+if [ "$#" -eq  "0" ]; then
+	docker exec -it $CONTAINER_NAME bash
+else
+	docker exec -it $CONTAINER_NAME $@
+fi

-docker run \
-	-v $PWD:/root/termux-packages \
-	-v $HOME/termux:/root/termux \
-	-ti $IMAGE_NAME bash

--- a/scripts/update-docker.sh
+++ b/scripts/update-docker.sh
@ -0,0 +1,19 @@
+#!/bin/sh
+set -e -u
+
+CONTAINER=termux-package-builder
+IMAGE=termux/package-builder
+
+docker pull $IMAGE
+
+LATEST=`docker inspect --format "{{.Id}}" $IMAGE`
+RUNNING=`docker inspect --format "{{.Image}}" $CONTAINER`
+
+if [ $LATEST = $RUNNING ]; then
+	echo "Image '$IMAGE' used in the container '$CONTAINER' is already up to date"
+else
+	echo "Image '$IMAGE' used in the container '$CONTAINER' has been updated - removing the outdated container"
+	docker stop $CONTAINER
+	docker rm -f $CONTAINER
+fi
+