Termux
/
termux-packages
mirror of https://github.com/termux/termux-packages
1
0
Fork 0
Code Issues Releases Wiki Activity

build-package: download dependencies recursively when fast-building 

Use scripts/buildorder.py with a new -i flag to get all dependencies
(including subpackages). The script now also spits out both package
name and package dir, to make it easier to build packages from another
repo.
This commit is contained in:
Henrik Grimler 2019-03-02 23:34:54 +01:00 committed by Leonid Pliushch
parent 379f2452f3
commit 10fdf172d5
4 changed files with 85 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
build-all.sh
View File

@ -55,28 +55,27 @@ fi
exec > >(tee -a $BUILDALL_DIR/ALL.out)
exec 2> >(tee -a $BUILDALL_DIR/ALL.err >&2)
trap "echo ERROR: See $BUILDALL_DIR/\${package}.err" ERR
trap "echo ERROR: See $BUILDALL_DIR/\${PKG}.err" ERR
for package_path in $(cat $BUILDORDER_FILE); do
package=$(basename $package_path)
while read PKG PKG_DIR; do
# Check build status (grepping is a bit crude, but it works)
if [ -e $BUILDSTATUS_FILE ] && grep "^$package\$" $BUILDSTATUS_FILE >/dev/null; then
echo "Skipping $package"
if [ -e $BUILDSTATUS_FILE ] && grep "^$PKG\$" $BUILDSTATUS_FILE >/dev/null; then
echo "Skipping $PKG"
continue
fi
echo -n "Building $package... "
echo -n "Building $PKG... "
BUILD_START=$(date "+%s")
bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \
${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \
> $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err
${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $PKG_DIR \
> $BUILDALL_DIR/${PKG}.out 2> $BUILDALL_DIR/${PKG}.err
BUILD_END=$(date "+%s")
BUILD_SECONDS=$(( $BUILD_END - $BUILD_START ))
echo "done in $BUILD_SECONDS"
# Update build status
echo "$package" >> $BUILDSTATUS_FILE
done
echo "$PKG" >> $BUILDSTATUS_FILE
done<${BUILDORDER_FILE}
# Update build status
rm -f $BUILDSTATUS_FILE

18
scripts/build/termux_extract_dep_info.sh
View File

@ -1,13 +1,14 @@
termux_extract_dep_info() {
package=$1
if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then
PKG=$1
PKG_DIR=$2
if [ "$PKG" != "$(basename ${PKG_DIR})" ]; then
# We are dealing with a subpackage
TERMUX_ARCH=$(
# set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed
# set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to parent package's value and override if needed
TERMUX_PKG_PLATFORM_INDEPENDENT=""
source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh
source ${PKG_DIR}/build.sh
TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT
source $(find packages/ -name "$package.subpackage.sh")
source ${PKG_DIR}/${PKG}.subpackage.sh
if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then
echo all
else
@ -15,10 +16,9 @@ termux_extract_dep_info() {
fi
)
package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh")))
elif [ "${package/-dev/}-dev" == "${package}" ]; then
elif [ "${PKG/-dev/}-dev" == "${PKG}" ]; then
# dev package
package=${package/-dev/}
PKG=${PKG/-dev/}
fi
(
# Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't
@ -26,7 +26,7 @@ termux_extract_dep_info() {
# deps that should have the default values
TERMUX_PKG_PLATFORM_INDEPENDENT=""
TERMUX_PKG_REVISION="0"
source packages/$package/build.sh
source ${PKG_DIR}/build.sh
if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi
if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then
TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION"

42
scripts/build/termux_step_start_build.sh
View File

@ -14,36 +14,21 @@ termux_step_start_build() {
if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then
# Download dependencies
local PKG DEP_ARCH DEP_VERSION DEB_FILE _PKG_DEPENDS _PKG_BUILD_DEPENDS _SUBPKG_DEPENDS=""
# remove (>= 1.0) and similar version tags:
_PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS// /} | sed "s/[(][^)]*[)]//g")
_PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS// /}
# Also download subpackages dependencies (except the parent package):
for SUBPKG in packages/$TERMUX_PKG_NAME/*.subpackage.sh; do
test -e $SUBPKG || continue
_SUBPKG_DEPENDS+=" $(. $SUBPKG; echo $TERMUX_SUBPKG_DEPENDS | sed s%$TERMUX_PKG_NAME%%g)"
done
for PKG in $(echo ${_PKG_DEPENDS//,/ } ${_SUBPKG_DEPENDS//,/ } ${_PKG_BUILD_DEPENDS//,/ } | tr ' ' '\n' | sort -u); do
# handle "or" in dependencies (use first one):
if [ ! "$PKG" = "${PKG/|/}" ]; then PKG=$(echo "$PKG" | sed "s%|.*%%"); fi
while read PKG PKG_DIR; do
if [ -z $PKG ]; then
continue
fi
# llvm doesn't build if ndk-sysroot is installed:
if [ "$PKG" = "ndk-sysroot" ]; then continue; fi
read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info "$PKG")
read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info $PKG "${PKG_DIR}")
if [ ! "$TERMUX_QUIET_BUILD" = true ]; then
echo "Downloading dependency $PKG@$DEP_VERSION if necessary..."
fi
if ! termux_download_deb $PKG $DEP_ARCH $DEP_VERSION; then
if find packages/ -type f -name ${PKG}.subpackage.sh -exec false {} +; then
echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
./build-package.sh -a $TERMUX_ARCH -I "$PKG"
continue
else
# subpackage, so we need to build parent package
PARENT=$(dirname $(find packages/ -name "${PKG}.subpackage.sh"))
echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building parent $PARENT instead"
./build-package.sh -a $TERMUX_ARCH -I $PARENT
fi
echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
./build-package.sh -a $TERMUX_ARCH -I "${PKG_DIR}"
continue
else
if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "extracting $PKG..."; fi
(
@ -64,14 +49,17 @@ termux_step_start_build() {
fi
mkdir -p /data/data/.built-packages
echo "$DEP_VERSION" > "/data/data/.built-packages/$PKG"
done
done<<<$(./scripts/buildorder.py -i "$TERMUX_PKG_BUILDER_DIR")
elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then
# Build dependencies
for PKG in $(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR"); do
while read PKG PKG_DIR; do
if [ -z $PKG ]; then
continue
fi
echo "Building dependency $PKG if necessary..."
# Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one
./build-package.sh -a $TERMUX_ARCH -s "$PKG"
done
./build-package.sh -a $TERMUX_ARCH -s "${PKG_DIR}"
done<<<$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR")
fi
TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION

67
scripts/buildorder.py
View File

@ -107,13 +107,26 @@ class TermuxSubPackage:
self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
self.parent = parent
self.deps = parse_build_file_dependencies(subpackage_file_path)
self.dir = parent.dir
self.needed_by = set() # Populated outside constructor, reverse of deps.
def __repr__(self):
return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
def read_packages_from_directories(directories):
def recursive_dependencies(self, pkgs_map):
"""All the dependencies of the subpackage, both direct and indirect.
Only relevant when building in fast-build mode"""
result = []
for dependency_name in sorted(self.deps):
dependency_package = pkgs_map[dependency_name]
result += dependency_package.recursive_dependencies(pkgs_map)
result += [dependency_package]
return unique_everseen(result)
def read_packages_from_directories(directories, fast_build_mode):
"""Construct a map from package name to TermuxPackage.
For subpackages this maps from the subpackage name to the parent package."""
Subpackages are mapped to the parent package if fast_build_mode is false."""
pkgs_map = {}
all_packages = []
@ -132,6 +145,8 @@ def read_packages_from_directories(directories):
for subpkg in new_package.subpkgs:
if subpkg.name in pkgs_map:
die('Duplicated package: ' + subpkg.name)
elif fast_build_mode:
pkgs_map[subpkg.name] = subpkg
else:
pkgs_map[subpkg.name] = new_package
all_packages.append(subpkg)
@ -141,7 +156,7 @@ def read_packages_from_directories(directories):
if dependency_name not in pkgs_map:
die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
dep_pkg = pkgs_map[dependency_name]
if not isinstance(pkg, TermuxSubPackage):
if fast_build_mode or not isinstance(pkg, TermuxSubPackage):
dep_pkg.needed_by.add(pkg)
return pkgs_map
@ -198,7 +213,7 @@ def generate_full_buildorder(pkgs_map):
return build_order
def generate_target_buildorder(target_path, pkgs_map):
def generate_target_buildorder(target_path, pkgs_map, fast_build_mode):
"Generate a build order for building the dependencies of the specified package."
if target_path.endswith('/'):
target_path = target_path[:-1]
@ -209,28 +224,50 @@ def generate_target_buildorder(target_path, pkgs_map):
def main():
"Generate the build order either for all packages or a specific one."
packages_directories = ['packages']
full_buildorder = len(sys.argv) == 1
import argparse
parser = argparse.ArgumentParser(description='Generate order in which to build dependencies for a package. Generates')
parser.add_argument('-i', default=False, action='store_true',
help='Generate dependency list for fast-build mode. This includes subpackages in output since these can be downloaded.')
parser.add_argument('package', nargs='?',
help='Package to generate dependency list for.')
parser.add_argument('package_dirs', nargs='*',
help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.')
args = parser.parse_args()
fast_build_mode = args.i
package = args.package
packages_directories = args.package_dirs + ['packages']
if not package:
full_buildorder = True
else:
full_buildorder = False
if fast_build_mode and full_buildorder:
die('-i mode does not work when building all packages')
if not full_buildorder:
packages_real_path = os.path.realpath('packages')
for path in sys.argv[1:]:
for path in packages_directories:
if not os.path.isdir(path):
die('Not a directory: ' + path)
if path.endswith('/'):
path = path[:-1]
parent_path = os.path.dirname(path)
if packages_real_path != os.path.realpath(parent_path):
packages_directories.append(parent_path)
pkgs_map = read_packages_from_directories(packages_directories)
if package:
if package[-1] == "/":
package = package[:-1]
if not os.path.isdir(package):
die('Not a directory: ' + package)
if not os.path.relpath(os.path.dirname(package), '.') in packages_directories:
packages_directories.insert(0, os.path.dirname(package))
pkgs_map = read_packages_from_directories(packages_directories, fast_build_mode)
if full_buildorder:
build_order = generate_full_buildorder(pkgs_map)
else:
build_order = generate_target_buildorder(sys.argv[1], pkgs_map)
build_order = generate_target_buildorder(package, pkgs_map, fast_build_mode)
for pkg in build_order:
print(pkg.dir)
print("%-30s %s" % (pkg.name, pkg.dir))
if __name__ == '__main__':
main()