build-package: download dependencies recursively when fast-building

Use scripts/buildorder.py with a new -i flag to get all dependencies
(including subpackages). The script now also spits out both package
name and package dir, to make it easier to build packages from another
repo.

build-all.sh

@@ -55,28 +55,27 @@ fi
 
 exec >	>(tee -a $BUILDALL_DIR/ALL.out)
 exec 2> >(tee -a $BUILDALL_DIR/ALL.err >&2)
-trap "echo ERROR: See $BUILDALL_DIR/\${package}.err" ERR
+trap "echo ERROR: See $BUILDALL_DIR/\${PKG}.err" ERR
 
-for package_path in $(cat $BUILDORDER_FILE); do
+while read PKG PKG_DIR; do
-	package=$(basename $package_path)
 	# Check build status (grepping is a bit crude, but it works)
-	if [ -e $BUILDSTATUS_FILE ] && grep "^$package\$" $BUILDSTATUS_FILE >/dev/null; then
+	if [ -e $BUILDSTATUS_FILE ] && grep "^$PKG\$" $BUILDSTATUS_FILE >/dev/null; then
-		echo "Skipping $package"
+		echo "Skipping $PKG"
 		continue
 	fi
 
-	echo -n "Building $package... "
+	echo -n "Building $PKG... "
 	BUILD_START=$(date "+%s")
 	bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \
-		${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \
+		${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $PKG_DIR \
-		> $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err
+		> $BUILDALL_DIR/${PKG}.out 2> $BUILDALL_DIR/${PKG}.err
 	BUILD_END=$(date "+%s")
 	BUILD_SECONDS=$(( $BUILD_END - $BUILD_START ))
 	echo "done in $BUILD_SECONDS"
 
 	# Update build status
-	echo "$package" >> $BUILDSTATUS_FILE
+	echo "$PKG" >> $BUILDSTATUS_FILE
-done
+done<${BUILDORDER_FILE}
 
 # Update build status
 rm -f $BUILDSTATUS_FILE

scripts/build/termux_extract_dep_info.sh

@@ -1,13 +1,14 @@
 termux_extract_dep_info() {
-	package=$1
+	PKG=$1
-	if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then
+	PKG_DIR=$2
+	if [ "$PKG" != "$(basename ${PKG_DIR})" ]; then
 		# We are dealing with a subpackage
 		TERMUX_ARCH=$(
-			# set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed
+			# set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to parent package's value and override if needed
 			TERMUX_PKG_PLATFORM_INDEPENDENT=""
-			source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh
+			source ${PKG_DIR}/build.sh
 			TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT
-			source $(find packages/ -name "$package.subpackage.sh")
+			source ${PKG_DIR}/${PKG}.subpackage.sh
 			if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then
 				echo all
 			else
@@ -15,10 +16,9 @@ termux_extract_dep_info() {
 			fi
 		)
 
-		package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh")))
+	elif [ "${PKG/-dev/}-dev" == "${PKG}" ]; then
-	elif [ "${package/-dev/}-dev" == "${package}" ]; then
 		# dev package
-		package=${package/-dev/}
+		PKG=${PKG/-dev/}
 	fi
 	(
 		# Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't
@@ -26,7 +26,7 @@ termux_extract_dep_info() {
 		# deps that should have the default values
 		TERMUX_PKG_PLATFORM_INDEPENDENT=""
 		TERMUX_PKG_REVISION="0"
-		source packages/$package/build.sh
+		source ${PKG_DIR}/build.sh
 		if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi
 		if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then
 			TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION"

scripts/build/termux_step_start_build.sh

@@ -14,36 +14,21 @@ termux_step_start_build() {
 
 	if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then
 		# Download dependencies
-		local PKG DEP_ARCH DEP_VERSION DEB_FILE _PKG_DEPENDS _PKG_BUILD_DEPENDS _SUBPKG_DEPENDS=""
+		while read PKG PKG_DIR; do
-		# remove (>= 1.0) and similar version tags:
+			if [ -z $PKG ]; then
-		_PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS// /} | sed "s/[(][^)]*[)]//g")
+				continue
-		_PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS// /}
+			fi
-		# Also download subpackages dependencies (except the parent package):
-		for SUBPKG in packages/$TERMUX_PKG_NAME/*.subpackage.sh; do
-			test -e $SUBPKG || continue
-			_SUBPKG_DEPENDS+=" $(. $SUBPKG; echo $TERMUX_SUBPKG_DEPENDS | sed s%$TERMUX_PKG_NAME%%g)"
-		done
-		for PKG in $(echo ${_PKG_DEPENDS//,/ } ${_SUBPKG_DEPENDS//,/ } ${_PKG_BUILD_DEPENDS//,/ } | tr ' ' '\n' | sort -u); do
-			# handle "or" in dependencies (use first one):
-			if [ ! "$PKG" = "${PKG/|/}" ]; then PKG=$(echo "$PKG" | sed "s%|.*%%"); fi
 			# llvm doesn't build if ndk-sysroot is installed:
 			if [ "$PKG" = "ndk-sysroot" ]; then continue; fi
-			read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info "$PKG")
+			read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info $PKG "${PKG_DIR}")
 
 			if [ ! "$TERMUX_QUIET_BUILD" = true ]; then
 				echo "Downloading dependency $PKG@$DEP_VERSION if necessary..."
 			fi
 			if ! termux_download_deb $PKG $DEP_ARCH $DEP_VERSION; then
-				if find packages/ -type f -name ${PKG}.subpackage.sh -exec false {} +; then
+				echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
-					echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead"
+				./build-package.sh -a $TERMUX_ARCH -I "${PKG_DIR}"
-					./build-package.sh -a $TERMUX_ARCH -I "$PKG"
+				continue
-					continue
-				else
-					# subpackage, so we need to build parent package
-					PARENT=$(dirname $(find packages/ -name "${PKG}.subpackage.sh"))
-					echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building parent $PARENT instead"
-					./build-package.sh -a $TERMUX_ARCH -I $PARENT
-				fi
 			else
 				if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "extracting $PKG..."; fi
 				(
@@ -64,14 +49,17 @@ termux_step_start_build() {
 			fi
 			mkdir -p /data/data/.built-packages
 			echo "$DEP_VERSION" > "/data/data/.built-packages/$PKG"
-		done
+		done<<<$(./scripts/buildorder.py -i "$TERMUX_PKG_BUILDER_DIR")
 	elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then
 		# Build dependencies
-		for PKG in $(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR"); do
+		while read PKG PKG_DIR; do
+			if [ -z $PKG ]; then
+				continue
+			fi
 			echo "Building dependency $PKG if necessary..."
 			# Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one
-			./build-package.sh -a $TERMUX_ARCH -s "$PKG"
+			./build-package.sh -a $TERMUX_ARCH -s "${PKG_DIR}"
-		done
+		done<<<$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR")
 	fi
 
 	TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION

scripts/buildorder.py

@@ -107,13 +107,26 @@ class TermuxSubPackage:
         self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0]
         self.parent = parent
         self.deps = parse_build_file_dependencies(subpackage_file_path)
+        self.dir = parent.dir
+
+        self.needed_by = set()  # Populated outside constructor, reverse of deps.
 
     def __repr__(self):
         return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent)
 
-def read_packages_from_directories(directories):
+    def recursive_dependencies(self, pkgs_map):
+        """All the dependencies of the subpackage, both direct and indirect.
+        Only relevant when building in fast-build mode"""
+        result = []
+        for dependency_name in sorted(self.deps):
+            dependency_package = pkgs_map[dependency_name]
+            result += dependency_package.recursive_dependencies(pkgs_map)
+            result += [dependency_package]
+        return unique_everseen(result)
+
+def read_packages_from_directories(directories, fast_build_mode):
     """Construct a map from package name to TermuxPackage.
-    For subpackages this maps from the subpackage name to the parent package."""
+    Subpackages are mapped to the parent package if fast_build_mode is false."""
     pkgs_map = {}
     all_packages = []
 
@@ -132,6 +145,8 @@ def read_packages_from_directories(directories):
                 for subpkg in new_package.subpkgs:
                     if subpkg.name in pkgs_map:
                         die('Duplicated package: ' + subpkg.name)
+                    elif fast_build_mode:
+                        pkgs_map[subpkg.name] = subpkg
                     else:
                         pkgs_map[subpkg.name] = new_package
                     all_packages.append(subpkg)
@@ -141,7 +156,7 @@ def read_packages_from_directories(directories):
             if dependency_name not in pkgs_map:
                 die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name))
             dep_pkg = pkgs_map[dependency_name]
-            if not isinstance(pkg, TermuxSubPackage):
+            if fast_build_mode or not isinstance(pkg, TermuxSubPackage):
                 dep_pkg.needed_by.add(pkg)
     return pkgs_map
 
@@ -198,7 +213,7 @@ def generate_full_buildorder(pkgs_map):
 
     return build_order
 
-def generate_target_buildorder(target_path, pkgs_map):
+def generate_target_buildorder(target_path, pkgs_map, fast_build_mode):
     "Generate a build order for building the dependencies of the specified package."
     if target_path.endswith('/'):
         target_path = target_path[:-1]
@@ -209,28 +224,50 @@ def generate_target_buildorder(target_path, pkgs_map):
 
 def main():
     "Generate the build order either for all packages or a specific one."
-    packages_directories = ['packages']
+    import argparse
-    full_buildorder = len(sys.argv) == 1
+
+    parser = argparse.ArgumentParser(description='Generate order in which to build dependencies for a package. Generates')
+    parser.add_argument('-i', default=False, action='store_true',
+                        help='Generate dependency list for fast-build mode. This includes subpackages in output since these can be downloaded.')
+    parser.add_argument('package', nargs='?',
+                        help='Package to generate dependency list for.')
+    parser.add_argument('package_dirs', nargs='*',
+                        help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.')
+    args = parser.parse_args()
+    fast_build_mode = args.i
+    package = args.package
+    packages_directories = args.package_dirs + ['packages']
+
+    if not package:
+        full_buildorder = True
+    else:
+        full_buildorder = False
+
+    if fast_build_mode and full_buildorder:
+        die('-i mode does not work when building all packages')
+
     if not full_buildorder:
         packages_real_path = os.path.realpath('packages')
-        for path in sys.argv[1:]:
+        for path in packages_directories:
             if not os.path.isdir(path):
                 die('Not a directory: ' + path)
-            if path.endswith('/'):
-                path = path[:-1]
-            parent_path = os.path.dirname(path)
-            if packages_real_path != os.path.realpath(parent_path):
-                packages_directories.append(parent_path)
 
-    pkgs_map = read_packages_from_directories(packages_directories)
+    if package:
+        if package[-1] == "/":
+            package = package[:-1]
+        if not os.path.isdir(package):
+            die('Not a directory: ' + package)
+        if not os.path.relpath(os.path.dirname(package), '.') in packages_directories:
+            packages_directories.insert(0, os.path.dirname(package))
+    pkgs_map = read_packages_from_directories(packages_directories, fast_build_mode)
 
     if full_buildorder:
         build_order = generate_full_buildorder(pkgs_map)
     else:
-        build_order = generate_target_buildorder(sys.argv[1], pkgs_map)
+        build_order = generate_target_buildorder(package, pkgs_map, fast_build_mode)
 
     for pkg in build_order:
-        print(pkg.dir)
+        print("%-30s %s" % (pkg.name, pkg.dir))
 
 if __name__ == '__main__':
     main()