diff --git a/build-all.sh b/build-all.sh index e65318c176..4137e5b0e9 100755 --- a/build-all.sh +++ b/build-all.sh @@ -55,28 +55,27 @@ fi exec > >(tee -a $BUILDALL_DIR/ALL.out) exec 2> >(tee -a $BUILDALL_DIR/ALL.err >&2) -trap "echo ERROR: See $BUILDALL_DIR/\${package}.err" ERR +trap "echo ERROR: See $BUILDALL_DIR/\${PKG}.err" ERR -for package_path in $(cat $BUILDORDER_FILE); do - package=$(basename $package_path) +while read PKG PKG_DIR; do # Check build status (grepping is a bit crude, but it works) - if [ -e $BUILDSTATUS_FILE ] && grep "^$package\$" $BUILDSTATUS_FILE >/dev/null; then - echo "Skipping $package" + if [ -e $BUILDSTATUS_FILE ] && grep "^$PKG\$" $BUILDSTATUS_FILE >/dev/null; then + echo "Skipping $PKG" continue fi - echo -n "Building $package... " + echo -n "Building $PKG... " BUILD_START=$(date "+%s") bash -x $BUILDSCRIPT -a $TERMUX_ARCH $TERMUX_DEBUG \ - ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $package \ - > $BUILDALL_DIR/${package}.out 2> $BUILDALL_DIR/${package}.err + ${TERMUX_DEBDIR+-o $TERMUX_DEBDIR} $TERMUX_INSTALL_DEPS $PKG_DIR \ + > $BUILDALL_DIR/${PKG}.out 2> $BUILDALL_DIR/${PKG}.err BUILD_END=$(date "+%s") BUILD_SECONDS=$(( $BUILD_END - $BUILD_START )) echo "done in $BUILD_SECONDS" # Update build status - echo "$package" >> $BUILDSTATUS_FILE -done + echo "$PKG" >> $BUILDSTATUS_FILE +done<${BUILDORDER_FILE} # Update build status rm -f $BUILDSTATUS_FILE diff --git a/scripts/build/termux_extract_dep_info.sh b/scripts/build/termux_extract_dep_info.sh index 91b99c3012..49d3be7d20 100755 --- a/scripts/build/termux_extract_dep_info.sh +++ b/scripts/build/termux_extract_dep_info.sh @@ -1,13 +1,14 @@ termux_extract_dep_info() { - package=$1 - if [ ! -d packages/$package ] && [ -f packages/*/${package}.subpackage.sh ]; then + PKG=$1 + PKG_DIR=$2 + if [ "$PKG" != "$(basename ${PKG_DIR})" ]; then # We are dealing with a subpackage TERMUX_ARCH=$( - # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to mother package's value and override if needed + # set TERMUX_SUBPKG_PLATFORM_INDEPENDENT to parent package's value and override if needed TERMUX_PKG_PLATFORM_INDEPENDENT="" - source $(dirname $(find packages/ -name "$package.subpackage.sh"))/build.sh + source ${PKG_DIR}/build.sh TERMUX_SUBPKG_PLATFORM_INDEPENDENT=$TERMUX_PKG_PLATFORM_INDEPENDENT - source $(find packages/ -name "$package.subpackage.sh") + source ${PKG_DIR}/${PKG}.subpackage.sh if [ "$TERMUX_SUBPKG_PLATFORM_INDEPENDENT" = yes ]; then echo all else @@ -15,10 +16,9 @@ termux_extract_dep_info() { fi ) - package=$(basename $(dirname $(find packages/ -name "$package.subpackage.sh"))) - elif [ "${package/-dev/}-dev" == "${package}" ]; then + elif [ "${PKG/-dev/}-dev" == "${PKG}" ]; then # dev package - package=${package/-dev/} + PKG=${PKG/-dev/} fi ( # Reset TERMUX_PKG_PLATFORM_INDEPENDENT and TERMUX_PKG_REVISION since these aren't @@ -26,7 +26,7 @@ termux_extract_dep_info() { # deps that should have the default values TERMUX_PKG_PLATFORM_INDEPENDENT="" TERMUX_PKG_REVISION="0" - source packages/$package/build.sh + source ${PKG_DIR}/build.sh if [ "$TERMUX_PKG_PLATFORM_INDEPENDENT" = yes ]; then TERMUX_ARCH=all; fi if [ "$TERMUX_PKG_REVISION" != "0" ] || [ "$TERMUX_PKG_VERSION" != "${TERMUX_PKG_VERSION/-/}" ]; then TERMUX_PKG_VERSION+="-$TERMUX_PKG_REVISION" diff --git a/scripts/build/termux_step_start_build.sh b/scripts/build/termux_step_start_build.sh index 9c8e59e21f..56c5589490 100644 --- a/scripts/build/termux_step_start_build.sh +++ b/scripts/build/termux_step_start_build.sh @@ -14,36 +14,21 @@ termux_step_start_build() { if [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = true ]; then # Download dependencies - local PKG DEP_ARCH DEP_VERSION DEB_FILE _PKG_DEPENDS _PKG_BUILD_DEPENDS _SUBPKG_DEPENDS="" - # remove (>= 1.0) and similar version tags: - _PKG_DEPENDS=$(echo ${TERMUX_PKG_DEPENDS// /} | sed "s/[(][^)]*[)]//g") - _PKG_BUILD_DEPENDS=${TERMUX_PKG_BUILD_DEPENDS// /} - # Also download subpackages dependencies (except the parent package): - for SUBPKG in packages/$TERMUX_PKG_NAME/*.subpackage.sh; do - test -e $SUBPKG || continue - _SUBPKG_DEPENDS+=" $(. $SUBPKG; echo $TERMUX_SUBPKG_DEPENDS | sed s%$TERMUX_PKG_NAME%%g)" - done - for PKG in $(echo ${_PKG_DEPENDS//,/ } ${_SUBPKG_DEPENDS//,/ } ${_PKG_BUILD_DEPENDS//,/ } | tr ' ' '\n' | sort -u); do - # handle "or" in dependencies (use first one): - if [ ! "$PKG" = "${PKG/|/}" ]; then PKG=$(echo "$PKG" | sed "s%|.*%%"); fi + while read PKG PKG_DIR; do + if [ -z $PKG ]; then + continue + fi # llvm doesn't build if ndk-sysroot is installed: if [ "$PKG" = "ndk-sysroot" ]; then continue; fi - read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info "$PKG") + read DEP_ARCH DEP_VERSION <<< $(termux_extract_dep_info $PKG "${PKG_DIR}") if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "Downloading dependency $PKG@$DEP_VERSION if necessary..." fi if ! termux_download_deb $PKG $DEP_ARCH $DEP_VERSION; then - if find packages/ -type f -name ${PKG}.subpackage.sh -exec false {} +; then - echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead" - ./build-package.sh -a $TERMUX_ARCH -I "$PKG" - continue - else - # subpackage, so we need to build parent package - PARENT=$(dirname $(find packages/ -name "${PKG}.subpackage.sh")) - echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building parent $PARENT instead" - ./build-package.sh -a $TERMUX_ARCH -I $PARENT - fi + echo "Download of $PKG@$DEP_VERSION from $TERMUX_REPO_URL failed, building instead" + ./build-package.sh -a $TERMUX_ARCH -I "${PKG_DIR}" + continue else if [ ! "$TERMUX_QUIET_BUILD" = true ]; then echo "extracting $PKG..."; fi ( @@ -64,14 +49,17 @@ termux_step_start_build() { fi mkdir -p /data/data/.built-packages echo "$DEP_VERSION" > "/data/data/.built-packages/$PKG" - done + done<<<$(./scripts/buildorder.py -i "$TERMUX_PKG_BUILDER_DIR") elif [ "$TERMUX_SKIP_DEPCHECK" = false ] && [ "$TERMUX_INSTALL_DEPS" = false ]; then # Build dependencies - for PKG in $(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR"); do + while read PKG PKG_DIR; do + if [ -z $PKG ]; then + continue + fi echo "Building dependency $PKG if necessary..." # Built dependencies are put in the default TERMUX_DEBDIR instead of the specified one - ./build-package.sh -a $TERMUX_ARCH -s "$PKG" - done + ./build-package.sh -a $TERMUX_ARCH -s "${PKG_DIR}" + done<<<$(./scripts/buildorder.py "$TERMUX_PKG_BUILDER_DIR") fi TERMUX_PKG_FULLVERSION=$TERMUX_PKG_VERSION diff --git a/scripts/buildorder.py b/scripts/buildorder.py index 52d6e41e32..770c588cf1 100755 --- a/scripts/buildorder.py +++ b/scripts/buildorder.py @@ -107,13 +107,26 @@ class TermuxSubPackage: self.name = os.path.basename(subpackage_file_path).split('.subpackage.sh')[0] self.parent = parent self.deps = parse_build_file_dependencies(subpackage_file_path) + self.dir = parent.dir + + self.needed_by = set() # Populated outside constructor, reverse of deps. def __repr__(self): return "<{} '{}' parent='{}'>".format(self.__class__.__name__, self.name, self.parent) -def read_packages_from_directories(directories): + def recursive_dependencies(self, pkgs_map): + """All the dependencies of the subpackage, both direct and indirect. + Only relevant when building in fast-build mode""" + result = [] + for dependency_name in sorted(self.deps): + dependency_package = pkgs_map[dependency_name] + result += dependency_package.recursive_dependencies(pkgs_map) + result += [dependency_package] + return unique_everseen(result) + +def read_packages_from_directories(directories, fast_build_mode): """Construct a map from package name to TermuxPackage. - For subpackages this maps from the subpackage name to the parent package.""" + Subpackages are mapped to the parent package if fast_build_mode is false.""" pkgs_map = {} all_packages = [] @@ -132,6 +145,8 @@ def read_packages_from_directories(directories): for subpkg in new_package.subpkgs: if subpkg.name in pkgs_map: die('Duplicated package: ' + subpkg.name) + elif fast_build_mode: + pkgs_map[subpkg.name] = subpkg else: pkgs_map[subpkg.name] = new_package all_packages.append(subpkg) @@ -141,7 +156,7 @@ def read_packages_from_directories(directories): if dependency_name not in pkgs_map: die('Package %s depends on non-existing package "%s"' % (pkg.name, dependency_name)) dep_pkg = pkgs_map[dependency_name] - if not isinstance(pkg, TermuxSubPackage): + if fast_build_mode or not isinstance(pkg, TermuxSubPackage): dep_pkg.needed_by.add(pkg) return pkgs_map @@ -198,7 +213,7 @@ def generate_full_buildorder(pkgs_map): return build_order -def generate_target_buildorder(target_path, pkgs_map): +def generate_target_buildorder(target_path, pkgs_map, fast_build_mode): "Generate a build order for building the dependencies of the specified package." if target_path.endswith('/'): target_path = target_path[:-1] @@ -209,28 +224,50 @@ def generate_target_buildorder(target_path, pkgs_map): def main(): "Generate the build order either for all packages or a specific one." - packages_directories = ['packages'] - full_buildorder = len(sys.argv) == 1 + import argparse + + parser = argparse.ArgumentParser(description='Generate order in which to build dependencies for a package. Generates') + parser.add_argument('-i', default=False, action='store_true', + help='Generate dependency list for fast-build mode. This includes subpackages in output since these can be downloaded.') + parser.add_argument('package', nargs='?', + help='Package to generate dependency list for.') + parser.add_argument('package_dirs', nargs='*', + help='Directories with packages. Can for example point to "../x11-packages/packages/". "packages/" is appended automatically.') + args = parser.parse_args() + fast_build_mode = args.i + package = args.package + packages_directories = args.package_dirs + ['packages'] + + if not package: + full_buildorder = True + else: + full_buildorder = False + + if fast_build_mode and full_buildorder: + die('-i mode does not work when building all packages') + if not full_buildorder: packages_real_path = os.path.realpath('packages') - for path in sys.argv[1:]: + for path in packages_directories: if not os.path.isdir(path): die('Not a directory: ' + path) - if path.endswith('/'): - path = path[:-1] - parent_path = os.path.dirname(path) - if packages_real_path != os.path.realpath(parent_path): - packages_directories.append(parent_path) - pkgs_map = read_packages_from_directories(packages_directories) + if package: + if package[-1] == "/": + package = package[:-1] + if not os.path.isdir(package): + die('Not a directory: ' + package) + if not os.path.relpath(os.path.dirname(package), '.') in packages_directories: + packages_directories.insert(0, os.path.dirname(package)) + pkgs_map = read_packages_from_directories(packages_directories, fast_build_mode) if full_buildorder: build_order = generate_full_buildorder(pkgs_map) else: - build_order = generate_target_buildorder(sys.argv[1], pkgs_map) + build_order = generate_target_buildorder(package, pkgs_map, fast_build_mode) for pkg in build_order: - print(pkg.dir) + print("%-30s %s" % (pkg.name, pkg.dir)) if __name__ == '__main__': main()