diff --git a/scripts/bin/update-packages b/scripts/bin/update-packages index a5ad2b5c85..77ea8621d5 100755 --- a/scripts/bin/update-packages +++ b/scripts/bin/update-packages @@ -75,6 +75,23 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$ # shellcheck source=scripts/updates/termux_pkg_auto_update.sh . "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh +# Converts milliseconds to human-readable format. +# Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms +ms_to_human_readable() { + echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //' +} + +# Runs a command without displaying its output in the case if trace is disabled and displays output in the case if it is enabled. +# Needed only for debugging +quiet() { + if [[ "$-" =~ x ]]; then + "$@" + else + &>/dev/null "$@" + fi + return $? +} + _update() { export TERMUX_PKG_NAME TERMUX_PKG_NAME="$(basename "$1")" @@ -103,11 +120,105 @@ _update() { termux_pkg_auto_update } +declare -A _LATEST_TAGS=() declare -A _FAILED_UPDATES=() declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped. +# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system. +# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time. +# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly. +# We avoid packages with overwritten termux_pkg_auto_update to prevent unexpected modifications to the package`s build.sh. +_fetch_and_cache_tags() { + if [ "$(uname -o)" = "Android" ] || [ -e "/system/bin/app_process" ]; then + if ! command -v ninja &> /dev/null; then + echo "WARNING: Skipping fetching and caching tags. Package 'ninja' is not installed." + return 0 + fi + fi + + if ! command -v ninja &> /dev/null; then + echo "INFO: Fetching ninja build system" + . "${TERMUX_SCRIPTDIR}"/scripts/build/termux_download.sh + . "${TERMUX_SCRIPTDIR}"/scripts/build/setup/termux_setup_ninja.sh + TERMUX_ON_DEVICE_BUILD=false TERMUX_COMMON_CACHEDIR=/tmp TERMUX_PKG_TMPDIR=/tmp termux_setup_ninja + fi + + echo "INFO: Fetching and caching tags" + + # First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata. + quiet termux_repology_api_get_latest_version ' ' + + local __PACKAGES=() + for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do + for pkg_dir in "${repo_dir}"/*; do + ! quiet _should_update "${pkg_dir}" && continue # Skip if not needed. + grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update + __PACKAGES+=("${pkg_dir}") + done + done + + __main__() { + cd ${TERMUX_SCRIPTDIR} + export TERMUX_PKG_NAME="${1##*/}" TERMUX_PKG_BUILDER_DIR=${1} + set +eu; + for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do source ${i}; done + set -eu + termux_pkg_upgrade_version() { + [[ -n "$1" ]] && echo "PKG|$TERMUX_PKG_NAME|${1#*:}" + exit 0 + } + termux_repology_auto_update() { + echo "PKG|$TERMUX_PKG_NAME|$(termux_repology_api_get_latest_version "${TERMUX_PKG_NAME}" | sed "s/^null$/${TERMUX_PKG_VERSION#*:}/")" + exit 0 + } + termux_pkg_auto_update + } + + __generate__() { + echo "rule update" + echo " command = bash -c \"\$\$F\" -- \$pkg_dir ||:" + sed 's/[^ ]\+/\nbuild &: update\n pkg_dir=&/g' <<< "${__PACKAGES[@]}" + echo "build run_all: phony ${__PACKAGES[@]}" + echo "default run_all" + } + + local LATEST_TAGS="$(\ + F="$(declare -p TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE); $(declare -f __main__ | sed 's/__main__ ()//')" \ + env --chdir=/tmp ninja -f /dev/stdin <<< "$(__generate__)" |& grep "^PKG|" + )" + + unset -f __main__ __generate__ + while IFS='|' read -r _ pkg version; do + _LATEST_TAGS["${pkg:-_}"]="$version" + done <<< "$LATEST_TAGS" + quiet declare -p _LATEST_TAGS +} + +_check_updated() { + if [[ -n "${_LATEST_TAGS[${1##*/}]:-}" ]]; then + ( + set +eu + quiet source "${1}/build.sh" + set -eu + export TERMUX_PKG_UPGRADE_VERSION_DRY_RUN=1 + if quiet termux_pkg_upgrade_version "${_LATEST_TAGS[${1##*/}]}"; then + echo "WARNING: Skipping ${1##*/}: already at version ${TERMUX_PKG_VERSION#*:}" + return 0 + fi + return 1 + ) + local _ANSWER=$? + if (( _ANSWER == 0 )) ; then + _ALREADY_SEEN+=("$(basename "${1}")") + fi + return $_ANSWER + fi + return 1 +} + _run_update() { local pkg_dir="$1" + [[ -n "${_LATEST_TAGS[${pkg_dir##*/}]:-}" ]] && export __CACHED_TAG="${_LATEST_TAGS[${pkg_dir##*/}]}" # Run each package update in separate process since we include their environment variables. local output="" { @@ -123,6 +234,7 @@ _run_update() { else _ALREADY_SEEN+=("$(basename "${pkg_dir}")") fi + unset __CACHED_TAG } declare -a _CACHED_ISSUE_TITLES=() @@ -187,16 +299,19 @@ _update_dependencies() { elif [[ "${dep}" == "ERROR" ]]; then termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")" fi - _should_update "${dep_dir}" && _run_update "${dep_dir}" + _should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}" done <<<"$("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")" } echo "INFO: Running update for: $*" if [[ "$1" == "@all" ]]; then + _fetch_and_cache_tags for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do for pkg_dir in "${repo_dir}"/*; do + _unix_millis=$(date +%s%N | cut -b1-13) ! _should_update "${pkg_dir}" && continue # Skip if not needed. + _check_updated "${pkg_dir}" && continue # Skip if already up-to-date. # Update all its dependencies first. _update_dependencies "${pkg_dir}" # NOTE: I am not cheacking whether dependencies were updated successfully or not. @@ -206,6 +321,7 @@ if [[ "$1" == "@all" ]]; then # and not care about anything else in between. If something fails to update, # it will be reported by failure handling code, so no worries. _run_update "${pkg_dir}" + echo "termux - took $(ms_to_human_readable $(( $(date +%s%N | cut -b1-13) - _unix_millis )))" done done else diff --git a/scripts/updates/termux_pkg_auto_update.sh b/scripts/updates/termux_pkg_auto_update.sh index 835da85871..0e26391d98 100644 --- a/scripts/updates/termux_pkg_auto_update.sh +++ b/scripts/updates/termux_pkg_auto_update.sh @@ -1,5 +1,10 @@ # shellcheck shell=bash termux_pkg_auto_update() { + if [[ -n "${__CACHED_TAG:-}" ]]; then + termux_pkg_upgrade_version ${__CACHED_TAG} + return $? + fi + local project_host project_host="$(echo "${TERMUX_PKG_SRCURL}" | cut -d"/" -f3)" diff --git a/scripts/updates/utils/termux_pkg_upgrade_version.sh b/scripts/updates/utils/termux_pkg_upgrade_version.sh index 2d753f8c52..f777b0da27 100755 --- a/scripts/updates/utils/termux_pkg_upgrade_version.sh +++ b/scripts/updates/utils/termux_pkg_upgrade_version.sh @@ -56,6 +56,10 @@ termux_pkg_upgrade_version() { return 0 fi fi + + if [[ -n "${TERMUX_PKG_UPGRADE_VERSION_DRY_RUN:-}" ]]; then + return 1 + fi if [[ "${BUILD_PACKAGES}" == "false" ]]; then echo "INFO: package needs to be updated to ${LATEST_VERSION}."