scripts(update-packages): speed-up package update checks
This commit is contained in:
Twaik Yont
parent
89bfb0f64d
commit
3022b58bcb
|
|
@ -75,6 +75,23 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$
|
|||
# shellcheck source=scripts/updates/termux_pkg_auto_update.sh
|
||||
. "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh
|
||||
|
||||
# Converts milliseconds to human-readable format.
|
||||
# Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms
|
||||
ms_to_human_readable() {
|
||||
echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //'
|
||||
}
|
||||
|
||||
# Runs a command without displaying its output in the case if trace is disabled and displays output in the case if it is enabled.
|
||||
# Needed only for debugging
|
||||
quiet() {
|
||||
if [[ "$-" =~ x ]]; then
|
||||
"$@"
|
||||
else
|
||||
&>/dev/null "$@"
|
||||
fi
|
||||
return $?
|
||||
}
|
||||
|
||||
_update() {
|
||||
export TERMUX_PKG_NAME
|
||||
TERMUX_PKG_NAME="$(basename "$1")"
|
||||
|
|
@ -103,11 +120,105 @@ _update() {
|
|||
termux_pkg_auto_update
|
||||
}
|
||||
|
||||
declare -A _LATEST_TAGS=()
|
||||
declare -A _FAILED_UPDATES=()
|
||||
declare -a _ALREADY_SEEN=() # Array of packages successfully updated or skipped.
|
||||
|
||||
# _fetch_and_cache_tags fetches all possible tags using termux_pkg_auto_update, but using Ninja build system.
|
||||
# The key difference is that we make the process concurrent, allowing us to fetch tags simultaneously rather than one at a time.
|
||||
# Once all tags are cached, the termux_pkg_auto_update function will operate much more quickly.
|
||||
# We avoid packages with overwritten termux_pkg_auto_update to prevent unexpected modifications to the package`s build.sh.
|
||||
_fetch_and_cache_tags() {
|
||||
if [ "$(uname -o)" = "Android" ] || [ -e "/system/bin/app_process" ]; then
|
||||
if ! command -v ninja &> /dev/null; then
|
||||
echo "WARNING: Skipping fetching and caching tags. Package 'ninja' is not installed."
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! command -v ninja &> /dev/null; then
|
||||
echo "INFO: Fetching ninja build system"
|
||||
. "${TERMUX_SCRIPTDIR}"/scripts/build/termux_download.sh
|
||||
. "${TERMUX_SCRIPTDIR}"/scripts/build/setup/termux_setup_ninja.sh
|
||||
TERMUX_ON_DEVICE_BUILD=false TERMUX_COMMON_CACHEDIR=/tmp TERMUX_PKG_TMPDIR=/tmp termux_setup_ninja
|
||||
fi
|
||||
|
||||
echo "INFO: Fetching and caching tags"
|
||||
|
||||
# First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata.
|
||||
quiet termux_repology_api_get_latest_version ' '
|
||||
|
||||
local __PACKAGES=()
|
||||
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
|
||||
for pkg_dir in "${repo_dir}"/*; do
|
||||
! quiet _should_update "${pkg_dir}" && continue # Skip if not needed.
|
||||
grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update
|
||||
__PACKAGES+=("${pkg_dir}")
|
||||
done
|
||||
done
|
||||
|
||||
__main__() {
|
||||
cd ${TERMUX_SCRIPTDIR}
|
||||
export TERMUX_PKG_NAME="${1##*/}" TERMUX_PKG_BUILDER_DIR=${1}
|
||||
set +eu;
|
||||
for i in scripts/updates/{**/,}*.sh "${1}/build.sh"; do source ${i}; done
|
||||
set -eu
|
||||
termux_pkg_upgrade_version() {
|
||||
[[ -n "$1" ]] && echo "PKG|$TERMUX_PKG_NAME|${1#*:}"
|
||||
exit 0
|
||||
}
|
||||
termux_repology_auto_update() {
|
||||
echo "PKG|$TERMUX_PKG_NAME|$(termux_repology_api_get_latest_version "${TERMUX_PKG_NAME}" | sed "s/^null$/${TERMUX_PKG_VERSION#*:}/")"
|
||||
exit 0
|
||||
}
|
||||
termux_pkg_auto_update
|
||||
}
|
||||
|
||||
__generate__() {
|
||||
echo "rule update"
|
||||
echo " command = bash -c \"\$\$F\" -- \$pkg_dir ||:"
|
||||
sed 's/[^ ]\+/\nbuild &: update\n pkg_dir=&/g' <<< "${__PACKAGES[@]}"
|
||||
echo "build run_all: phony ${__PACKAGES[@]}"
|
||||
echo "default run_all"
|
||||
}
|
||||
|
||||
local LATEST_TAGS="$(\
|
||||
F="$(declare -p TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE); $(declare -f __main__ | sed 's/__main__ ()//')" \
|
||||
env --chdir=/tmp ninja -f /dev/stdin <<< "$(__generate__)" |& grep "^PKG|"
|
||||
)"
|
||||
|
||||
unset -f __main__ __generate__
|
||||
while IFS='|' read -r _ pkg version; do
|
||||
_LATEST_TAGS["${pkg:-_}"]="$version"
|
||||
done <<< "$LATEST_TAGS"
|
||||
quiet declare -p _LATEST_TAGS
|
||||
}
|
||||
|
||||
_check_updated() {
|
||||
if [[ -n "${_LATEST_TAGS[${1##*/}]:-}" ]]; then
|
||||
(
|
||||
set +eu
|
||||
quiet source "${1}/build.sh"
|
||||
set -eu
|
||||
export TERMUX_PKG_UPGRADE_VERSION_DRY_RUN=1
|
||||
if quiet termux_pkg_upgrade_version "${_LATEST_TAGS[${1##*/}]}"; then
|
||||
echo "WARNING: Skipping ${1##*/}: already at version ${TERMUX_PKG_VERSION#*:}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
)
|
||||
local _ANSWER=$?
|
||||
if (( _ANSWER == 0 )) ; then
|
||||
_ALREADY_SEEN+=("$(basename "${1}")")
|
||||
fi
|
||||
return $_ANSWER
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
_run_update() {
|
||||
local pkg_dir="$1"
|
||||
[[ -n "${_LATEST_TAGS[${pkg_dir##*/}]:-}" ]] && export __CACHED_TAG="${_LATEST_TAGS[${pkg_dir##*/}]}"
|
||||
# Run each package update in separate process since we include their environment variables.
|
||||
local output=""
|
||||
{
|
||||
|
|
@ -123,6 +234,7 @@ _run_update() {
|
|||
else
|
||||
_ALREADY_SEEN+=("$(basename "${pkg_dir}")")
|
||||
fi
|
||||
unset __CACHED_TAG
|
||||
}
|
||||
|
||||
declare -a _CACHED_ISSUE_TITLES=()
|
||||
|
|
@ -187,16 +299,19 @@ _update_dependencies() {
|
|||
elif [[ "${dep}" == "ERROR" ]]; then
|
||||
termux_error_exit "ERROR: Obtaining update order failed for $(basename "${pkg_dir}")"
|
||||
fi
|
||||
_should_update "${dep_dir}" && _run_update "${dep_dir}"
|
||||
_should_update "${dep_dir}" && ! _check_updated "${dep_dir}" && _run_update "${dep_dir}"
|
||||
done <<<"$("${TERMUX_SCRIPTDIR}"/scripts/buildorder.py "${pkg_dir}" $TERMUX_PACKAGES_DIRECTORIES || echo "ERROR")"
|
||||
}
|
||||
|
||||
echo "INFO: Running update for: $*"
|
||||
|
||||
if [[ "$1" == "@all" ]]; then
|
||||
_fetch_and_cache_tags
|
||||
for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
|
||||
for pkg_dir in "${repo_dir}"/*; do
|
||||
_unix_millis=$(date +%s%N | cut -b1-13)
|
||||
! _should_update "${pkg_dir}" && continue # Skip if not needed.
|
||||
_check_updated "${pkg_dir}" && continue # Skip if already up-to-date.
|
||||
# Update all its dependencies first.
|
||||
_update_dependencies "${pkg_dir}"
|
||||
# NOTE: I am not cheacking whether dependencies were updated successfully or not.
|
||||
|
|
@ -206,6 +321,7 @@ if [[ "$1" == "@all" ]]; then
|
|||
# and not care about anything else in between. If something fails to update,
|
||||
# it will be reported by failure handling code, so no worries.
|
||||
_run_update "${pkg_dir}"
|
||||
echo "termux - took $(ms_to_human_readable $(( $(date +%s%N | cut -b1-13) - _unix_millis )))"
|
||||
done
|
||||
done
|
||||
else
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
# shellcheck shell=bash
|
||||
termux_pkg_auto_update() {
|
||||
if [[ -n "${__CACHED_TAG:-}" ]]; then
|
||||
termux_pkg_upgrade_version ${__CACHED_TAG}
|
||||
return $?
|
||||
fi
|
||||
|
||||
local project_host
|
||||
project_host="$(echo "${TERMUX_PKG_SRCURL}" | cut -d"/" -f3)"
|
||||
|
||||
|
|
|
|||
|
|
@ -56,6 +56,10 @@ termux_pkg_upgrade_version() {
|
|||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "${TERMUX_PKG_UPGRADE_VERSION_DRY_RUN:-}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${BUILD_PACKAGES}" == "false" ]]; then
|
||||
echo "INFO: package needs to be updated to ${LATEST_VERSION}."
|
||||
|
|
|
|||
Loading…
Reference in New Issue