From fb8ce2e2194aa71986a5bf72599f06b86bc74081 Mon Sep 17 00:00:00 2001 From: Ivan Max Date: Tue, 2 Apr 2024 20:36:49 +0000 Subject: [PATCH] scripts/generate-bootstraps.sh: improve pacman-based bootstrap creations (#19619) --- scripts/boot-set-up-pkgs.sh | 47 ++++++++++++++++ scripts/generate-bootstraps.sh | 98 ++++++++++++++++------------------ 2 files changed, 94 insertions(+), 51 deletions(-) create mode 100644 scripts/boot-set-up-pkgs.sh diff --git a/scripts/boot-set-up-pkgs.sh b/scripts/boot-set-up-pkgs.sh new file mode 100644 index 0000000000..4ede01e2d2 --- /dev/null +++ b/scripts/boot-set-up-pkgs.sh @@ -0,0 +1,47 @@ +#!@TERMUX_PREFIX@/bin/bash + +# --- boot-set-up-pkgs.sh --- +# This script runs the packages 'postinst' function. +# They are not run during the creation of bootstrap based +# on pacman as this is not possible. They must be running +# on Termux devices for the packages to function completely properly. +# +# This script is one-time only. After launching, it will +# be automatically deleted to avoid running it again. + +( +export TERMUX_PREFIX="@TERMUX_PREFIX@" +export TERMUX_PACKAGE_MANAGER="@TERMUX_PACKAGE_MANAGER@" + +_boot_message() { + echo "[*] ${1}" +} + +_boot_message "Automatic one-time setup of packages has been launched, setup is in progress..." + +case "${TERMUX_PACKAGE_MANAGER}" in + "pacman") + for i in ${TERMUX_PREFIX}/var/lib/pacman/local/*/install; do + ( + source $i + if type post_install &> /dev/null; then + DIR="${i::-8}" + _boot_message "Setting up the ${DIR##*/} package..." + post_install &> /dev/null || true + fi + ) + done;; + "apt") + for i in ${TERMUX_PREFIX}/var/lib/dpkg/info/*.postinst; do + DIR="${i##*/}" + _boot_message "Setting up the ${DIR::-9} package..." + bash $i + done;; + *) + _boot_message "unknown package manager '${TERMUX_PACKAGE_MANAGER}', because of this it is not possible to configure packages" +esac + +_boot_message "Setting up packages is complete, pleasant use Termux :)" + +rm ${BASH_SOURCE[0]} +) diff --git a/scripts/generate-bootstraps.sh b/scripts/generate-bootstraps.sh index 1d85d52434..b73f1e71fe 100755 --- a/scripts/generate-bootstraps.sh +++ b/scripts/generate-bootstraps.sh @@ -24,7 +24,7 @@ TERMUX_PACKAGE_MANAGERS=("apt" "pacman") # The repository base urls mapping for package managers. declare -A REPO_BASE_URLS=( ["apt"]="https://packages-cf.termux.dev/apt/termux-main" - ["pacman"]="https://s3.amazonaws.com/termux-pacman.us/main" + ["pacman"]="https://service.termux-pacman.dev/main" ) # The package manager that will be installed in bootstrap. @@ -41,7 +41,7 @@ declare -a ADDITIONAL_PACKAGES # Check for some important utilities that may not be available for # some reason. -for cmd in ar awk curl grep gzip find sed tar xargs xz zip; do +for cmd in ar awk curl grep gzip find sed tar xargs xz zip jq; do if [ -z "$(command -v $cmd)" ]; then echo "[!] Utility '$cmd' is not available in PATH." exit 1 @@ -93,22 +93,21 @@ read_package_list_deb() { done } -read_package_list_pac() { - if [ ! -e "${BOOTSTRAP_TMPDIR}/main_${1}.db" ]; then - echo "[*] Downloading package list for architecture '${1}'..." +download_db_packages_pac() { + if [ ! -e "${PATH_DB_PACKAGES}" ]; then + echo "[*] Downloading package list for architecture '${package_arch}'..." curl --fail --location \ - --output "${BOOTSTRAP_TMPDIR}/main_${1}.db" \ - "${REPO_BASE_URL}/${1}/main.db" + --output "${PATH_DB_PACKAGES}" \ + "${REPO_BASE_URL}/${package_arch}/main.json" fi +} - echo "[*] Reading package list for '${1}'..." - mkdir -p "${BOOTSTRAP_TMPDIR}/packages" - tar -xf "${BOOTSTRAP_TMPDIR}/main_${1}.db" -C "${BOOTSTRAP_TMPDIR}/packages" - local packages_name=($(grep -h -A 1 "%NAME%" "${BOOTSTRAP_TMPDIR}"/packages/*/desc | sed 's/%NAME%//g; s/--//g')) - local packages_filename=($(grep -h -A 1 "%FILENAME%" "${BOOTSTRAP_TMPDIR}"/packages/*/desc | sed 's/%FILENAME%//g; s/--//g')) - for i in $(seq 0 $((${#packages_name[@]}-1))); do - PACKAGE_METADATA["${packages_name[$i]}"]="${1}/${packages_filename[$i]}" - done +read_db_packages_pac() { + jq -r '."'${package_name}'"."'${1}'" | if type == "array" then .[] else . end' "${PATH_DB_PACKAGES}" +} + +print_desc_package_pac() { + echo -e "%${1}%\n${2}\n" } # Download specified package, its depenencies and then extract *.deb or *.pkg.tar.xz files to @@ -201,17 +200,9 @@ pull_package() { ) fi else - local package_desc=$(grep -l $(echo ${PACKAGE_METADATA[${package_name}]} | awk -F "/" '{printf $2}') "${BOOTSTRAP_TMPDIR}"/packages/${package_name}*/desc | head -1) - local package_dependencies - local i=0 - package_dependencies=$( - while [[ -n $(grep -A ${i} %DEPENDS% ${package_desc} | tail -1) ]]; do - i=$((${i}+1)) - echo $(grep -A ${i} %DEPENDS% ${package_desc} | tail -1 | sed 's// /g; s/=/ /g' | awk '{printf $1}') - done - ) + local package_dependencies=$(read_db_packages_pac "DEPENDS" | sed 's/<.*$//g; s/>.*$//g; s/=.*$//g') - if [ -n "$package_dependencies" ]; then + if [ "$package_dependencies" != "null" ]; then local dep for dep in $package_dependencies; do if [ ! -e "${BOOTSTRAP_PKGDIR}/${dep}" ]; then @@ -223,36 +214,31 @@ pull_package() { if [ ! -e "$package_tmpdir/package.pkg.tar.xz" ]; then echo "[*] Downloading '$package_name'..." - curl --fail --location --output "$package_tmpdir/package.pkg.tar.xz" "${REPO_BASE_URL}/${PACKAGE_METADATA[${package_name}]}" + local package_filename=$(read_db_packages_pac "FILENAME") + curl --fail --location --output "$package_tmpdir/package.pkg.tar.xz" "${REPO_BASE_URL}/${package_arch}/${package_filename}" echo "[*] Extracting '$package_name'..." (cd "$package_tmpdir" - tar xJf package.pkg.tar.xz - if [ -d ./data ]; then - cp -r ./data "$BOOTSTRAP_ROOTFS" - fi - local metadata_package_sp=(${PACKAGE_METADATA[${package_name}]//// }) - if [ $(echo "${metadata_package_sp[1]}" | grep "any.") ]; then - local local_dir_sp=(${metadata_package_sp[1]//-any/ }) - else - local local_dir_sp=(${metadata_package_sp[1]//-${metadata_package_sp[0]}/ }) - fi - if [ $(echo "${package_name}" | grep "+") ]; then - local package_name_in_host="${package_name//+/0}" - local_dir_sp[0]="${local_dir_sp[0]//${package_name_in_host}/${package_name}}" - fi - mkdir -p "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${local_dir_sp[0]}" - cp -r .MTREE "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${local_dir_sp[0]}/mtree" - cp -r "${package_desc}" "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${local_dir_sp[0]}/desc" - if [ -f .INSTALL ]; then - cp -r .INSTALL "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${local_dir_sp[0]}/install" - fi + local package_desc="${package_name}-$(read_db_packages_pac VERSION)" + mkdir -p "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${package_desc}" { echo "%FILES%" - if [ -d ./data ]; then - find data - fi - } >> "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${local_dir_sp[0]}/files" + tar xvf package.pkg.tar.xz -C "$BOOTSTRAP_ROOTFS" .INSTALL .MTREE data 2> /dev/null | grep '^data/' || true + } >> "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${package_desc}/files" + mv "${BOOTSTRAP_ROOTFS}/.MTREE" "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${package_desc}/mtree" + if [ -f "${BOOTSTRAP_ROOTFS}/.INSTALL" ]; then + mv "${BOOTSTRAP_ROOTFS}/.INSTALL" "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${package_desc}/install" + fi + { + local keys_desc="VERSION BASE DESC URL ARCH BUILDDATE PACKAGER ISIZE GROUPS LICENSE REPLACES DEPENDS OPTDEPENDS CONFLICTS PROVIDES" + for i in "NAME ${package_name}" \ + "INSTALLDATE $(date +%s)" \ + "VALIDATION $(test $(read_db_packages_pac PGPSIG) != 'null' && echo 'pgp' || echo 'sha256')"; do + print_desc_package_pac ${i} + done + jq -r -j '."'${package_name}'" | to_entries | .[] | select(.key | contains('$(sed 's/^/"/; s/ /","/g; s/$/"/' <<< ${keys_desc})')) | "%",(if .key == "ISIZE" then "SIZE" else .key end),"%\n",.value,"\n\n" | if type == "array" then (.| join("\n")) else . end' \ + "${PATH_DB_PACKAGES}" + } >> "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/var/lib/pacman/local/${package_desc}/desc" ) fi fi @@ -393,6 +379,7 @@ if [ -z "$REPO_BASE_URL" ]; then fi for package_arch in "${TERMUX_ARCHITECTURES[@]}"; do + PATH_DB_PACKAGES="$BOOTSTRAP_TMPDIR/main_${package_arch}.json" BOOTSTRAP_ROOTFS="$BOOTSTRAP_TMPDIR/rootfs-${package_arch}" BOOTSTRAP_PKGDIR="$BOOTSTRAP_TMPDIR/packages-${package_arch}" @@ -423,7 +410,7 @@ for package_arch in "${TERMUX_ARCHITECTURES[@]}"; do if [ ${TERMUX_PACKAGE_MANAGER} = "apt" ]; then read_package_list_deb "$package_arch" else - read_package_list_pac "$package_arch" + download_db_packages_pac fi # Package manager. @@ -477,6 +464,15 @@ for package_arch in "${TERMUX_ARCHITECTURES[@]}"; do done unset add_pkg + # Adding pacman package configuration + if [ ${TERMUX_PACKAGE_MANAGER} = "pacman" ] || [ ${TERMUX_PACKAGE_MANAGER} = "apt" ]; then + mkdir -p "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/etc/profile.d" + sed -e "s|@TERMUX_PREFIX@|${TERMUX_PREFIX}|g" \ + -e "s|@TERMUX_PACKAGE_MANAGER@|${TERMUX_PACKAGE_MANAGER}|g" \ + $(dirname "$(realpath "$0")")/boot-set-up-pkgs.sh \ + > "${BOOTSTRAP_ROOTFS}/${TERMUX_PREFIX}/etc/profile.d/boot-set-up-pkgs.sh" + fi + # Create bootstrap archive. create_bootstrap_archive "$package_arch" done