acdw
/
bollux
1
1
Fork 1
Code Issues 5 Pull Requests Projects Releases Wiki Activity

Fix download method 

Thanks White_Rabbit!
This commit is contained in:
Case Duckworth 2022-06-08 17:29:22 -05:00
parent 21cef8fe43
commit 38bac094bc
1 changed files with 36 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
bollux
View File

@ -1512,18 +1512,44 @@ extract_links() {
# `download' works by reading the end of the pipe from `display', which means
# that sometimes, due to something with the way bash or while or ... something
# ... chunks the data, sometimes binary data gets corrupted. This is an area
# that requires more research.
# that requires more research. UPDATE 2022-06-08: White_Rabbit has figured out
# solution! From their email:
#
# > by the time we're ready to save a non-text/* resource, it's already
# > corrupted beyond repair. One possibile solution, attached, is to request it
# > again, presume the reply is 20 and save the data to the filesystem. What do
# > you think?
#
# I think this is great, thanks!!!
download() {
tn="$(mktemp)"
log x "Downloading: '$BOLLUX_URL' => '$tn'..."
dd status=progress >"$tn"
fn="$BOLLUX_DOWNDIR/${BOLLUX_URL##*/}"
if [[ -f "$fn" ]]; then
log x "Saved '$tn'."
elif mv "$tn" "$fn"; then
log x "Saved '$fn'."
# The binary file has been corrupted by normalize, which strips 0x0d
# bytes. Something also drops NULL bytes. So, we'll discard this data
cat >/dev/null
# Now it's time to re-download the binary file
temp_data="$(mktemp)"
log x "Downloading: '$BOLLUX_URL' => '$temp_data'..."
gemini_request "$BOLLUX_URL" | dd status=progress >"$temp_data"
# Now $temp_data holds both the header and the data
HEADER=$(head -1 "$temp_data")
# To get the header length we use ${#HEADER} syntax, but this gives
# a bad value because it doesn't count the last byte 0x0A.
# We sum 2 because tail wants the first useful byte.
let FIRST_BYTE=$((${#HEADER} + 2))
temp_name="$(mktemp)"
if tail --bytes=+$FIRST_BYTE "$temp_data" >"$temp_name"; then
rm "$temp_data"
else
log error "Error saving '$fn': downloaded to '$tn'."
log error "Error removing the header from '$temp_data'."
fi
final_name="$BOLLUX_DOWNDIR/${BOLLUX_URL##*/}"
if [[ -f "$final_name" ]]; then
log x "Saved '$temp_name'."
elif mv "$temp_name" "$final_name"; then
log x "Saved '$final_name'."
else
log error "Error saving '$final_name': downloaded to '$temp_name'."
fi
}