Implemented a task to test feeds before adding.

This commit is contained in:
Emilis Dambauskas 2020-11-20 03:26:34 +02:00
parent 31fdfe805b
commit e06073443b
2 changed files with 15 additions and 3 deletions

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
downloaded.gmi
sorted.gmi
index.gmi
backup-feeds.txt
test-feed.txt

View File

@ -1,4 +1,4 @@
NUM_LINKS_TO_SHOW= 50
NUM_LINKS_TO_SHOW= 100
PARALLEL_DOWNLOADS= 8
@ -15,6 +15,15 @@ get-feeds:
rm downloaded.gmi
@make sorted.gmi
.PHONY: test-feed
test-feed:
@echo 'Current feeds.txt:'
cat feeds.txt
cp -i feeds.txt backup-feeds.txt
cp -i test-feed.txt feeds.txt
@make default
cp backup-feeds.txt feeds.txt
index.gmi: \
header.gmi \
@ -52,7 +61,8 @@ downloaded.gmi: feeds.txt
BASE_URL=$$( echo $$URL | grep -oE ".+/" ); \
echo "BASE_URL: $$BASE_URL"; \
echo "DOWNLOADING: $$DOMAIN $$URL into $$TMP_FILE"; \
./gcat "$$URL" > $$TMP_FILE; \
./gcat "$$URL" \
| tee $$TMP_FILE; \
TITLE=$$(grep -s -m 1 -E "^# " $$TMP_FILE | cut -c 3- ); \
echo "TITLE: $$TITLE"; \
echo "full urls:"; \
@ -69,7 +79,7 @@ downloaded.gmi: feeds.txt
echo "no-slash urls:"; \
grep -hsE "^=>\s*\S+\s+[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}(:[0-9]{2})?(Z|\+[0-9]{1,2}:[0-9]{2}))?\s+\S.*$$" $$TMP_FILE \
| grep -hsvE "^=>\s*gemini://" \
| grep -hsE "^=>\s*[^/]" \
| grep -hsvE "^=>\s*/" \
| sed -E -e "s#^=>[ ]*#=> $${BASE_URL}#g" \
| awk "{ if(\"$$TITLE\"){ \$$4= \"$$TITLE - \" \$$4; } print }" \
| tee -a "$@"; \