diff --git a/.gitignore b/.gitignore index 4b2b94e..393b35a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ downloaded.gmi sorted.gmi index.gmi +backup-feeds.txt +test-feed.txt diff --git a/Makefile b/Makefile index ea27a75..c998b98 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -NUM_LINKS_TO_SHOW= 50 +NUM_LINKS_TO_SHOW= 100 PARALLEL_DOWNLOADS= 8 @@ -15,6 +15,15 @@ get-feeds: rm downloaded.gmi @make sorted.gmi +.PHONY: test-feed +test-feed: + @echo 'Current feeds.txt:' + cat feeds.txt + cp -i feeds.txt backup-feeds.txt + cp -i test-feed.txt feeds.txt + @make default + cp backup-feeds.txt feeds.txt + index.gmi: \ header.gmi \ @@ -52,7 +61,8 @@ downloaded.gmi: feeds.txt BASE_URL=$$( echo $$URL | grep -oE ".+/" ); \ echo "BASE_URL: $$BASE_URL"; \ echo "DOWNLOADING: $$DOMAIN $$URL into $$TMP_FILE"; \ - ./gcat "$$URL" > $$TMP_FILE; \ + ./gcat "$$URL" \ + | tee $$TMP_FILE; \ TITLE=$$(grep -s -m 1 -E "^# " $$TMP_FILE | cut -c 3- ); \ echo "TITLE: $$TITLE"; \ echo "full urls:"; \ @@ -69,7 +79,7 @@ downloaded.gmi: feeds.txt echo "no-slash urls:"; \ grep -hsE "^=>\s*\S+\s+[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}(:[0-9]{2})?(Z|\+[0-9]{1,2}:[0-9]{2}))?\s+\S.*$$" $$TMP_FILE \ | grep -hsvE "^=>\s*gemini://" \ - | grep -hsE "^=>\s*[^/]" \ + | grep -hsvE "^=>\s*/" \ | sed -E -e "s#^=>[ ]*#=> $${BASE_URL}#g" \ | awk "{ if(\"$$TITLE\"){ \$$4= \"$$TITLE - \" \$$4; } print }" \ | tee -a "$@"; \