more aggressive url filtering

2020-06-25 13:42:07 +00:00 · 2020-06-25 13:42:07 +00:00 · 56e3aab304
parent a6c8f78ec7
commit 56e3aab304
1 changed files with 1 additions and 1 deletions
--- a/2
+++ b/2
@ -42,7 +42,7 @@ visit $SITE urls.txt

 fi

-URLLIST=`cat urls.txt|sort|uniq`
+URLLIST=`cat urls.txt|grep -v '<'|grep -v ' '|sort|uniq`

 rm urls.txt