dont re-crawl the same page you dingus

2020-06-23 16:04:44 +00:00 · 2020-06-23 16:04:44 +00:00 · c58681421e
parent 112f6a5b19
commit c58681421e
1 changed files with 1 additions and 1 deletions
--- a/2
+++ b/2
@ -42,7 +42,7 @@ visit $SITE urls.txt

 fi

-URLLIST=`cat urls.txt`
+URLLIST=`cat urls.txt|sort|uniq`

 rm urls.txt