diff --git a/crawly b/crawly
index a93b436..12dd2a4 100755
--- a/crawly
+++ b/crawly
@@ -42,7 +42,7 @@ visit $SITE urls.txt
 
 fi
 
-URLLIST=`cat urls.txt`
+URLLIST=`cat urls.txt|sort|uniq`
 
 rm urls.txt