From 13a3e1d128b62a5c3928f17856bb2006b3e52f99 Mon Sep 17 00:00:00 2001
From: lickthecheese <lickthecheese@tilde.team>
Date: Fri, 20 Mar 2020 10:23:33 -0400
Subject: [PATCH] fixed cleaning script

---
 clean  | 2 +-
 crawly | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/clean b/clean
index 0eac0fc..70bbda6 100755
--- a/clean
+++ b/clean
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 mv urls.txt urls.txt.bak
-cat urls.txt.bak | sort | uniq > urls.txt
+cat urls.txt.bak | sort | uniq -w 20 > urls.txt
 rm urls.txt.bak
 
 
diff --git a/crawly b/crawly
index 2977dac..a93b436 100755
--- a/crawly
+++ b/crawly
@@ -52,7 +52,7 @@ visit $fn urls.txt
 done
 
 mv urls.txt urls.txt.bak
-cat urls.txt.bak | sort | uniq > urls.txt
+cat urls.txt.bak | sort | uniq -w 20 > urls.txt
 rm urls.txt.bak
 
 find ./content/ -type 'f' -size -100c -delete