From 06cffd61a2e8e7fe12ded8492cd6b2d1a0661bc8 Mon Sep 17 00:00:00 2001
From: lickthecheese <lickthecheese@tilde.team>
Date: Thu, 19 Mar 2020 08:48:05 -0400
Subject: [PATCH] recursive crawling

---
 crawlreq | 11 +++++++++++
 crawly   |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100755 crawlreq

diff --git a/crawlreq b/crawlreq
new file mode 100755
index 0000000..b71db55
--- /dev/null
+++ b/crawlreq
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+URLSTOCRAWL=`cat urls.txt`
+
+rm urls.txt -i
+
+for fn in $URLSTOCRAWL; do
+echo "recursively finding urls from $fn"
+./crawly $fn
+done
+
diff --git a/crawly b/crawly
index e3033f1..b1df062 100755
--- a/crawly
+++ b/crawly
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-rm urls.txt -i
+
 
 cleanxss(){ sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g; s/"/\&quot;/g; s/'"'"'/\&#39;/g' | tr "\n" " " ; }