mirror of https://xfnw.ttm.sh/git/huntnw.git
fix slep
This commit is contained in:
parent
1f83896371
commit
d94901f6f2
5
crawly
5
crawly
|
@ -16,12 +16,12 @@ if [[ $string =~ $regex && $string != *"<"* && $string != *">"* ]]
|
||||||
then
|
then
|
||||||
|
|
||||||
echo visiting $1
|
echo visiting $1
|
||||||
|
sleep 1
|
||||||
# get URLs
|
# get URLs
|
||||||
|
|
||||||
CRAWLED="./content/"`echo $1 | shasum | head -c 5`
|
CRAWLED="./content/"`echo $1 | shasum | head -c 5`
|
||||||
echo $CRAWLED
|
echo $CRAWLED
|
||||||
curl $1 -m 5 -L > $TEMPIDC
|
curl $1 -m 1 -L > $TEMPIDC
|
||||||
|
|
||||||
cat $TEMPIDC | grep href=\" | grep "https://" | grep -o "https:\/\/[^\"]*" | awk '/html$/ || /php$/ || /txt$/ || /\/$/ { print $0 }' >> $2
|
cat $TEMPIDC | grep href=\" | grep "https://" | grep -o "https:\/\/[^\"]*" | awk '/html$/ || /php$/ || /txt$/ || /\/$/ { print $0 }' >> $2
|
||||||
cat $TEMPIDC | grep href=\" | grep "https://" | grep -o "https:\/\/[^\"]*" | awk '!/html$/ && !/php$/ && !/txt$/ && !/\/$/ { print $0"/" }' >> $2
|
cat $TEMPIDC | grep href=\" | grep "https://" | grep -o "https:\/\/[^\"]*" | awk '!/html$/ && !/php$/ && !/txt$/ && !/\/$/ { print $0"/" }' >> $2
|
||||||
|
@ -45,7 +45,6 @@ rm urls.txt
|
||||||
|
|
||||||
for fn in $URLLIST; do
|
for fn in $URLLIST; do
|
||||||
echo "found URL $fn"
|
echo "found URL $fn"
|
||||||
sleep 1
|
|
||||||
visit $fn urls.txt
|
visit $fn urls.txt
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue