xfnw
/
huntnw
mirror of https://xfnw.ttm.sh/git/huntnw.git
1
0
Fork 0
Code Issues Releases Wiki Activity

change agent

This commit is contained in:
vulpine 2020-09-15 14:05:36 -04:00
parent f79b2e8cff
commit 280bb20c36
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawly
View File

@ -18,7 +18,7 @@ then
CRAWLED="./content/"`echo $1 | shasum | head -c 5`
echo $CRAWLED
curl $1 -m 1 -L -A "tser/1.0" > $TEMPIDC
curl $1 -m 1 -L -A "huntnw/1.0" > $TEMPIDC
cat $TEMPIDC | grep href=\" | grep "https://" | grep -o "https:\/\/[^\"]*" | awk '/html$/ || /php$/ || /txt$/ || /\/$/ { print $0 }' >> $2
cat $TEMPIDC | grep href=\" | grep "https://" | grep -o "https:\/\/[^\"]*" | awk '!/html$/ && !/php$/ && !/txt$/ && !/\/$/ { print $0"/" }' >> $2