#!/usr/local/in/bash
# dump privoxy.log and urls.txt in this path


# backup urls ..
wc -l urls.txt
cp /data/scripts/urls.txt /data/scripts/urls_$RANDOM.txt


wc -l privoxy.log
cat privoxy.log | awk '{print $6}' | sed 's/zero.webappsecurity.com:80//g'|grep -v zero >> /data/scripts/urls.txt


# pull niktoDB and convert to just URLS
wget -A asdfasdf 'http://174.142.17.165/nikto/UPDATES/db_tests' -O db_tests
wc -l db_tests
cat db_tests |grep -v '^#' | sed -e s'/@NUKE/\/phpnuke/g' -e 's/@PHPMYADMIN/\/phpmyadmin/g' -e 's/@CGIDIR/\/cgi-bin/g' | sed 's/.*"\//\//g'  | sed 's/",".*//g' >> urls.txt

# dos2unix and pull out the dupes and sort
dos2unix /data/scripts/urls.txt
awk '!x[$0]++' /data/scripts/urls.txt |sort > /data/scripts/tmp
mv /data/scripts/tmp /data/scripts/urls.txt


wc -l urls.txt



echo creating nikto_v10_$out1.db
out1=$RANDOM
for i in `cat urls.txt`
do
echo \"generic\",\"$i\",\"200\",\"GET\",\"rmccurdy.com\" >> nikto_v10_$out1.db
done



echo creating nikto_v20_$out1.db
n=400001
for i in `cat urls.txt`
do
echo \"$n\",\"0\",\"1\",\"$i\",\"GET\",\"200\",\"\",\"\",\"\",\"\",\"rmccurdy.com\",\"\",\"\" >> nikto_v20_$out1.db
n=`echo $n + 1 |bc`
done