#!/bin/bash
#ort varagent='"msnbot/2.0b (+http://search.msn.com/msnbot.htm)"'
export varagent='Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)'
export outputdir=/srv/http/nginx/proxy

# rmccurdy.com
# need curl and spidermonkey   
#echo "UPDATED: no multi connections support one at a time"
#echo "also no cookie needed for proxyies.my-proxy.com anymore !!?"
#echo "FIXED 04/15/2010 : nntime.com changed script"
#echo "UPDATE 11/09 : crontab the script so it should auto update"
#echo "UPDATE 4/20 : fixed my-proxy.com"
#echo killing cron snort snortsam
#killall snort snortsam cron

rm cookie sakura good.txt proxylist.txt
rm $outputdir/elite.txt $outputdir/anony.txt $outputdir/trans.txt

if [ ! -f "proxylist.txt" ]
then
 echo "proxylist.txt not found downloading list"
 sleep 5
 echo "ripping freeproxylists.com"
 sleep 1
 for img in `curl -A "$varagent"  -s http://www.freeproxylists.com/elite.php | grep "elite " | sed 's/.*elite\///g' | sed 's/\.html.*//g' | sed 's/^/http:\/\/www.freeproxylists.com\/load_elite_/g' | sed 's/$/\.html/g'`
 do
  	curl -A "$varagent"  -s $img | awk '{gsub("&lt;/td&gt;&lt;/tr&gt;&lt;tr&gt;&lt;td&gt;","\n"); print}' | awk '{gsub("&lt;/td&gt;&lt;td&gt;",":"); print}' | sed 's/&lt.*//g' | grep -v "<" >> proxylist.txt
 done

 echo "ripping proxylist.sakura.ne.jp"
 sleep 1
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=0' | grep 'proxy([1-4]' > tmp
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=1' | grep 'proxy([1-4]' >> tmp
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=2' | grep 'proxy([1-4]' >> tmp
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=3' | grep 'proxy([1-4]' >> tmp
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=4' | grep 'proxy([1-4]' >> tmp
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=5' | grep 'proxy([1-4]' >> tmp
 curl -s -A "$varagent"  'http://proxylist.sakura.ne.jp/index.htm?pages=6' | grep 'proxy([1-4]' >> tmp
 mv tmp sakura
 cat sakura.js sakura > tmp 
 js tmp >> proxylist.txt

 echo "ripping multiproxy.org"
 lynx -connect_timeout=3 -width=999 -dump -nolist "http://www.multiproxy.org/cgi-bin/search-proxy.pl" | sed 's/ //'g | grep ':'  >> proxylist.txt

 echo "ripping nntime.com"
 sleep 1
 rm tmp
 for i in {01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17}
 do
  	curl -A "$varagent"  -s "http://nntime.com/proxy-list-$i.htm"  -A "SAMSUNG-SGH-E250/1.0 Profile/MIDP-2.0 Configuration/CLDC-1.1 UP.Browser/6.2.3.3.c.1.101 (GUI) MMP/2.0 (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)" | egrep "(document.write| = )|;<\/script>" |sed -e 's/.*<td>/print("/g' -e 's/<script type="text\/javascript">document.write(//g' -e 's/":/:/g' -e 's/<\/script>.*/;/g' >> tmp
 done
 js tmp >> proxylist.txt

 echo "ripping www.my-proxy.com"
 sleep 1
 rm cookie
 curl -A "$varagent"  -s  -c cookie -b cookie http://www.my-proxy.com/list/proxy.php > /dev/null
 cat cookie  | sed 's/0$/2/' >tmp
 mv tmp cookie
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-s1.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-s2.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  | grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-s3.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  | grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-1.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-2.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-3.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-4.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-5.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-6.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-7.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-8.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-9.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  |grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt
 curl -A "$varagent"  -s  -c cookie -b cookie 'http://proxies.my-proxy.com/proxy-list-10.html' -e 'http://www.m.com/list/verify.php'http://www.my-proxy.com/list/verify.php  | grep br | awk '{gsub("<br>","\n"); print}' | grep "[0-9]\.[0-9]" >> proxylist.txt

 echo "ripping 'http://www.z4.cn/kit/proxy/?act=list&page=1'"
 sleep 1
 lynx -connect_timeout=3 -width=999 -dump -nolist  'http://www.z4.cn/kit/proxy/?act=list&page=1' | grep ":" | awk '{print $2}' >> proxylist.txt

 echo "ripping www.proxylists.net , www.shroomery.org/ythan/proxylist.php  http://proxygo.com.ru ..."
 lynx -connect_timeout=3 -width=999 -dump -nolist 'http://www.proxylists.net/http_highanon.txt' >> proxylist.txt
 lynx -connect_timeout=3 -width=999 -dump -nolist 'http://www.shroomery.org/ythan/proxylist.php' >> proxylist.txt

 echo "removing dupes"
 # remove ^M
 # cat proxylist.txt | tr -d '\015\032' > tmp
 #mv tmp proxylist.txt
 #echo "Found `cat proxylist.txt | wc -l` proxies"
 cat proxylist.txt | {
     awk '!x[$0]++'
 } > tmp
 mv tmp proxylist.txt
fi

echo Checking `cat proxylist.txt | wc -l`  proxies
sleep 1 

for img in `
grep "^[0-9]" proxylist.txt`
do
 echo $img >> good.txt
# curl -A "$varagent"   -s -x $img --url http://whatismyip.org -L --connect-timeout 1 -m 10 | grep '^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$' >>  /srv/http/nginx/proxytools/good.txt
#if [[ `ps aux|grep curl|grep -v grep|wc -l` -gt 5 ]] ;then echo Max connect sleeping `date`; sleep 5  ;fi
done

echo `date` > $outputdir/elite.txt
echo `date` > $outputdir/anony.txt
echo `date` > $outputdir/trans.txt

x=1
while [ $x -lt $(cat good.txt | wc -l) ]
do
  let x=x+1
  PROXY=`head -n $x good.txt | tail -n 1`
  echo "Checking proxy $PROXY, number $x"
  CONTENT=`curl "http://afinet.free.fr/track.php" --proxy "$PROXY" --silent --compressed --progress-bar --connect-timeout 30`
  if [[ "$CONTENT" =~ "class=GOOD><td>RESUME" ]]
  then
    echo "proxy is elite!"
    echo $PROXY >> $outputdir/elite.txt
  elif [[ "$CONTENT" =~ "class=NOTSOBAD><td>RESUME" ]]
  then
    echo "proxy is anonymous!"
    echo $PROXY >> $outputdir/anony.txt
  elif [[ "$CONTENT" =~ "class=BAD><td>RESUME" ]]
  then
    echo "proxy is transparent!"
    echo $PROXY >> $outputdir/trans.txt
  else
    echo "proxy is DOWN!"
  fi
done

#cp proxylist.txt good.txt

#echo `date` > good.txt
#for img in `
#grep "^[0-9]" proxylist.txt`
#do
# echo $img >> /srv/http/nginx/proxytools/good.txt
# curl -A "$varagent"   -s -x $img --url http://whatismyip.org -L --connect-timeout 1 -m 10 | grep '^[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}$' >>  /srv/http/nginx/proxytools/good.txt
#if [[ `ps aux|grep curl|grep -v grep|wc -l` -gt 5 ]] ;then echo Max connect sleeping `date`; sleep 5  ;fi
#done

#echo OK, finish
#echo `date` >> /srv/http/nginx/proxytools/good.txt