判断URL是否被百度收录 #coding=utf-8 import urllib,urllib2,re,random def get_proxies(): html = urllib2.urlopen('http://www.ykit.cn/').read() pa = re.compile(r'<td>(\d+.\d+.\d+.\d+)</td>[\s\S]+?<td>(\d+)</td>[\s\S]+?<div title="(\d\.\d+?)秒" class="bar">') ip_list = pa.findall(html) return [x for x in ip_list if x[2] < '3'] ip_list = get_proxies() print 'collect %s ip' % len(ip_list) def is_index(line): while 1: try: ip = random.choice(ip_list) proxies = {} proxies['http'] = 'http://%s:%s' % (ip[0], ip[1]) proxy_handler = urllib2.ProxyHandler(proxies) opener = urllib2.build_opener(proxy_handler) html=opener.open('http://www.baidu.com/s?wd=%s' % urllib.quote_plus(line), timeout=15).read() if '="http://verify.baidu.com' in html: ip_list.remove(ip) print ("yzm show", ip[0]) continue if '<div class="nors"><p>抱歉,没有找到与' in html or 'class=f14>没有找到该URL。您可以直接访问' in html: return (0, ip[0]) else: return (1, ip[0]) except: ip_list.remove(ip) is_index(line) for line in open('url.txt'): line = line.strip() print line + '\t' + '%s\t%s' % is_index(line)
批量查询URL是否被收录(Python多线程)
上一篇:返回列表