无捻解决方案:
from urlparse import urlparsefrom threading import Threadimport httplib, sysfrom Queue import Queueconcurrent = 200def doWork(): while True: url = q.get() status, url = getStatus(url) doSomethingWithResult(status, url) q.task_done()def getStatus(ourl): try: url = urlparse(ourl) conn = httplib.HTTPConnection(url.netloc)conn.request("HEAD", url.path) res = conn.getresponse() return res.status, ourl except: return "error", ourldef doSomethingWithResult(status, url): print status, urlq = Queue(concurrent * 2)for i in range(concurrent): t = Thread(target=doWork) t.daemon = True t.start()try: for url in open('urllist.txt'): q.put(url.strip()) q.join()except KeyboardInterrupt: sys.exit(1)这比扭曲的解决方案要快一点,并且使用的CPU更少。



