这个爬虫的作用是，对于一个给定的url，查找页面里面所有的url连接并依次贪婪爬取

 

主要需要注意的地方：

1.lxml.html.iterlinks()  可以实现对页面所有url的查找

2.获取页面要设置超时，否则遇到没有响应的页面，代码容易卡死

3.对所有异常的捕获

4.广度优先搜索实现

 

具体代码如下：

#!/usr/bin/env python#encoding:utf8#这是一个爬虫代码，从初始url页面中查找所有的url并根据广度优先的顺序依次贪婪爬取#爬取的页面以1.html,2.html...的方式命名 #author：匡子语 import reimport timeimport lxml.htmlimport urllib2from collections import dequeclass ScrawURL:    def __init__(self):        self.total = 50000   #要获取的页面数        self.urls = ["http://www.cnblogs.com/"]  #初始url        self.html_num = 13047   #当前存储的是第几个页面        self.url_num = 0   #当前要爬取的url在urls中的位置        def getHtml(self, url):        print "url:%s" % url        user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'        headers = { 'User-Agent' : user_agent }        try:            request = urllib2.Request(url, headers = headers)            response = urllib2.urlopen(request, timeout=5)   #要设置timeout,有时页面长时间无法响应不加timeout会导致程序卡死            content = response.read()            self.html_num += 1            name = "htmls/%d.html" % self.html_num            with open(name, "w") as f:                f.write(content)            print self.html_num            return content        except urllib2.HTTPError, e:            return ''        except urllib2.URLError, e:            return ''        except Exception, ex:   #对于所有异常形式，都返回空内容            return ''        def getUrl(self, content):        try:            print "getUrl"            html = lxml.html.fromstring(content)            links = lxml.html.iterlinks(html)   #这个函数可以自动获取页面中的所有url，非常好用            urls = []            for link in links:                if "http" in link[2]:                    urls.append(link[2])            return urls        except Exception, ex:            return []        def scrawl(self):   #广度优先爬取        print "scrawl"        while len(self.urls) < self.total and len(self.urls) > self.url_num:   #获取足够的url            url = self.urls[self.url_num]            self.url_num += 1            content = self.getHtml(url)            if content:                urls = self.getUrl(content)                if urls:                    for url in urls:                        if url not in self.urls:   #对于已经爬取过的url跳过                            self.urls.append(url)                while self.html_num < self.total and len(self.urls) > 0:   #获取足够的页面            url = self.urls[self.url_num]            self.url_num += 1            self.getHtml(url)        if __name__ == "__main__":    surl = ScrawURL()    surl.scrawl()