python爬蟲scrapy基于CrawlSpider類的全站數(shù)據(jù)爬取示例解析

link = LinkExtractor(
# Items只能是一個(gè)正則表達(dá)式,會(huì)提取當(dāng)前頁(yè)面中滿足該"正則表達(dá)式"的url	
　 allow=r'Items/'
)

import scrapy
# 導(dǎo)入相關(guān)的包
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

class TextSpider(CrawlSpider):
 name = 'text'
 allowed_domains = ['www.xxx.com']
 start_urls = ['http://www.xxx.com/']

# 鏈接提取器,從接受到的response對(duì)象中,根據(jù)item正則表達(dá)式提取頁(yè)面中的鏈接
	link = LinkExtractor(allow=r'Items/')
	link2 = LinkExtractor(allow=r'Items/')
# 規(guī)則解析器,根據(jù)callback將鏈接提取器提取到的鏈接進(jìn)行數(shù)據(jù)解析
# follow為true,則表示將鏈接提取器繼續(xù)作用到鏈接提取器所提取到的鏈接頁(yè)面中
# 故:在我們提取多頁(yè)數(shù)據(jù)時(shí),若第一頁(yè)對(duì)應(yīng)的網(wǎng)頁(yè)中包含了第2,3,4,5頁(yè)的鏈接,
# 當(dāng)跳轉(zhuǎn)到第5頁(yè)時(shí),第5頁(yè)又包含了第6,7,8,9頁(yè)的鏈接,
# 令follow=True,就可以持續(xù)作用,從而提取到所有頁(yè)面的鏈接
 rules = (Rule(link, callback='parse_item', follow=True),
 		Rule(link2,callback='parse_content',follow=False))
 # 鏈接提取器link使用parse_item解析數(shù)據(jù)
	def parse_item(self, response):
 item = {}
 
 yield item
 # 鏈接提取器link2使用parse_content解析數(shù)據(jù)
	def parse_content(self, response):
		item = {}
		
		yield item

scrapy genspider crawl -t spiderName www.xxx.com 

import scrapy
from scrapy.linkextractors import LinkExtractor

from scrapy.spiders import CrawlSpider, Rule
from gushiPro.items import GushiproItem,ContentItem

class GushiSpider(CrawlSpider):
 name = 'gushi'
 #allowed_domains = ['www.xxx.com']
 start_urls = ['https://www.gushiwen.org/']

 # 鏈接提取器:只能使用正則表達(dá)式，提取當(dāng)前頁(yè)面的滿足allow條件的鏈接
 link = LinkExtractor(allow=r'/default_\d+\.aspx')

 # 鏈接提取器，提取所有標(biāo)題對(duì)應(yīng)的詳情頁(yè)url
 content_link = LinkExtractor(allow=r'cn/shiwenv_\w+\.aspx')
 rules = (
 # 規(guī)則解析器，需要解析所有的頁(yè)面，所有follow=True
 Rule(link, callback='parse_item', follow=True),

 # 不需要寫follow，因?yàn)槲覀冎恍枰馕鲈斍轫?yè)中的數(shù)據(jù)，而不是詳情頁(yè)中的url
 Rule(content_link, callback='content_item'),
 )

 # 解析當(dāng)前頁(yè)面的標(biāo)題
 def parse_item(self, response):
 p_list = response.xpath('//div[@class="sons"]/div[1]/p[1]')

 for p in p_list:
 title = p.xpath('./a//text()').extract_first()
 item = GushiproItem()
 item['title'] = title
 yield item
 
 # 解析詳情頁(yè)面的標(biāo)題和內(nèi)容
 def content_item(self,response):
 # //div[@id="sonsyuanwen"]/div[@class="cont"]/div[@class="contson"]
 # 解析詳情頁(yè)面的內(nèi)容
 content = response.xpath('//div[@id="sonsyuanwen"]/div[@class="cont"]/div[@class="contson"]//text()').extract()
 content = "".join(content)
 # # 解析詳情頁(yè)面的標(biāo)題
 title = response.xpath('//div[@id="sonsyuanwen"]/div[@class="cont"]/h1/text()').extract_first()
 # print("title:"+title+"\ncontent:"+content)
 item = ContentItem()
 item["content"] = content
 item["title"] = title
 # 將itme對(duì)象傳給管道
 yield item

import scrapy

# 不同的item類是獨(dú)立的，他們可以創(chuàng)建不同的item對(duì)象
class GushiproItem(scrapy.Item):
 # define the fields for your item here like:
 # name = scrapy.Field()
 title = scrapy.Field()

class ContentItem(scrapy.Item):
 title = scrapy.Field()
 content = scrapy.Field()

from itemadapter import ItemAdapter

class GushiproPipeline:
 def __init__(self):
 self.fp = None

 def open_spider(self,spider):
 self.fp = open("gushi.txt",'w',encoding='utf-8')
 print("開始爬蟲")

 def process_item(self, item, spider):
 # 從詳情頁(yè)獲取標(biāo)題和內(nèi)容，所以需要判斷爬蟲文件中傳來(lái)的item是什么類的item
 # item.__class__.__name__判斷屬于什么類型的item
 if item.__class__.__name__ == "ContentItem":
 content = "《"+item['title']+"》",item['content']
 content = "".join(content) 
 print(content)
 self.fp.write(content)
 return item

 def close_spider(self,spider):
 self.fp.close()
 print("結(jié)束爬蟲")