Python 爬取淘寶商品信息欄目的實(shí)現(xiàn)

 #提取公共的爬取信息的api
 def commonsdk(self,url):
 browser = webdriver.Chrome('D:/chromedriver.exe')
 try:
 browser.get(url)
 except Exception as e:
 browser.execute_script('window.stop()') # 超出時(shí)間則不加載
 print(e, 'dynamic web load timeout')
 return browser;

#模擬登錄
 def logon(self,url,a_href_list_next):
 username = "淘寶賬戶名"
 password = "密碼"
 browser1 = self.commonsdk(url)
 #登錄賬號(hào)
 browser1.find_element_by_id('fm-login-id').send_keys(username)
 browser1.find_element_by_id('fm-login-password').send_keys(password)
 #模擬用戶點(diǎn)擊登錄
 browser1.find_element_by_xpath('//*[@id="login-form"]/div[4]/button').click()
 #解析商品信息
 self.Buy_information(a_href_list_next,browser1)

#爬取目錄
 def List(self,url):
 browser = self.commonsdk(url)
 #ActionChains類實(shí)現(xiàn)鼠標(biāo)的單擊、雙擊、拖拽等功能
 action = ActionChains(browser)
 li_list = browser.find_elements_by_css_selector('.service-bd li')[1]
 #實(shí)現(xiàn)執(zhí)行鼠標(biāo)懸停,便于爬取懸停內(nèi)容
 action.move_to_element(li_list).perform()
 time.sleep(5)

 #爬取整個(gè)目錄的div
 div_list = browser.find_element_by_css_selector('.service-fi-links')
 #爬取其中的總的名稱
 h5_list = div_list.find_elements_by_css_selector('h5')
 #爬取小標(biāo)題的名稱
 p_list = div_list.find_elements_by_css_selector('p')
 #獲取a標(biāo)簽
 a_href_list = div_list.find_elements_by_css_selector('a')
 #獲取a標(biāo)簽的超鏈接
 a_href_list_next = div_list.find_elements_by_css_selector('a')[1].get_attribute('href')

 print(li_list.text)
 for j in range(len(p_list)):
 if jlen(p_list):
 print(h5_list[j].text)
 print(p_list[j].text)
 for i in range(len(a_href_list)):
 print(a_href_list[i].get_attribute('href'))

 #獲取登錄框
 logon = browser.find_element_by_id('J_SiteNavBd')
 #獲取登錄框的超鏈接
 logon_url = logon.find_element_by_css_selector('a').get_attribute('href')
 #先關(guān)閉第一個(gè)網(wǎng)頁(yè)
 browser.close()
 self.logon(logon_url,a_href_list_next)

#爬取商品信息
 def Buy_information(self,url,browser):
 browser.get(url)
 div_list = browser.find_element_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div[1]')
 img = div_list.find_element_by_css_selector('img')
 img_url = "https:"+img.get_attribute('data-src')
 price = div_list.find_element_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div[1]/div[2]/div[1]/div[1]').text
 number = div_list.find_element_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div[1]/div[2]/div[1]/div[2]').text
 shoping_information = div_list.find_element_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div[1]/div[2]/div[2]').text
 shop = div_list.find_element_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div[1]/div[2]/div[3]/div[1]/a').text
 adress = div_list.find_element_by_xpath('//*[@id="mainsrp-itemlist"]/div/div/div[1]/div[1]/div[2]/div[3]/div[2]').text
 path = self.img_baocun(img_url)
 data={
 '圖片路徑':path,
 '價(jià)格':price,
 '購(gòu)買人數(shù)':number,
 '商品信息':shoping_information,
 '商家':shop,
 '籍貫':adress
 }
 self.write_dictionary_to_csv(data,'information')

#下載照片
 def img_baocun(self,url):
 root = "文件夾下載的路徑"http://電腦上的絕對(duì)路徑
 path = root + url.split('?')[0].split('/')[-1].strip('')#獲取jpg的名稱
 #判斷是否存在該路徑，不存在則創(chuàng)建
 if not os.path.exists(root):
 os.mkdir(root)
 #判斷是否存在該圖片，存在則不下載
 if not os.path.exists(path):
 r = requests.get(url)
 r.raise_for_status()
 with open(path,'wb') as f:
 f.write(r.content)
 return path

#將解析得到的內(nèi)容（字典格式）逐行寫入csv文件
 def write_dictionary_to_csv(self,dict,filename):
 #格式化文件名
 file_name='{}.csv'.format(filename)
 with open(file_name, 'a',encoding='utf-8') as f: 
 file_exists = os.path.isfile(filename)
 #delimiter(定界符) 
 w =csv.DictWriter(f, dict.keys(),delimiter=',', quotechar='"', lineterminator='\n',quoting=csv.QUOTE_ALL, skipinitialspace=True)
 if not file_exists :
 w.writeheader()
 w.writerow(dict)
 print('當(dāng)前行寫入csv成功！')