import re

from collections import OrderedDict

from bs4 import BeautifulSoup

from pyquery import PyQuery as pq #獲取整個網(wǎng)頁的源代碼

from config import * #可引用congif的所有變量

import pymysql

import urllib

import json

import bs4

import requests

from selenium import webdriver

from selenium.webdriver.support.ui import WebDriverWait

from pyquery import PyQuery as pq #獲取整個網(wǎng)頁的源代碼

import pandas as pd

# 測試 淘寶+天貓，可完整輸出及保存

browser = webdriver.Firefox()

wait = WebDriverWait(browser,10)

####### 天貓上半部分詳情 #############

def get_tianmao_header(url):

browser.get(url)

# wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item'))) #加載所有寶貝

html=browser.page_source

doc = pq(html)

# print(doc)

info = OrderedDict() # 存放該商品所具有的全部信息

items = doc('#page')

# info['店鋪名'] = items.find('.slogo').find('.slogo-shopname').text()

# info['ID'] = items.find('#LineZing').attr['itemid']

info['寶貝'] = items.find('.tb-detail-hd').find('h1').text()

info['促銷價'] = items.find('#J_PromoPrice').find('.tm-promo-price').find('.tm-price').text()

info['原價'] = items.find('#J_StrPriceModBox').find('.tm-price').text()

# '月銷量' :items.find('.tm-ind-panel').find('.tm-ind-item tm-ind-sellCount').find('.tm-indcon').find('.tm-count').text(),

info['月銷量'] = items.find('.tm-ind-panel').find('.tm-indcon').find('.tm-count').text().split(' ',2)[0]

info['累計評價'] = items.find('#J_ItemRates').find('.tm-indcon').find('.tm-count').text()

# print(info)

return info

######## 淘寶上半部分詳情 ###############

def get_taobao_header(url):

browser.get(url)

# wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR,'#mainsrp-itemlist .items .item'))) #加載所有寶貝

html=browser.page_source

doc = pq(html)

# print(doc)

info = OrderedDict() # 存放該商品所具有的全部信息

items = doc('#page')

# info['店鋪名'] = items.find('.tb-shop-seller').find('.tb-seller-name').text()

# info['ID'] = items.find('#J_Pine').attr['data-itemid']

info['寶貝'] = items.find('#J_Title').find('h3').text()

info['原價'] = items.find('#J_StrPrice').find('.tb-rmb-num').text()

info['促銷價'] = items.find('#J_PromoPriceNum').text()

# '月銷量' :items.find('.tm-ind-panel').find('.tm-ind-item tm-ind-sellCount').find('.tm-indcon').find('.tm-count').text(),

info['月銷量'] = items.find('#J_SellCounter').text()

info['累計評價'] = items.find('#J_RateCounter').text()

# print(info)

return info

####################### 詳情 ############################

# 抓取所有商品詳情

def get_Details(attrs,info):

# res = requests.get(url)

# soup = BeautifulSoup(res.text, "html.parser")

#

# attrs = soup.select('.attributes-list li')

# attrs= [

厚薄: 薄, 材質(zhì)成分: 其他100%,]

attrs_name = []

attrs_value = []

'''''

[\s] 匹配空格，[\s]*,后面有 *，則可以為空

* : 匹配前面的子表達式任意次

'''

for attr in attrs:

attrs_name.append(re.search(r'(.*?):[\s]*(.*)', attr.text).group(1))

attrs_value.append(re.search(r'(.*?):[\s]*(.*)', attr.text).group(2))

# print('attrs_name=',attrs_name) # attrs_name= ['厚薄', '材質(zhì)成分', ...]

# print('attrs_value=',attrs_value) # attrs_value= ['薄', '其他100%', ...]

allattrs = OrderedDict() # 存放該產(chǎn)品詳情頁面所具有的屬性

for k in range(0, len(attrs_name)):

allattrs[attrs_name[k]] = attrs_value[k]

# print('allattrs=',allattrs) # allattrs= OrderedDict([('厚薄', '薄'), ('材質(zhì)成分', '其他100%'),...])

# info = OrderedDict() # 存放該商品所具有的全部信息

# info = get_headdetail2(url)

# 下面三條語句獲取描述、服務(wù)、物流的評分信息

# 下面的語句用來判斷該商品具有哪些屬性，如果具有該屬性，將屬性值插入有序字典，否則，該屬性值為空

# 適用場景

if '材質(zhì)成分' in attrs_name:

info['材質(zhì)成分'] = allattrs['材質(zhì)成分']

elif '面料' in attrs_name:

info['材質(zhì)成分'] = allattrs['面料']

else:

info['材質(zhì)成分'] = 'NA'

# 適用對象

if '流行元素' in attrs_name:

info['流行元素'] = allattrs['流行元素']

else:

info['流行元素'] = 'NA'

#季節(jié)

if '年份季節(jié)' in attrs_name:

info['年份季節(jié)'] = allattrs['年份季節(jié)']

else:

info['年份季節(jié)'] = 'NA'

# 款式

if '袖長' in attrs_name:

info['袖長'] = allattrs['袖長']

else:

info['袖長'] = 'NA'

# 尺碼

if '銷售渠道類型' in attrs_name:

info['銷售渠道類型'] = allattrs['銷售渠道類型']

else:

info['銷售渠道類型'] = 'NA'

# 帽頂款式

if '貨號' in attrs_name:

info['貨號'] = allattrs['貨號']

else:

info['貨號'] = 'NA'

# 帽檐款式

if '服裝版型' in attrs_name:

info['服裝版型'] = allattrs['服裝版型']

else:

info['服裝版型'] = 'NA'

# 檐形

if '衣長' in attrs_name:

info['衣長'] = allattrs['衣長']

else:

info['衣長'] = 'NA'

# 主要材質(zhì)

if '領(lǐng)型' in attrs_name:

info['領(lǐng)型'] = allattrs['領(lǐng)型']

else:

info['領(lǐng)型'] = 'NA'

# 人群

if '袖型' in attrs_name:

info['袖型'] = allattrs['袖型']

else:

info['袖型'] = 'NA'

# 品牌

if '品牌' in attrs_name:

info['品牌'] = allattrs['品牌']

else:

info['品牌'] = 'NA'

# 風格

if '圖案' in attrs_name:

info['圖案'] = allattrs['圖案']

elif '中老年女裝圖案' in attrs_name:

info['圖案'] = allattrs['中老年女裝圖案']

else:

info['圖案'] = 'NA'

# 款式細節(jié)

if '服裝款式細節(jié)' in attrs_name:

info['服裝款式細節(jié)'] = allattrs['服裝款式細節(jié)']

else:

info['服裝款式細節(jié)'] = 'NA'

# 適用年齡

if '適用年齡' in attrs_name:

info['適用年齡'] = allattrs['適用年齡']

else:

info['適用年齡'] = 'NA'

# 風格

if '風格' in attrs_name:

info['風格'] = allattrs['風格']

elif '中老年風格' in attrs_name:

info['風格'] = allattrs['中老年風格']

else:

info['風格'] = 'NA'

#通勤

if '通勤' in attrs_name:

info['通勤'] = allattrs['通勤']

else:

info['通勤'] = 'NA'

if '裙長' in attrs_name:

info['裙長'] = allattrs['裙長']

else:

info['裙長'] = 'NA'

if '裙型' in attrs_name:

info['裙型'] = allattrs['裙型']

else:

info['裙型'] = 'NA'

if '腰型' in attrs_name:

info['腰型'] = allattrs['腰型']

else:

info['腰型'] = 'NA'

# 顏色分類

if '主要顏色' in attrs_name:

info['主要顏色'] = allattrs['主要顏色']

else:

info['主要顏色'] = 'NA'

if '顏色分類' in attrs_name:

info['主要顏色'] = allattrs['顏色分類']

else:

info['主要顏色'] = 'NA'

#尺碼

if '尺碼' in attrs_name:

info['尺碼'] = allattrs['尺碼']

else:

info['尺碼'] = 'NA'

if '組合形式' in attrs_name:

info['組合形式'] = allattrs['組合形式']

else:

info['組合形式'] = 'NA'

if '褲長' in attrs_name:

info['褲長'] = allattrs['褲長']

else:

info['褲長'] = 'NA'

return info

import csv

def main():

# 提取 列

with open('clothes_detai.csv', 'w', newline='', encoding='utf-8') as csvfile:

# fieldnames = ['店鋪ID','店鋪名','鏈接','寶貝','原價','促銷價','月銷量','累計評價','材質(zhì)成分','流行元素','袖長','年份季節(jié)','銷售渠道類型','貨號','服裝版型','衣長','領(lǐng)型','袖型',

# '裙型','裙長','腰型','褲長','組合形式','品牌','圖案','服裝款式細節(jié)', '適用年齡','風格','通勤','主要顏色','尺碼']

fieldnames=[ 'Link','Brand','Title','Price','Sale price','Sales','Evaluations',

'Component', 'Fashion elements','Sleeve','Seasons','Sales channels',

'Number','Clothes_Style','Long','Collar type','Sleeve type',

'Skirt type','Skirt length','Waist','Combining form','Outseam',

'Design','Fashion pattern detail','Applicable age',

'Style','Commuter','color','Size']

# 'Shop','Data_id','Shop_id','Shop','Link','Data_id',

writer = csv.DictWriter(csvfile, fieldnames = fieldnames)

writer.writeheader()

# urls = ['//detail.tmall.com/item.htm?spm=a230r.1.14.1.ebb2eb2eGyUw1&id=549177691667&ns=1&abbucket=4',

# '//item.taobao.com/item.htm?id=548443640333&ns=1&abbucket=0#detail']

f = pd.read_csv('women_clothes_sales2.csv')

urls = f['link'][0:100]

# sh = f['shop_id'][0:3]

# s = f['shop'][0:3]

# for url in urls:

# print(url)

# writer.writerow({'店鋪ID':f['shop_id'],'店鋪名':f['shop']})

keys, values = [], []

# for url in urls:

for i in urls:

url = 'http:' + i

# endswith 判斷字符串是否以指定的字符串結(jié)尾

if url.endswith('detail'):

info = get_taobao_header(url)

res = requests.get(url)

soup = BeautifulSoup(res.text, "html.parser")

attrs = soup.select('.attributes-list li') # 淘寶 class

else:

info = get_tianmao_header(url)

res = requests.get(url)

soup = BeautifulSoup(res.text, "html.parser")

attrs = soup.select('#J_AttrUL li') # 天貓 id

# print('attrs=',attrs)

d = get_Details(attrs,info)

print(d)

# for j in f[shop_id]:

# d['店鋪ID'] = j

# for s in f['shop']:

# d['店鋪名'] = s

#'Shop':d['店鋪名'],'Data_id':d['ID'],

writer.writerow({'Link':url,'Brand':d['品牌'],'Title':d['寶貝'], 'Price':d['原價'], 'Sale price':d['促銷價'], 'Sales':d['月銷量'], 'Evaluations':d['累計評價'],

'Component':d['材質(zhì)成分'], 'Fashion elements':d['流行元素'], 'Sleeve':d['袖長'], 'Seasons':d['年份季節(jié)'], 'Sales channels':d['銷售渠道類型'],

'Number':d['貨號'],'Clothes_Style':d['服裝版型'],'Long':d['衣長'],'Collar type':d['領(lǐng)型'], 'Sleeve type':d['袖型'],

'Skirt type':d['裙型'], 'Skirt length':d['裙長'], 'Waist':d['腰型'], 'Combining form':d['組合形式'], 'Outseam':d['褲長'],

'Design':d['圖案'], 'Fashion pattern detail':d['服裝款式細節(jié)'], 'Applicable age':d['適用年齡'],

'Style':d['風格'], 'Commuter':d['通勤'], 'color':d['主要顏色'], 'Size':d['尺碼']})

if __name__=='__main__':

main()

總結(jié)

以上是生活随笔為你收集整理的python爬虫淘宝和天猫的区别_python爬虫获取淘宝天猫商品详细参数的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。