python 爬取当当网商品价格并写入数据库
生活随笔
收集整理的這篇文章主要介紹了
python 爬取当当网商品价格并写入数据库
小編覺得挺不錯的,現在分享給大家,幫大家做個參考.
python 爬取數據并寫入數據庫案例：
import requests import pymysql import re from fake_useragent import UserAgent from lxml import etreeua = UserAgent() values = [] def spider(db, cursor,key,index):headers = { 'User-Agent':ua.random}url = "http://search.dangdang.com/?key={key}&act=input&page_index={index}".format(key = key,index = index)response = requests.get(url,headers = headers)html = response.textcontent = etree.HTML(html)ul_list = content.xpath('//div[@id="search_nature_rg"]/ul[@id="component_59"]/li')for li in ul_list:title = li.xpath('./a/@title')[0] # 商品名price = li.xpath('.//p[@class="price"]/span[@class="price_n"]/text()') # 價格price = ''.join(price).replace("￥","")link = li.xpath('./a/@href')[0] # 鏈接value = (title,price,link)values.append(value)#print(value)def table_exists(cursor,table_name): #判斷數據庫中表是否存在,不存在則新建findsql = "show tables;"cursor.execute(findsql)tables = [cursor.fetchall()]table_list = re.findall('(\'.*?\')',str(tables))table_list = [re.sub("'",'',each) for each in table_list]if table_name in table_list:print('表已存在不需要創建')else:sql = """CREATE TABLE IF NOT EXISTS `{}` (`title` VARCHAR(2000),`price` VARCHAR (10),`link` VARCHAR(2000))""" .format(table_name)cursor.execute("drop table if exists {}".format(table_name))cursor.execute(sql) # 創建表print("創建成功")def insertData(db, cursor):for item in range(1, 30):spider(db, cursor,"手機", item) # 搜索需要爬取的物品for i in values:idtext = i[0]nametext = i[1]linktext = i[2]sql = "INSERT INTO %s (title, price, link)VALUES ('%s', '%s', '%s')" %(table_name,idtext, nametext, linktext)try:cursor.execute(sql) # 執行sql語句db.commit() # 提交到數據庫執行print("成功寫入數據" + idtext)except:db.rollback() # 發生錯誤時回滾print("寫入失敗" + sql)if __name__ == '__main__':db = pymysql.connect(host='localhost', port=3306, user='root', passwd='123456', db='students', charset='utf8')cursor = db.cursor()table_name = 'dangdangwang' # 數據庫表名table_exists(cursor,table_name)insertData(db, cursor)cursor.close() db.close()spider(db, cursor,“手機”, item) # 搜索需要爬取的物品
這句的效果就是這里搜索，
結果展示：
總結
以上是生活随笔為你收集整理的python 爬取当当网商品价格并写入数据库的全部內容，希望文章能夠幫你解決所遇到的問題。
