• 本節(jié)內(nèi)容的數(shù)據(jù)見電腦F:\python數(shù)據(jù)\Python海量數(shù)據(jù)（精縮版） 或 百度網(wǎng)盤“我的數(shù)據(jù)文件/Python海量數(shù)據(jù)”

本節(jié)內(nèi)容基于大文件里面內(nèi)容的快速查找的基礎(chǔ)上延申

1.先了解以下QQ.txt

2.讀取數(shù)據(jù)的行數(shù)

filepath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/QQ.txt" QQfile=open(filepath,"rb") i=0 while True:line=QQfile.readline()if i%10000000==0: #讀取的時候，每100萬讀取一次print(i)if not line:breakelse:i+=1 print("end=",i) QQfile.close()

3.將倒排索引存到文件里

def getuser(num):csdninputfile.seek(num,0) #跳到文件的位置。根據(jù)文件排序line=csdninputfile.readline()line=line.decode("gbk","ignore")linelist=line.split("----")if len(linelist)==2:return linelist[0]else:return ""filepath="QQ.txt" csdninputfile=open(filepath,"rb") csdnlist=csdninputfile.readlines()lengthlist=[0] for line in csdnlist:lengthlist.append(len(line)) #讀取每一行的長度到數(shù)組 del csdnlist print("read")i=0 length=len(lengthlist) while i <length -1:lengthlist[i]+=lengthlist[i-1] #疊加，確定每一行的文件位置i+=1 del lengthlist[len(lengthlist)-1] #刪除最后一個 print("get") lengthlist.sort(key=lambda x:getuser(x))#根據(jù)文件排序索引， print("sort")print("index made") indexfilepath="QQindex.txt" saveindexfile=open(indexfilepath,"wb") for i in range(len(lengthlist)-1):#保存為等長，為了隨機訪問saveindexfile.write(format(lengthlist[i],"15d").encode("utf-8")) saveindexfile.close()

事實上，這個程序要運行四五十分鐘而且會中途崩掉好幾次（數(shù)據(jù)量巨大）

---

---

---

因為數(shù)據(jù)量實在是太巨大了，所以我們必須對大文件進行切割和歸并處理

---

---

---

一、數(shù)據(jù)切割算法

1.把num平均分成N等分

• 我們想把100個數(shù)切分成10個文件，那么切割得到[10,10,10,10,10,10,10,10,10,10]
• 而如果這個數(shù)不是整數(shù)，如97，那我們想把它切割成10個文件，可以切分成[10,10,10,10,10,10,10,10,10,7]
  具體的算法是[10,10,10,10,10,10,10,10,10,7]里面包含[（97-97%10）//（10-1）]個10和1個7
• 103想切分成10個文件，那么就是[11,11,11,11,11,11,11,11,11,4],里面包含[(103-103%10)//(10-1)]個11和1個4
• num想切分成N個文件，每個文件包含[(num-num%N)//(N-1)]個數(shù)以及一個額外的數(shù)

def evgSplit(num,N):lastlist=[]if num%N==0:for i in range(N):lastlist.append(num//N)else:evg= (num-num%N)//(N-1)for i in range(N-1):lastlist.append(evg)num=num-evg lastlist.append(num)return lastlistprint( evgSplit(100,10)) print( evgSplit(97,10)) print( evgSplit(103,10)) print( evgSplit(84319637,10))

2.把大文件均分成N個小文件

def evgSplit(num,N):lastlist=[]if num%N==0:for i in range(N):lastlist.append(num//N)else:evg= (num-num%N)//(N-1)for i in range(N-1):lastlist.append(evg)num-=evglastlist.append(num)return lastlistfilelineslist=evgSplit(84319637,10) #84319637是QQ.txt的行數(shù)，filelineslist得到一個[9368847, 9368847, 9368847, 9368847, 9368847, 9368847, 9368847, 9368847, 9368847, 14]的列表 filepath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/QQ.txt" filedir="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/qqsplit" allfile=open(filepath,"rb") for i in range(len(filelineslist)):tmpfilepath=filedir+"/QQ"+str(i)+".txt" #如：E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/qqsplit//QQ0.txt"tmpfile=open(tmpfilepath,"wb")for j in range(filelineslist[i]):line=allfile.readline()tmpfile.write(line)tmpfile.close() allfile.close()

二、數(shù)據(jù)歸并算法

import os filedir="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/qqsplit" #原來的那10個文件都在這個目錄下 #print(os.listdir(filedir)) 得到['QQ0.txt','QQ1.txt','QQ2.txt','QQ3.txt'...] allfilepath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/QQMerge.txt" #這是合并的文件 allfile=open(allfilepath,"wb") for filename in os.listdir(filedir):print(filedir+"//"+filename) #如："E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/QQfile/qqsplit//QQ0.txt"tmpfile=open(filedir+"//"+filename,"rb")while True:line=tmpfile.readline()if not line:breakelse:allfile.write(line)tmpfile.close() allfile.close()

三、數(shù)據(jù)歸并排序法

1.頭索引排序
我們不能對一個大文件一次性排序那就只能先對切割后的小文件分別排序然后歸并

def merge(mylist1,mylist2):listall=[]while len(mylist1)>0 and len(mylist2)>0:if mylist1[0] <mylist2[0]:listall.append(mylist1[0])del mylist1[0]elif mylist1[0]>mylist2[0]:listall.append(mylist2[0])del mylist2[0]else: #兩個數(shù)相等的時候listall.append(mylist1[0])del mylist1[0]listall.append(mylist2[0])del mylist2[0]listall.extend(mylist2) #假如列表1已經(jīng)空了，那就直接把列表2里的元素添加進listall里去listall.extend(mylist1) #假如列表2已經(jīng)空了，那就直接把列表1里的元素添加進listall里去return listallmylist1=[5,7,6,7,8,1,3] mylist2=[2,4,2,6,10,8] mylist1.sort() mylist2.sort() print(merge(mylist1,mylist2))

2.下標排序
之前排序時會邊合并邊刪去小文件里的元素，如果小文件不能刪的時候，那就得用下表排序

def merge(mylist1,mylist2):listall=[]i=0j=0while i<len(mylist1) and j <len(mylist2):if mylist1[i]<mylist2[j]:listall.append(mylist1[i])i+=1elif mylist1[i]>mylist2[j]:listall.append(mylist2[j])j += 1else:listall.append(mylist1[i])i += 1listall.append(mylist2[j])j += 1while i <len(mylist1): #當mylist2已經(jīng)遍歷完而mylist1還沒有遍歷完的時候listall.append(mylist1[i])i+=1while j <len(mylist2):listall.append(mylist2[j])j+=1return listallmylist1=[5,7,6,7,8,1,3] mylist2=[2,4,2,6,10,8] mylist1.sort() mylist2.sort() print(merge(mylist1,mylist2))

四、對人人網(wǎng)的數(shù)據(jù)文件進行歸并排序

1.直接對人人網(wǎng)數(shù)據(jù)排序

renrenpath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/xh-2.txt" renrenfile=open(renrenpath,"rb") renrenlist=renrenfile.readlines() renrenfile.close() print("read") renrenlist.sort(key=lambda x:x.decode("gbk","ignore")) #別忘了解碼，要不然你看不了 print("sort")renrensortpath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/xhsort-2.txt" renrensortfile=open(renrensortpath,"wb") for line in renrenlist:renrensortfile.write(line) renrensortfile.close() print("save")

2.將人人網(wǎng)數(shù)據(jù)切割

#已知該文件有4768600行，我們把它切割成兩個文件，一個是2000000行，一個是2768600filelineslist=[2000000,2768600] filepath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/xh-2.txt" filedir="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/" allfile=open(filepath,"rb") for i in range(len(filelineslist)):tmpfilepath=filedir+"/renren____"+str(i)+".txt"tmpfile=open(tmpfilepath,"wb")for j in range(filelineslist[i]):line=allfile.readline()tmpfile.write(line)tmpfile.close() allfile.close()

3.將切割后兩個數(shù)據(jù)文件進行內(nèi)部排序

renrenpath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren____0.txt" renrenfile=open(renrenpath,"rb") renrenlist=renrenfile.readlines() renrenfile.close() print("read") renrenlist.sort(key=lambda x:x.decode("gbk","ignore")) #別忘了解碼，要不然你看不了 print("sort")renrensortpath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren____0sort.txt" renrensortfile=open(renrensortpath,"wb") for line in renrenlist:renrensortfile.write(line) renrensortfile.close() print("save")renrenpath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren____1.txt" renrenfile=open(renrenpath,"rb") renrenlist=renrenfile.readlines() renrenfile.close() print("read") renrenlist.sort(key=lambda x:x.decode("gbk","ignore")) #別忘了解碼，要不然你看不了 print("sort")renrensortpath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren____1sort.txt" renrensortfile=open(renrensortpath,"wb") for line in renrenlist:renrensortfile.write(line) renrensortfile.close() print("save")

4.將切割后的兩個文件進行歸并排序

#renren____0sort.txt 有：2000000行 #renren____0sort.txt 有：2768600行path1="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren____0sort.txt" path2="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren____1sort.txt" mergepath="E:/python數(shù)據(jù)分析/百度網(wǎng)盤/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/Python數(shù)據(jù)分析海量數(shù)據(jù)營銷day4/renren500w_16610/renren_Merge_sort.txt" file1=open(path1,"rb") file2=open(path2,"rb") filelast=open(mergepath,"wb") file1length=2000000 file2length=2768600i=0 j=0 str1=file1.readline() #讀一行 str2=file2.readline() #讀一行 linestr1=str1.decode("gbk","ignore") linestr2=str2.decode("gbk","ignore") while i<file1length and j<file2length:if linestr1<linestr2:filelast.write(str1)str1=file1.readline()linestr1=str1.decode("gbk","ignore")i+=1elif linestr1>linestr2:filelast.write(str2)str2 = file2.readline()linestr2 = str2.decode("gbk", "ignore")j+=1else: #（當linestr1=linestr2時，把兩個都寫進去）filelast.write(str1)str1 = file1.readline()linestr1 = str1.decode("gbk", "ignore")i += 1filelast.write(str2)str2 = file2.readline()linestr2 = str2.decode("gbk", "ignore")j += 1 #當有一個文件已經(jīng)遍歷完的時候（i=file1length or j=file2length）時就執(zhí)行下面這個程序 while i <file1length: #（當file2已經(jīng)遍歷完而file1還沒有遍歷完）filelast.write(str1)str1=file1.readline()i+=1while j <file2length:filelast.write(str2)str2=file2.readline()j+=1file1.close() file2.close() filelast.close()

四、對QQ數(shù)據(jù)文件進行歸并排序

之前歸并的是兩個文件，如果要是有10個文件，你怎么歸并效率最高呢？

1.如何做到兩兩歸并
我們要把列表里的第一個數(shù)還有第二個數(shù)合并放到最后一位

def getlast(mylist):if len(mylist)<2: #長度小于二說明已經(jīng)只剩下1個數(shù)了print(mylist)return mylistelse:mylist.append(mylist[0]+mylist[1])del mylist[0] #刪除第一個del mylist[0] #刪除原來的第二個，刪除原來的第一個后原來的第二個就成了現(xiàn)在的第一個文件了print(mylist)getlast(mylist)mylist=['1','2','3','4','5','6','7','8','9','0'] getlast(mylist)

2.批量兩兩合并

def getlastX(mylist):if len(mylist)<2: #長度小于二說明已經(jīng)只剩下1個數(shù)據(jù)了print(mylist)return mylistelse:n=len(mylist)//2 #計算出我們要合并幾對，10個那就合并5對，5個那就合并2對，3個合并1對for i in range(n): #將第0個和第1個合并，將第2個和第3個合并，將第4個和第5個合并mylist.append(mylist[2*i]+mylist[2*i+1])for i in range(n):del mylist[0] #刪除第一個del mylist[0] #刪除原來的第二個，刪除原來的第一個后原來的第二個就成了現(xiàn)在的第一個文件了print(mylist)getlastX(mylist) mylist=['1','2','3','4','5','6','7','8','9','0'] getlastX(mylist)

總結(jié)

以上是生活随笔為你收集整理的大文件的切割与归并（数据分析案例三）的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。