# _*_coding:utf-8_*_

import time, threading, ConfigParser

'''

Reader類，繼承threading.Thread

@__init__方法初始化

@run方法實(shí)現(xiàn)了讀文件的操作

'''

class Reader(threading.Thread):

def __init__(self, file_name, start_pos, end_pos):

super(Reader, self).__init__()

self.file_name = file_name

self.start_pos = start_pos

self.end_pos = end_pos

def run(self):

fd = open(self.file_name, 'r')

'''

該if塊主要判斷分塊后的文件塊的首位置是不是行首，

是行首的話，不做處理

否則，將文件塊的首位置定位到下一行的行首

'''

if self.start_pos != 0:

fd.seek(self.start_pos-1)

if fd.read(1) != '\n':

line = fd.readline()

self.start_pos = fd.tell()

fd.seek(self.start_pos)

'''

對(duì)該文件塊進(jìn)行處理

'''

while (self.start_pos <= self.end_pos):

line = fd.readline()

'''

do somthing

'''

self.start_pos = fd.tell()

'''

對(duì)文件進(jìn)行分塊，文件塊的數(shù)量和線程數(shù)量一致

'''

class Partition(object):

def __init__(self, file_name, thread_num):

self.file_name = file_name

self.block_num = thread_num

def part(self):

fd = open(self.file_name, 'r')

fd.seek(0, 2)

pos_list = []

file_size = fd.tell()

block_size = file_size/self.block_num

start_pos = 0

for i in range(self.block_num):

if i == self.block_num-1:

end_pos = file_size-1

pos_list.append((start_pos, end_pos))

break

end_pos = start_pos+block_size-1

if end_pos >= file_size:

end_pos = file_size-1

if start_pos >= file_size:

break

pos_list.append((start_pos, end_pos))

start_pos = end_pos+1

fd.close()

return pos_list

if __name__ == '__main__':

'''

讀取配置文件

'''

config = ConfigParser.ConfigParser()

config.readfp(open('conf.ini'))

#文件名

file_name = config.get('info', 'fileName')

#線程數(shù)量

thread_num = int(config.get('info', 'threadNum'))

#起始時(shí)間

start_time = time.clock()

p = Partition(file_name, thread_num)

t = []

pos = p.part()

#生成線程

for i in range(thread_num):

t.append(Reader(file_name, *pos[i]))

#開啟線程

for i in range(thread_num):

t[i].start()

for i in range(thread_num):

t[i].join()

#結(jié)束時(shí)間

end_time = time.clock()

print "Cost time is %f" % (end_time - start_time)

總結(jié)

以上是生活随笔為你收集整理的python多线程处理文件_python多线程分块读取文件的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。