首先,可以使用以下方法對孔對象的不同部分進(jìn)行pickle:

# gen_objects.py

import random

import pickle

class BigBadObject(object):

def __init__(self):

self.a_dictionary={}

for x in xrange(random.randint(1, 1000)):

self.a_dictionary[random.randint(1,98675676)]=random.random()

self.a_list=[]

for x in xrange(random.randint(1000, 10000)):

self.a_list.append(random.random())

self.a_string=''.join([chr(random.randint(65, 90))

for x in xrange(random.randint(100, 10000))])

if __name__=="__main__":

output=open('lotsa_objects.pickled', 'wb')

for i in xrange(10000):

pickle.dump(BigBadObject(), output, pickle.HIGHEST_PROTOCOL)

output.close()

一旦在不同的部分中生成了BigFile,就可以用一個(gè)python程序讀取它,同時(shí)運(yùn)行多個(gè)python程序讀取每個(gè)不同的部分。

# reader.py

from threading import Thread

from Queue import Queue, Empty

import cPickle as pickle

import time

import operator

from gen_objects import BigBadObject

class Reader(Thread):

def __init__(self, filename, q):

Thread.__init__(self, target=None)

self._file=open(filename, 'rb')

self._queue=q

def run(self):

while True:

try:

one_object=pickle.load(self._file)

except EOFError:

break

self._queue.put(one_object)

class uncached(object):

def __init__(self, filename, queue_size=100):

self._my_queue=Queue(maxsize=queue_size)

self._my_reader=Reader(filename, self._my_queue)

self._my_reader.start()

def __iter__(self):

while True:

if not self._my_reader.is_alive():

break

# Loop until we get something or the thread is done processing.

try:

print "Getting from the queue. Queue size=", self._my_queue.qsize()

o=self._my_queue.get(True, timeout=0.1) # Block for 0.1 seconds

yield o

except Empty:

pass

return

# Compute an average of all the numbers in a_lists, just for show.

list_avg=0.0

list_count=0

for x in uncached('lotsa_objects.pickled'):

list_avg+=reduce(operator.add, x.a_list)

list_count+=len(x.a_list)

print "Average: ", list_avg/list_count

這種讀取pickle文件的方式將占用另一種方式所需時(shí)間的1%。這是因?yàn)槟瑫r(shí)運(yùn)行100個(gè)并行線程。

總結(jié)

以上是生活随笔為你收集整理的python queue 调试_python:如何创建用于调试的持久内存结构的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯(cuò)，歡迎將生活随笔推薦給好友。