Map na procesach
24.08.2009
Z braku lepszego zajęcia, napisałem funkcję map działającą na
procesach
import time import itertools # python 2.4 from processing import Queue, Process def procmap(target, args, proc_nr): """Processes powered map function Acts like build-in `map` function, but result is in random order. """ def _sub_procmap(target, args, resultbox): resultbox.put(map(target, args)) step = len(args) / proc_nr q = Queue() processes = [] for i in xrange(proc_nr): arg = itertools.islice(args, i * step, i * step + step) p = Process(target=_sub_procmap, args=[target, arg, q]) processes.append(p) for p in processes: p.start() for p in processes: p.join() result = [] while not q.empty(): result.extend(q.get()) return result def _test(): _test1() _test2() _test3() def _test1(): def double(value): return value * 2 data = range(10000) assert sorted(procmap(double, data, 10)) == map(double, data) print "test1 - ok" def _test2(): def double(value): return value * 2 data = range(10000) t = time.time() procmap(double, data, 20) print "test2 - procmap: %f sec" % (time.time() - t) t = time.time() map(double, data) print "test2 - map: %f sec" % (time.time() - t) def _test3(): def stress(value): while value < 100000: value += 1 data = range(1000) t = time.time() procmap(stress, data, 20) print "test3 - procmap: %f sec" % (time.time() - t) t = time.time() map(stress, data) print "test3 - map: %f sec" % (time.time() - t) if __name__ == '__main__': _test()
wyniki:
test1 - ok test2 - procmap: 0.067223 sec test2 - map: 0.007202 sec test3 - procmap: 14.432514 sec test3 - map: 26.298147 sec