{"id":3286,"date":"2017-04-18T21:51:33","date_gmt":"2017-04-18T13:51:33","guid":{"rendered":"https:\/\/ixyzero.com\/blog\/?p=3286"},"modified":"2017-04-18T21:51:33","modified_gmt":"2017-04-18T13:51:33","slug":"python%e4%b8%ad%e7%9a%84%e5%a4%9a%e8%bf%9b%e7%a8%8b%e7%bc%96%e7%a8%8b","status":"publish","type":"post","link":"https:\/\/ixyzero.com\/blog\/archives\/3286.html","title":{"rendered":"Python\u4e2d\u7684\u591a\u8fdb\u7a0b\u7f16\u7a0b"},"content":{"rendered":"

=Start=<\/p>\n

\u7f18\u7531\uff1a<\/h4>\n

\u4e4b\u524d\u7684Python\u7f16\u7a0b\u4efb\u52a1\u4e2d\u4e00\u822c\u90fd\u662f\u5355\u7ebf\u7a0b\u8fdb\u884c\u7684\uff0c\u56e0\u4e3a\u5927\u591a\u6570\u5c5e\u4e8eCPU\u5bc6\u96c6\u578b\uff0c\u591a\u7ebf\u7a0b\u6216\u591a\u8fdb\u7a0b\u5bf9\u7a0b\u5e8f\u6027\u80fd\u7684\u63d0\u5347\u4e0d\u5927\uff1b\u4f46\u6700\u8fd1\u6709\u51e0\u4e2a\u7f51\u7edc\u5bc6\u96c6\u578b\u7684\u4efb\u52a1\uff0c\u5355\u7ebf\u7a0b\u7684\u7248\u672c\u660e\u663e\u592a\u6162\uff0c\u4e0d\u5f97\u5df2\uff0c\u5f00\u59cb\u5b66\u4e60Python\u7684\u591a\u8fdb\u7a0b\u7f16\u7a0b\u3002<\/p>\n

\u6b63\u6587\uff1a<\/h4>\n

\u53c2\u8003\u89e3\u7b54\uff1a<\/h5>\n
1.\u521b\u5efa\u8fdb\u7a0b\u6c60Pool<\/h6>\n

multiprocessing \u662f Python \u7684\u591a\u8fdb\u7a0b\u5e76\u884c\u5e93\uff0c\u6211\u4f7f\u7528\u8fdb\u7a0b\u6c60 multiprocessing.Pool \u6765\u81ea\u52a8\u7ba1\u7406\u8fdb\u7a0b\u4efb\u52a1\u3002\u53ef\u4ee5\u901a\u8fc7\u4e00\u4e0b\u8bed\u53e5\u521d\u59cb\u5316 Pool\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
multiprocessing.freeze_support() # Windows \u5e73\u53f0\u8981\u52a0\u4e0a\u8fd9\u53e5\uff0c\u907f\u514d RuntimeError<\/code><\/div>\n
pool = multiprocessing.Pool()<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n

\u5047\u8bbe\u6211\u4eec\u8981\u5e76\u884c\u6267\u884c\u7684\u4efb\u52a1\u662f\u4ee5\u4e0b\u51fd\u6570\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
def task(pid):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code># <\/code>do<\/code> something<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>return<\/code> result<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n

\u7136\u540e\u5728\u4e3b\u51fd\u6570\u8c03\u7528\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
results = []<\/code><\/div>\n
for<\/code> i in xrange(<\/code>0<\/code>, <\/code>4<\/code>):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>result = pool.apply_async(task, args=(i,)) #\u8fd9\u884ci\u540e\u9762\u7684\u9017\u53f7\u662f\u4e0d\u80fd\u7701\u7565\u7684\uff0c\u5426\u5219\u4e0d\u4f1a\u6267\u884ctask\u65b9\u6cd5<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>results.append(result)<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n

\u4e0a\u9762\u7684 pool.apply_async \u91c7\u7528\u5f02\u6b65\u65b9\u5f0f\u8c03\u7528 task\uff0c\u800c pool.apply \u5219\u662f\u540c\u6b65\u65b9\u5f0f\u8c03\u7528\u3002\u540c\u6b65\u65b9\u5f0f\u610f\u5473\u7740\u4e0b\u4e00\u4e2a task \u9700\u8981\u7b49\u5f85\u4e0a\u4e00\u4e2a task \u5b8c\u6210\u540e\u624d\u80fd\u5f00\u59cb\u8fd0\u884c\uff0c\u8fd9\u663e\u7136\u4e0d\u662f\u6211\u4eec\u60f3\u8981\u7684\u529f\u80fd\uff0c\u6240\u4ee5\u91c7\u7528\u5f02\u6b65\u65b9\u5f0f\u8fde\u7eed\u5730\u63d0\u4ea4\u4efb\u52a1\u3002\u5728\u4e0a\u9762\u7684\u8bed\u53e5\u4e2d\uff0c\u6211\u4eec\u63d0\u4ea4\u4e86 4 \u4e2a\u4efb\u52a1\uff0c\u5047\u8bbe\u6211\u7684 CPU \u662f 4 \u6838\uff0c\u90a3\u4e48\u6211\u7684\u6bcf\u4e2a\u6838\u8fd0\u884c\u4e00\u4e2a\u4efb\u52a1\u3002\u5982\u679c\u6211\u63d0\u4ea4\u591a\u4e8e 4 \u4e2a\u4efb\u52a1\uff0c\u90a3\u4e48\u6bcf\u4e2a\u6838\u5c31\u9700\u8981\u540c\u65f6\u8fd0\u884c 2 \u4e2a\u4ee5\u4e0a\u7684\u4efb\u52a1\uff0c\u8fd9\u4f1a\u5e26\u6765\u4efb\u52a1\u5207\u6362\u6210\u672c\uff0c\u964d\u4f4e\u4e86\u6548\u7387\uff08Pool\u8fdb\u7a0b\u6c60\u53ef\u4ee5\u63d0\u4f9b\u6307\u5b9a\u6570\u91cf\u7684\u8fdb\u7a0b\u4f9b\u7528\u6237\u8c03\u7528\uff0c\u5f53\u6709\u65b0\u7684\u8bf7\u6c42\u63d0\u4ea4\u5230Pool\u4e2d\u65f6\uff0c\u5982\u679c\u8fdb\u7a0b\u6c60\u8fd8\u6ca1\u6709\u6ee1\uff0c\u90a3\u4e48\u5c31\u4f1a\u521b\u5efa\u4e00\u4e2a\u65b0\u7684\u8fdb\u7a0b\u7528\u6765\u6267\u884c\u8be5\u8bf7\u6c42\uff1b\u4f46\u5982\u679c\u6c60\u4e2d\u7684\u8fdb\u7a0b\u6570\u5df2\u7ecf\u8fbe\u5230\u89c4\u5b9a\u6700\u5927\u503c\uff0c\u90a3\u4e48\u8be5\u8bf7\u6c42\u5c31\u4f1a\u7b49\u5f85\uff0c\u76f4\u5230\u6c60\u4e2d\u6709\u8fdb\u7a0b\u7ed3\u675f\uff0c\u624d\u4f1a\u521b\u5efa\u65b0\u7684\u8fdb\u7a0b\u6765\u5904\u7406\u5b83\uff09\u3002\u6240\u4ee5\u6211\u4eec\u8bbe\u7f6e\u7684\u5e76\u884c\u4efb\u52a1\u6570\u6700\u597d\u7b49\u4e8e CPU \u6838\u5fc3\u6570\uff0c CPU \u6838\u53ef\u4ee5\u901a\u8fc7\u4e0b\u9762\u8bed\u53e5\u5f97\u5230\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
cpus = multiprocessing.cpu_count()<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n

\u63a5\u4e0b\u6765\u6211\u4eec\u4f7f\u7528 result.get() \u6765\u83b7\u53d6 task \u7684\u8fd4\u56de\u503c\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
for<\/code> result in results:<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>print(result.get())<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n

\u5728\u8fd9\u91cc\u4e0d\u514d\u6709\u4eba\u8981\u7591\u95ee\uff0c\u4e3a\u4ec0\u4e48\u4e0d\u76f4\u63a5\u5728 for \u5faa\u73af\u4e2d\u76f4\u63a5 result.get() \u5462\uff1f\u8fd9\u662f\u56e0\u4e3apool.apply_async\u4e4b\u540e\u7684\u8bed\u53e5\u90fd\u662f\u963b\u585e\u6267\u884c\u7684\uff0c\u8c03\u7528 result.get() \u4f1a\u7b49\u5f85\u4e0a\u4e00\u4e2a\u4efb\u52a1\u6267\u884c\u5b8c\u4e4b\u540e\u624d\u4f1a\u5206\u914d\u4e0b\u4e00\u4e2a\u4efb\u52a1\u3002\u4e8b\u5b9e\u4e0a\uff0c\u83b7\u53d6\u8fd4\u56de\u503c\u7684\u8fc7\u7a0b\u6700\u597d\u653e\u5728\u8fdb\u7a0b\u6c60\u56de\u6536\u4e4b\u540e\u8fdb\u884c\uff0c\u907f\u514d\u963b\u585e\u540e\u9762\u7684\u8bed\u53e5\u3002<\/p>\n

\u6700\u540e\u6211\u4eec\u4f7f\u7528\u4e00\u4e0b\u8bed\u53e5\u56de\u6536\u8fdb\u7a0b\u6c60\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
pool.close() #\u8c03\u7528join\u51fd\u6570\u4e4b\u524d\uff0c\u5148\u8c03\u7528close\u51fd\u6570\uff0c\u5426\u5219\u4f1a\u51fa\u9519\u3002\u6267\u884c\u5b8cclose\u540e\u4e0d\u4f1a\u6709\u65b0\u7684\u8fdb\u7a0b\u52a0\u5165\u5230Pool\u3002<\/code><\/div>\n
pool.join()\u00a0 #\u8c03\u7528join\u51fd\u6570\u963b\u585e\u4e3b\u8fdb\u7a0b\u7b49\u5f85\u6240\u6709\u5b50\u8fdb\u7a0b\u7ed3\u675f<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n

\u6700\u540e\u9644\u4e0a\u5b8c\u6574\u7684\u4ee3\u7801\u5982\u4e0b\uff1a<\/p>\n

\n
\n
\n
\n\n\n\n
\n
\n
def task(pid):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code># <\/code>do<\/code> something<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>return<\/code> result<\/code><\/div>\n
<\/div>\n
def main():<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>multiprocessing.freeze_support()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool = multiprocessing.Pool()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>cpus = multiprocessing.cpu_count()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>results = []<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>for<\/code> i in xrange(<\/code>0<\/code>, cpus):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>result = pool.apply_async(task, args=(i,))<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>results.append(result)<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool.close()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool.join()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>for<\/code> result in results:<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>print(result.get())<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n
2\u3001\u4f7f\u7528\u8fdb\u7a0b\u6c60Pool\uff0c\u4e0d\u9700\u8981\u5173\u6ce8\u7ed3\u679c<\/h6>\n
\n
\n
\n
\n\n\n\n
\n
\n
import<\/code> multiprocessing<\/code><\/div>\n
import<\/code> time<\/code><\/div>\n
<\/div>\n
#\u53ea\u662f\u7b80\u5355\u7684print\u548csleep\uff0c\u5e76\u4e0d\u9700\u8981\u8fd4\u56de\u4ec0\u4e48\u7ed3\u679c\/\u5185\u5bb9<\/code><\/div>\n
def func(msg):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>for<\/code> i in xrange(<\/code>3<\/code>):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>print msg<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>time.sleep(<\/code>1<\/code>)<\/code><\/div>\n
<\/div>\n
if<\/code> __name__ == <\/code>\"__main__\"<\/code>:<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool = multiprocessing.Pool(processes=<\/code>4<\/code>)<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>for<\/code> i in xrange(<\/code>10<\/code>):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>msg = <\/code>\"hello %d\"<\/code> %(i)<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>pool.apply_async(func, (msg, ))<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool.close()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool.join()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>print <\/code>\"Sub-process(es) done.\"<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n
3\u3001\u4f7f\u7528\u8fdb\u7a0b\u6c60Pool\uff0c\u5e76\u9700\u8981\u5173\u6ce8\u7ed3\u679c\uff08\u901a\u8fc7callback\u7684\u65b9\u5f0f\uff09<\/h6>\n
\n
\n
\n
\n\n\n\n
\n
\n
import<\/code> multiprocessing as mp<\/code><\/div>\n
import<\/code> time<\/code><\/div>\n
<\/div>\n
def foo_pool(x):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>time.sleep(<\/code>2<\/code>)<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>return<\/code> x*x<\/code><\/div>\n
<\/div>\n
result_list = []<\/code><\/div>\n
def log_result(result):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code># This is called whenever foo_pool(i) returns a result.<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code># result_list is modified only by the main process, not the pool workers.<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>result_list.append(result)<\/code><\/div>\n
<\/div>\n
def apply_async_with_callback():<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool = mp.Pool()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>for<\/code> i in range(<\/code>10<\/code>):<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0<\/code>pool.apply_async(foo_pool, args = (i, ), callback = log_result)<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool.close()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>pool.join()<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>print(result_list)<\/code><\/div>\n
<\/div>\n
if<\/code> __name__ == <\/code>'__main__'<\/code>:<\/code><\/div>\n
\u00a0\u00a0\u00a0\u00a0<\/code>apply_async_with_callback()<\/code><\/div>\n<\/div>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n
\u53c2\u8003\u94fe\u63a5\uff1a<\/h5>\n