[Python] multiprocess 패키지를 이용한 병렬처리

import time
import os
from multiprocess import Pool
import multiprocess as mp

import multiprocess as mp
mp.cpu_count()

16

1. multiprocess package intro

import time, os

def work_func(x):
    print("value %s is in PID : %s" % (x, os.getpid()))
    time.sleep(1)
    return x**5

def main():
    start = int(time.time())
    print(list(map(work_func, range(0,12))))
    print("***run time(sec) :", int(time.time()) - start)

if __name__ == "__main__":
    main()

value 0 is in PID : 22008
value 1 is in PID : 22008
value 2 is in PID : 22008
value 3 is in PID : 22008
value 4 is in PID : 22008
value 5 is in PID : 22008
value 6 is in PID : 22008
value 7 is in PID : 22008
value 8 is in PID : 22008
value 9 is in PID : 22008
value 10 is in PID : 22008
value 11 is in PID : 22008
[0, 1, 32, 243, 1024, 3125, 7776, 16807, 32768, 59049, 100000, 161051]
***run time(sec) : 12

def work_func(x):
    import time, os

    print("value %s is in PID : %s" % (x, os.getpid()))
    time.sleep(1)
    return x**5

def main():
    start = int(time.time())
    num_cores = 12
    pool = Pool(num_cores)
    print(pool.map(work_func, range(1,13)))
    print("***run time(sec) :", int(time.time()) - start)

if __name__ == "__main__":
    main()

[1, 32, 243, 1024, 3125, 7776, 16807, 32768, 59049, 100000, 161051, 248832]
***run time(sec) : 2

2. Apply to DataFrame

import time, os
import numpy as np
import pandas as pd
from multiprocess import Pool

def df_work_func(data):
    import time, os
    
    print('PID :', os.getpid())
    data['length_str'] = data['Name'].apply(lambda x : len(x))
    return data

def parallel_dataframe(df, func, num_cores):
    df_split = np.array_split(df, num_cores)
    pool = Pool(num_cores)
    df = pd.concat(pool.map(func, df_split))
    pool.close()
    pool.join()
    return df

start = int(time.time())
df = pd.read_csv("train.csv", dtype=str)
num_cores = 8
df = parallel_dataframe(df, df_work_func, num_cores)
print("***run time(sec) :", int(time.time()) - start)

df

***run time(sec) : 3

	PassengerId	Survived	Pclass	Name	Sex	Age	SibSp	Parch	Ticket	Fare	Cabin	Embarked	length_str
0	1	0	3	Braund, Mr. Owen Harris	male	22	1	0	A/5 21171	7.25	NaN	S	23
1	2	1	1	Cumings, Mrs. John Bradley (Florence Briggs Th...	female	38	1	0	PC 17599	71.2833	C85	C	51
2	3	1	3	Heikkinen, Miss. Laina	female	26	0	0	STON/O2. 3101282	7.925	NaN	S	22
3	4	1	1	Futrelle, Mrs. Jacques Heath (Lily May Peel)	female	35	1	0	113803	53.1	C123	S	44
4	5	0	3	Allen, Mr. William Henry	male	35	0	0	373450	8.05	NaN	S	24
...	...	...	...	...	...	...	...	...	...	...	...	...	...
886	887	0	2	Montvila, Rev. Juozas	male	27	0	0	211536	13	NaN	S	21
887	888	1	1	Graham, Miss. Margaret Edith	female	19	0	0	112053	30	B42	S	28
888	889	0	3	Johnston, Miss. Catherine Helen "Carrie"	female	NaN	1	2	W./C. 6607	23.45	NaN	S	40
889	890	1	1	Behr, Mr. Karl Howell	male	26	0	0	111369	30	C148	C	21
890	891	0	3	Dooley, Mr. Patrick	male	32	0	0	370376	7.75	NaN	Q	19

891 rows × 13 columns