代码1

import json
import time
import pandas as pddata_sets = pd.read_pickle("val_token_id.pandas_pickle")
data_sets=[str(i) for i in data_sets]
start=time.time()
[json.loads(i) for i in data_sets]
print(time.time()-start)start=time.time()
[eval(i) for i in data_sets]
print(time.time()-start)

结果图

代码2

import json
import time
from multiprocessing import Process, Manager, freeze_support
import pandas as pd
from tqdm import tqdm
def json_loads_data(return_list,one_data):return_list+=[json.loads(i)  for i in tqdm(one_data)]if __name__ == '__main__':freeze_support()data_sets = pd.read_pickle("val_token_id.pandas_pickle")data_sets = [str(i) for i in data_sets]start = time.time()data = Manager().list()num = 5p_list = []for i in range(0, len(data_sets), len(data_sets)//num):j = i + len(data_sets)//nump = Process(target=json_loads_data, args=(data, data_sets[i:j]))p.start()p_list.append(p)for p in p_list:p.join()print("multi_json_loads", time.time() - start)start = time.time()[json.loads(i) for i in data_sets]print("json_loads", time.time() - start)start = time.time()pd.DataFrame(data_sets)[0].apply(lambda x: json.loads(x)).values.tolist()print("dataFrame_apply", time.time() - start)start = time.time()json.loads(str(data_sets).replace("'", ""))print("json_loads_str", time.time() - start)start = time.time()[eval(i) for i in data_sets]print("eval", time.time() - start)

参考地址

https://blog.csdn.net/qq_35869630/article/details/105919104
Python 在大数据处理下的优化（一）用json.loads比eval快10倍！