使用迭代的方式有助于减少内存使用,看情况使用。
import time
import numpy
from itertools import islice
step = 30000
big_list = range(1000000)
# 方式一
t = time.time()
for new_list in numpy.array_split(big_list, round(len(big_list)/ step)):
list(new_list)
print(new_list[-2]) # 测试结果
print(time.time() - t)
# 方式二
t = time.time()
def group_elements(lst, chunk_size):
lst = iter(lst)
return iter(lambda: tuple(islice(lst, chunk_size)), ())
for new_list in group_elements(big_list, step):
list(new_list)
print(new_list[-2]) # 测试结果
print(time.time() - t)
# 方式三
t = time.time()
def split_list(lst, n):
for i in range(0, len(lst), n):
yield lst[i:i + n]
for new_list in split_list(big_list, step):
list(new_list)
print(new_list[-2]) # 测试结果
print(time.time() - t)
# 方式四
t = time.time()
for i in range(0, len(big_list), step):
new_list = list(big_list[i:i + step])
print(new_list[-2]) # 测试结果
print(time.time() - t)
输出结果:
999998
0.21463561058044434
999998
0.03289985656738281
999998
0.047107696533203125
999998
0.022406339645385742
参考文章:
https://www.delftstack.com/howto/python/python-split-list-into-chunks/