Joblib parallel

Reason is the light and the light of life.

Jerry Su Mar 23, 2022 2 mins
from joblib import Parallel, delayed
import pandas as pd
def inference(node_id, biz_code, pool):
    df_to_detection = pd.DataFrame((node_id, biz_code, pool), columns=["node_id", "biz_code", "pool"])
    return df_to_detection
params = [(1, "a", "a_pool"),
          (2, "b", "b_pool"),
          (3, "c", "c_pool"),
          (4, "d", "d_pool"),
          (5, "e", "e_pool"),
          (6, "f", "f_pool"),
          (7, "g", "g_pool")]

jobs_num = len(params)

results = Parallel(n_jobs=jobs_num, backend="multiprocessing")(
    delayed(inference)(*param) for param in params)
---------------------------------------------------------------------------

RemoteTraceback                           Traceback (most recent call last)

RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/pandas/core/internals/managers.py", line 1675, in create_block_manager_from_blocks
    make_block(
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/pandas/core/internals/blocks.py", line 2711, in make_block
    return klass(values, ndim=ndim, placement=placement)
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/pandas/core/internals/blocks.py", line 142, in __init__
    raise ValueError(
ValueError: Wrong number of items passed 1, placement implies 3

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/conda/envs/blog/lib/python3.8/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/joblib/_parallel_backends.py", line 595, in __call__
    return self.func(*args, **kwargs)
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/joblib/parallel.py", line 262, in __call__
    return [func(*args, **kwargs)
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/joblib/parallel.py", line 262, in <listcomp>
    return [func(*args, **kwargs)
  File "/tmp/ipykernel_2910/2425433030.py", line 2, in inference
    df_to_detection = pd.DataFrame((node_id, biz_code, pool), columns=["node_id", "biz_code", "pool"])
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/pandas/core/frame.py", line 584, in __init__
    mgr = init_ndarray(data, index, columns, dtype=dtype, copy=copy)
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/pandas/core/internals/construction.py", line 238, in init_ndarray
    return create_block_manager_from_blocks(block_values, [columns, index])
  File "/opt/conda/envs/blog/lib/python3.8/site-packages/pandas/core/internals/managers.py", line 1687, in create_block_manager_from_blocks
    raise construction_error(tot_items, blocks[0].shape[1:], axes, e)
ValueError: Shape of passed values is (3, 1), indices imply (3, 3)
"""


The above exception was the direct cause of the following exception:


ValueError                                Traceback (most recent call last)

/tmp/ipykernel_2910/496635141.py in <module>
      9 jobs_num = len(params)
     10 
---> 11 results = Parallel(n_jobs=jobs_num, backend="multiprocessing")(
     12     delayed(inference)(*param) for param in params)


/opt/conda/envs/blog/lib/python3.8/site-packages/joblib/parallel.py in __call__(self, iterable)
   1052 
   1053             with self._backend.retrieval_context():
-> 1054                 self.retrieve()
   1055             # Make sure that we get a last message telling us we are done
   1056             elapsed_time = time.time() - self._start_time


/opt/conda/envs/blog/lib/python3.8/site-packages/joblib/parallel.py in retrieve(self)
    931             try:
    932                 if getattr(self._backend, 'supports_timeout', False):
--> 933                     self._output.extend(job.get(timeout=self.timeout))
    934                 else:
    935                     self._output.extend(job.get())


/opt/conda/envs/blog/lib/python3.8/multiprocessing/pool.py in get(self, timeout)
    769             return self._value
    770         else:
--> 771             raise self._value
    772 
    773     def _set(self, i, obj):


ValueError: Shape of passed values is (3, 1), indices imply (3, 3)


Read more:

Related posts: