I usually rely on the following for most of my stuff:
Code:
import contextlib
import joblib
@contextlib.contextmanager
def tqdm_joblib(tqdm_object):
"""Context manager to patch joblib to report into tqdm progress bar given as argument"""
class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def __call__(self, *args, **kwargs):
tqdm_object.update(n=self.batch_size)
return super().__call__(*args, **kwargs)
old_batch_callback = joblib.parallel.BatchCompletionCallBack
joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
try:
yield tqdm_object
finally:
joblib.parallel.BatchCompletionCallBack = old_batch_callback
tqdm_object.close()
So instead of something like this:
Code:
class Checker:
def __init__(self) -> None:
DoSomething()
def Check(self, combo):
print(f'Checking {combo}')
for combo in combos:
Checker.Check(combo)
You would use this:
Code:
from joblib import Parallel, delayed
from tqdm import tqdm
class Checker:
def __init__(self) -> None:
DoSomething()
def Check(self, combo):
tqdm.write(f'Checking {combo}')
with tqdm_joblib(tqdm(desc="Checking", total=len(combos), unit='combo', colour='GREEN', leave=False)):
Parallel(n_jobs=20, require='sharedmem')(delayed(Checker.Check)(combo) for combo in combos)
It gives you a basic progress bar and executes whatever function you are needing in parallel. The only thing changed in your code would be instead of using print() for console output, you should use tqdm.write(), otherwise it messes with the progress bar.