diff --git a/setup.py b/setup.py index 89778c14d41..e45ea3eef7c 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ # Minimum 8.0.0 to be able to use .to_reader() "pyarrow>=8.0.0", # For smart caching dataset processing - "dill>=0.3.0,<0.3.7", # tmp pin until next 0.3.7 release: see https://github.com/huggingface/datasets/pull/5166 + "dill>=0.3.0,<0.3.8", # tmp pin until dill has official support for determinism see https://github.com/uqfoundation/dill/issues/19 # For performance gains with apache arrow "pandas", # for downloading datasets over HTTPS diff --git a/src/datasets/utils/py_utils.py b/src/datasets/utils/py_utils.py index 3ee7e0f56d0..65bb0c2c357 100644 --- a/src/datasets/utils/py_utils.py +++ b/src/datasets/utils/py_utils.py @@ -606,7 +606,7 @@ def save(self, obj, save_persistent_id=True): def dill_log(pickler, msg): dill._dill.log.info(msg) - elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release: + elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]: def dill_log(pickler, msg): dill._dill.logger.trace(pickler, msg) @@ -826,7 +826,7 @@ def _save_code(pickler, obj): dill._dill.log.info("# Co") return -elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release: +elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]: # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104 @pklregister(CodeType) def save_code(pickler, obj): @@ -1193,7 +1193,7 @@ def save_function(pickler, obj): dill._dill.log.info("# F2") return -elif config.DILL_VERSION.release[:3] == version.parse("0.3.6").release: +elif config.DILL_VERSION.release[:3] in [version.parse("0.3.6").release, version.parse("0.3.7").release]: # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1739 @pklregister(FunctionType) def save_function(pickler, obj):