Failing to write final step to S3

[scharlottej13]

When running this on the cloud, in pipeline/reduce.py, dask-deltatable doesn't successfully write to S3 due to a TypeError("AioSession.__init__() got an unexpected keyword argument 'AWS_S3_ALLOW_UNSAFE_RENAME'").

My guess is AWS_S3_ALLOW_UNSAFE_RENAME was originally added in response to this issue: delta-io/delta-rs#883 (comment)

Full traceback:

File "/opt/coiled/env/lib/python3.11/site-packages/dask_expr/_expr.py", line 3758, in _execute_task
    return dask.core.get(graph, name)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/dask/core.py", line 157, in get
    result = _execute_task(task, cache)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/dask/core.py", line 127, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/dask/utils.py", line 78, in apply
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/dask_deltatable/core.py", line 76, in _read_delta_partition
    table = pa_ds.dataset(
            ^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/pyarrow/dataset.py", line 794, in dataset
    return _filesystem_dataset(source, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/pyarrow/dataset.py", line 476, in _filesystem_dataset
    fs, paths_or_selector = _ensure_single_source(source, filesystem)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/pyarrow/dataset.py", line 432, in _ensure_single_source
    file_info = filesystem.get_file_info(path)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "pyarrow/_fs.pyx", line 590, in pyarrow._fs.FileSystem.get_file_info
  File "pyarrow/error.pxi", line 155, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 89, in pyarrow.lib.check_status
  File "pyarrow/_fs.pyx", line 1498, in pyarrow._fs._cb_get_file_info
  File "/opt/coiled/env/lib/python3.11/site-packages/pyarrow/fs.py", line 322, in get_file_info
    info = self.fs.info(path)
           ^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/fsspec/asyn.py", line 118, in wrapper
    return sync(self.loop, func, *args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/fsspec/asyn.py", line 103, in sync
    raise return_result
  File "/opt/coiled/env/lib/python3.11/site-packages/fsspec/asyn.py", line 56, in _runner
    result[0] = await coro
                ^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/s3fs/core.py", line 1374, in _info
    out = await self._call_s3(
          ^^^^^^^^^^^^^^^^^^^^
  File "/opt/coiled/env/lib/python3.11/site-packages/s3fs/core.py", line 358, in _call_s3
    await self.set_session()
  File "/opt/coiled/env/lib/python3.11/site-packages/s3fs/core.py", line 519, in set_session
    self.session = aiobotocore.session.AioSession(**self.kwargs)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

TypeError("AioSession.__init__() got an unexpected keyword argument 'AWS_S3_ALLOW_UNSAFE_RENAME'")