ClientError with PutObject operation

[zhihaozheng]

multiprocess.pool.RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/multiprocess/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/pathos/helpers/mp_helper.py", line 15, in <lambda>
    func = lambda args: f(*args)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tem2ng/__init__.py", line 127, in process
    vol[bbx] = img
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/frontends/precomputed.py", line 826, in __setitem__
    self.image.upload(img, bbox.minpt, self.mip, parallel=self.parallel)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/__init__.py", line 53, in guardfn
    return fn(self, *args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/__init__.py", line 259, in upload
    return tx.upload(
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/tx.py", line 95, in upload
    upload_aligned(
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/tx.py", line 169, in upload_aligned
    threaded_upload_chunks(
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/tx.py", line 361, in threaded_upload_chunks
    schedule_jobs(
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/scheduler.py", line 104, in schedule_jobs
    return schedule_threaded_jobs(fns, concurrency, progress, total)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/scheduler.py", line 30, in schedule_threaded_jobs
    tq.put(updatefn(fn))
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/threaded_queue.py", line 257, in __exit__
    self.wait(progress=self.with_progress)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/threaded_queue.py", line 234, in wait
    self._check_errors()
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/threaded_queue.py", line 191, in _check_errors
    raise err
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/threaded_queue.py", line 153, in _consume_queue
    self._consume_queue_execution(fn)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/threaded_queue.py", line 180, in _consume_queue_execution
    fn()
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/scheduler.py", line 23, in realupdatefn
    res = fn()
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/tx.py", line 356, in process_and_update
    process(*args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/tx.py", line 353, in process
    do_upload(imgchunk, cloudpath)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudvolume/datasource/precomputed/image/tx.py", line 303, in do_upload
    remote.put(
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudfiles/cloudfiles.py", line 580, in put
    return self.puts({
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudfiles/cloudfiles.py", line 98, in inner_decor
    return fn(*args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudfiles/cloudfiles.py", line 544, in puts
    uploadfn(first(files))
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudfiles/cloudfiles.py", line 527, in uploadfn
    conn.put_file(
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tenacity/__init__.py", line 324, in wrapped_f
    return self(f, *args, **kw)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tenacity/__init__.py", line 404, in __call__
    do = self.iter(retry_state=retry_state)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tenacity/__init__.py", line 360, in iter
    raise retry_exc.reraise()
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tenacity/__init__.py", line 193, in reraise
    raise self.last_attempt.result()
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/concurrent/futures/_base.py", line 437, in result
    return self.__get_result()
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/concurrent/futures/_base.py", line 389, in __get_result
    raise self._exception
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tenacity/__init__.py", line 407, in __call__
    result = fn(*args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/cloudfiles/interfaces.py", line 721, in put_file
    self._conn.put_object(**attrs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/botocore/client.py", line 386, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/botocore/client.py", line 705, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (InternalError) when calling the PutObject operation (reached max retries: 4): We encountered an internal error. Please try again.
"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/bin/tem2ng", line 8, in <module>
    sys.exit(main())
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/click/core.py", line 1137, in __call__
    return self.main(*args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/click/core.py", line 1062, in main
    rv = self.invoke(ctx)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/click/core.py", line 1668, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/click/core.py", line 763, in invoke
    return __callback(*args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func
    return f(get_current_context(), *args, **kwargs)
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/tem2ng/__init__.py", line 135, in upload
    for num_inserted in pool.imap(process, to_upload):
  File "/home/voxa/miniconda3/envs/tem2ng_stage_grid/lib/python3.8/site-packages/multiprocess/pool.py", line 868, in next
    raise value
botocore.exceptions.ClientError: An error occurred (InternalError) when calling the PutObject operation (reached max retries: 4): We encountered an internal error. Please try again.

[zhihaozheng]

Something like tem2ng -p 10 upload imap_test/ tigerdata://sseung-test1/ca3-alignment-temp/test/ --z 0 using https://github.com/seung-lab/tem2ng/tree/stitch_upload