Experiment failure during 'test' - Could not infer dtype of NoneType

[mmartin9684-sil]

An experiment is preprocessing and training successfully, but at the start of the test step, it fails with the error 'Could not infer dtype of NoneType'.

The experiment config is set up with train and test set only - no validation / early stopping.

A separate experiment with all available verse pairs used as training data runs successfully and results in credible book drafts.

 0% 0/150 [00:00<?, ?ex/s]
Traceback (most recent call last):
  File "/usr/lib/python3.10/runpy.py", line 196, in _run_module_as_main
    return _run_code(code, main_globals, None,
  File "/usr/lib/python3.10/runpy.py", line 86, in _run_code
    exec(code, run_globals)
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/experiment.py", line 222, in <module>
    main()
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/experiment.py", line 218, in main
    exp.run()
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/experiment.py", line 48, in run
    self.test()
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/experiment.py", line 82, in test
    test(
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/test.py", line 620, in test
    results[step] = test_checkpoint(
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/test.py", line 417, in test_checkpoint
    model.translate_test_files(
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/hugging_face_config.py", line 1131, in translate_test_files
    list(
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/hugging_face_config.py", line 1168, in _translate_test_sentences
    for output_group in tqdm(
  File "/usr/local/lib/python3.10/dist-packages/tqdm/std.py", line 1181, in __iter__
    for obj in iterable:
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/hugging_face_config.py", line 1434, in _translate_sentences
    yield from self._translate_sentence_helper(
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/hugging_face_config.py", line 1542, in _translate_sentence_helper
    for translated_sentence in self._translate_with_beam_search(
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/hugging_face_config.py", line 1582, in _translate_with_beam_search
    translations = [[t] for t in translations]
  File "/root/.clearml/venvs-builds/3.10/task_repository/silnlp.git/silnlp/nmt/hugging_face_config.py", line 1582, in <listcomp>
    translations = [[t] for t in translations]
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/pt_utils.py", line 124, in __next__
    item = next(self.iterator)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/pt_utils.py", line 125, in __next__
    processed = self.infer(item, **self.params)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/base.py", line 1209, in forward
    model_outputs = self._forward(model_inputs, **forward_params)
  File "/usr/local/lib/python3.10/dist-packages/transformers/pipelines/text2text_generation.py", line 196, in _forward
    output_ids = self.model.generate(**model_inputs, **generate_kwargs)
  File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py", line 2246, in generate
    result = self._beam_search(
  File "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py", line 3476, in _beam_search
    next_token_scores_processed = logits_processor(input_ids, next_token_scores)
  File "/usr/local/lib/python3.10/dist-packages/transformers/generation/logits_process.py", line 104, in __call__
    scores = processor(input_ids, scores)
  File "/usr/local/lib/python3.10/dist-packages/transformers/generation/logits_process.py", line 1562, in __call__
    scores_processed[:, self.bos_token_id] = 0
RuntimeError: Could not infer dtype of NoneType