ReturnnTrainingJob torchrun, use --standalone for single node (#462)

Fix #459
rwth-i6 · Nov 27, 2023 · 3d0dd86 · 3d0dd86
1 parent 80bcfb2
commit 3d0dd86
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/returnn/training.py b/returnn/training.py
@@ -250,11 +250,14 @@ def _get_run_cmd(self):
         if self.horovod_num_processes:
             if self.distributed_launch_cmd == "torchrun":
                 # use torchrun to lauch DDP training when the backend is torch
-                run_cmd = [
-                    "torchrun",
+                prefix = ["torchrun"]
+                if (self.multi_node_slots or 1) == 1:
+                    prefix += ["--standalone"]
+                prefix += [
                     f"--nnodes={self.multi_node_slots or 1}",
                     f"--nproc-per-node={self.horovod_num_processes}",
-                ] + run_cmd[1:]
+                ]
+                run_cmd = prefix + run_cmd[1:]
             elif self.distributed_launch_cmd == "mpirun":
                 # Normally, if the engine (e.g. SGE or Slurm) is configured correctly,
                 # it automatically provides the information on multiple nodes to mpirun,