Skip to content
Snippets Groups Projects

Add random delay to nested sbatch submissions

2 files
+ 16
1
Compare changes
  • Side-by-side
  • Inline
Files
2
import argparse
import subprocess
import time
import random
from pathlib import Path
import polars as pl
@@ -76,6 +78,12 @@ def submit_batch(**kwargs):
script = f"#!/bin/bash\n#\n{slurm_opts}\n{BATCH_CMDS.format(**kwargs)}"
# Wait between 1 and 5 seconds before batch submission. This helps avoid a situation where this setup is running in
# a batch array job and all of the array tasks submit their child array jobs at the same time. That results in jobs
# failing to be submitted due to overwhelming the scheduler with simultaneous requests. Adding a random delay should
# fix that
time.sleep(random.uniform(1,5))
subprocess.run(['sbatch'],input=script,shell=True,text=True)
pass
Loading