Add random delay to nested sbatch submissions

src/rc_gpfs/cli/convert_flat_to_hive.py

 import argparse
 import subprocess
+import time
+import random
 from pathlib import Path
 import polars as pl
 
@@ -76,6 +78,12 @@ def submit_batch(**kwargs):
 
     script = f"#!/bin/bash\n#\n{slurm_opts}\n{BATCH_CMDS.format(**kwargs)}"
 
+    # Wait between 1 and 5 seconds before batch submission. This helps avoid a situation where this setup is running in
+    # a batch array job and all of the array tasks submit their child array jobs at the same time. That results in jobs
+    # failing to be submitted due to overwhelming the scheduler with simultaneous requests. Adding a random delay should
+    # fix that
+    time.sleep(random.uniform(1,5))
+
     subprocess.run(['sbatch'],input=script,shell=True,text=True)
     pass
 

src/rc_gpfs/cli/convert_to_parquet.py

 import argparse
-import re
+import time
+import random
 import subprocess
 from pathlib import Path
 import multiprocessing
@@ -71,6 +72,12 @@ def submit_batch(**kwargs):
     
     script = BATCH_SCRIPT.format(**kwargs)
 
+    # Wait between 1 and 5 seconds before batch submission. This helps avoid a situation where this setup is running in
+    # a batch array job and all of the array tasks submit their child array jobs at the same time. That results in jobs
+    # failing to be submitted due to overwhelming the scheduler with simultaneous requests. Adding a random delay should
+    # fix that
+    time.sleep(random.uniform(1, 5))
+
     subprocess.run(['sbatch'],input=script,shell=True,text=True)
     pass