Draft: Improvements to post processing workflow to make ad hoc use of scripts easier

Let caller control workflow parameters without having to edit the script.

Added array task protections for lazy callers who don't care if array size aligns with work tasks.

example-job-scripts/10-convert-logs.sh

@@ -13,7 +13,17 @@
 module load Anaconda3
 conda activate gpfs
 
-logs=($(find /data/rc/gpfs-policy/data -path "*/list-policy_data-project_list-path-external_slurm-*/chunks"))
+# listcmd env var sets the command to enumerate datasets to process
+# supports passing args during sbatch, e.g. listcmd="cat split-list" sbatch <thisscript>
+# note: maxdepth speeds execution of find by avoiding deep dirs
+listcmd=${listcmd:-find /data/rc/gpfs-policy/data -maxdepth 2 -path "*/list-policy_data-project_list-path-external_slurm-*/chunks"}
+
+logs=($($listcmd))
 log=${logs[${SLURM_ARRAY_TASK_ID}]}
 
-convert-to-parquet --batch --no-clobber --partition=amd-hdr100,express,intel-dcb ${log}
+
\ No newline at end of file
+# for lazy submit. only do work if there is work to do
+if [ ${SLURM_ARRAY_TASK_ID} -lt ${#logs[@]} ]
+then
+    convert-to-parquet --batch --no-clobber --partition=amd-hdr100,express,intel-dcb ${log}
+fi