Merge branch '5-create-script-wrappers-to-run-notebooks-in-parallel' into 'main'

Resolve "create script wrappers to run notebooks in parallel"

Closes #5

See merge request !7

run-list-pickle-nb.sh

+#!/bin/bash
+#
+# run RS notebook with specific data set.
+#
+#
+# if we have gpus load cuda modules
+if [ "$CUDA_VISIBLE_DEVICES" != "" ]
+then
+  module load cuda11.3/toolkit
+fi
+module load Anaconda3/2021.11
+#module load Singularity
+
+#source /share/apps/rc/software/Anaconda3/2020.11/etc/profile.d/conda.sh
+conda  activate gpfs-policy
+
+# active openmp if multiple CPUs
+#export OMP_NUM_THREADS=${SLURM_NPROCS:-1}
+
+#./watchme.sh ${SLURM_TASK_PID} &
+
+#TAGNAME=slurm-${SLURM_JOBID}
+
+# add tmp dir to path for singularity support
+# this is where the interface to the kernel started in singularity is found
+#PATH=$PATH:$TMPDIR
+#echo $PATH
+
+NB="$3"
+jobid=${SLURM_ARRAY_JOB_ID:-$SLURM_JOB_ID}
+
+papermillcmd="papermill \
+	-p dirname $1 \
+	-p glob_pattern $2 \
+	-k ${NBKERNEL:-base} \
+	${NB}.ipynb \
+	slurm-${jobid}_${SLURM_ARRAY_TASK_ID}_${NB}.ipynb"
+
+echo $papermillcmd
+
+$papermillcmd
+
+pmreturn=$?
+
+# make notebook read only after papermill to avoid corrupting experiment record
+#chmod -w ${RESULTSDIR}${NB}_${CNAME}_${DATASET}_slurm-${SLURM_JOBID}.ipynb
+
+if [ $pmreturn -ne 0 ]
+then
+  echo "papermill failed"
+  exit 1
+fi
+

wrap-list-pickle.sh

+#!/bin/bash
+
+# call the list-pickle notebook runner with a array task wrapper to process specifc files
+
+fname=`printf "list-%.3d.gz" $SLURM_ARRAY_TASK_ID`
+
+./run-list-pickle-nb.sh $dirname "$fname" $NB