diff --git a/run-list-pickle-nb.sh b/run-list-pickle-nb.sh new file mode 100755 index 0000000000000000000000000000000000000000..3ae2867bf5c57b765192f373a34cbe3bb3b43849 --- /dev/null +++ b/run-list-pickle-nb.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# run RS notebook with specific data set. +# +# +# if we have gpus load cuda modules +if [ "$CUDA_VISIBLE_DEVICES" != "" ] +then + module load cuda11.3/toolkit +fi +module load Anaconda3/2021.11 +#module load Singularity + +#source /share/apps/rc/software/Anaconda3/2020.11/etc/profile.d/conda.sh +conda activate gpfs-policy + +# active openmp if multiple CPUs +#export OMP_NUM_THREADS=${SLURM_NPROCS:-1} + +#./watchme.sh ${SLURM_TASK_PID} & + +#TAGNAME=slurm-${SLURM_JOBID} + +# add tmp dir to path for singularity support +# this is where the interface to the kernel started in singularity is found +#PATH=$PATH:$TMPDIR +#echo $PATH + +NB="$3" +jobid=${SLURM_ARRAY_JOB_ID:-$SLURM_JOB_ID} + +papermillcmd="papermill \ + -p dirname $1 \ + -p glob_pattern $2 \ + -k ${NBKERNEL:-base} \ + ${NB}.ipynb \ + slurm-${jobid}_${SLURM_ARRAY_TASK_ID}_${NB}.ipynb" + +echo $papermillcmd + +$papermillcmd + +pmreturn=$? + +# make notebook read only after papermill to avoid corrupting experiment record +#chmod -w ${RESULTSDIR}${NB}_${CNAME}_${DATASET}_slurm-${SLURM_JOBID}.ipynb + +if [ $pmreturn -ne 0 ] +then + echo "papermill failed" + exit 1 +fi + diff --git a/wrap-list-pickle.sh b/wrap-list-pickle.sh new file mode 100755 index 0000000000000000000000000000000000000000..45ad6ea5f16e1267498d463d776a8e484ee2de05 --- /dev/null +++ b/wrap-list-pickle.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# call the list-pickle notebook runner with a array task wrapper to process specifc files + +fname=`printf "list-%.3d.gz" $SLURM_ARRAY_TASK_ID` + +./run-list-pickle-nb.sh $dirname "$fname" $NB