diff --git a/src/convert-to-parquet/run-convert-to-parquet.sh b/src/convert-to-parquet/run-convert-to-parquet.sh index d08a2db3c477a3d8514ff620a857ce876dd015bf..c729a08c32bf61d0c838f046f765490802d46386 100755 --- a/src/convert-to-parquet/run-convert-to-parquet.sh +++ b/src/convert-to-parquet/run-convert-to-parquet.sh @@ -11,7 +11,6 @@ mem="16G" time="02:00:00" partition="amd-hdr100" outdir="" -sif="gitlab.rc.uab.edu:4567/mdefende/gpfs-policy:latest" ############################################################ # Help # @@ -87,15 +86,12 @@ if [[ -z "$outdir" ]]; then outdir="${gpfs_logdir}/parquet" fi -singularity pull --force gpfs.sif docker://${sif} - nlogs=$(ls ${gpfs_logdir}/list-* | wc -l) -cmd="singularity exec --bind /data,/scratch gpfs.sif python3 convert-to-parquet.py -o ${outdir} -f \${log}" +cmd="python3 convert-to-parquet.py -o ${outdir} -f \${log}" >&2 cat << EOF -------------------------------------------------------------------------------- -sif: ${sif} output dir: ${outdir} GPFS logs: ${gpfs_logdir} @@ -127,6 +123,8 @@ mkdir -p err #SBATCH --error=err/%A_%a.err #SBATCH --array=1-${nlogs} +source /data/rc/gpfs-policy/venv/bin/activate + log=\$(ls ${gpfs_logdir}/list-* | awk "NR==\${SLURM_ARRAY_TASK_ID} { print \$1 }") ${cmd}