20-convert-to-hive.sh

#! /bin/bash
#
#SBATCH --job-name=hive-setup
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=16
#SBATCH --mem=90G
#SBATCH --partition=amperenodes-reserve
#SBATCH --time=02:00:00
#SBATCH --reservation=rc-gpfs
#SBATCH --gres=gpu:1
#SBATCH --output=out/hive-setup-%A-%a.out
#SBATCH --error=out/hive-setup-%A-%a.err
#SBATCH --array=0

module load Anaconda3
conda activate gpfs

device="data-project" # data-project, data-user, or scratch
parquets=($(find /data/rc/gpfs-policy/data -path "*/list-policy_${device}_list-path-external_slurm-*2025-01-21*/parquet"))
pq=${parquets[${SLURM_ARRAY_TASK_ID}]}

convert-to-hive --batch \
    --reservation=rc-gpfs \
    --partition=amperenodes-reserve \
    --mem=120G \
    ${pq} \
    /data/rc/gpfs-policy/data/gpfs-hive/${device}