run-mmpol.sh

#!/bin/bash

set -euxo pipefail

# run an mmapply policy across the cluster via slurm

############################################################
# Default Values                                           #
############################################################

outdir="/data/rc/gpfs-policy/data"
policy_file="./policy-def/list-path-external"
output_log_prefix=""
dry_run=false

############################################################
# Help                                                     #
############################################################

usage()
{
>&2 cat << EOF
Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file] device
EOF
exit 1
}

help()
{
>&2 cat << EOF
Runs mmapplypolicy on the specified device/fileset. The policy file dictates the actions performed including list, delete, add, etc. This is most often called by the submit-pol-job wrapper instead of invoked directly

Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file ] device

options:
    -h|--help           Print this Help.
    --dry-run           Do not run the policy command or any command that modifies any files or directories such as 
                            mkdir

Required:
    device              GPFS fileset/directory apply the policy to. Can be 
                            specified as either the name of the fileset or the 
                            full path to the directory 
                            (Examples: scratch, /data/user/[username])

Path:   
    -o|--outdir         Parent directory to save policy output to 
                            (default: /data/rc/gpfs-policy/data)
    -f|--output-prefix  Prefix of the policy output file. Appended with a metadata string containing the policy name, 
                            job ID, and date

Policy Options:
    -P|--policy-file    Path to policy file to apply to the given GPFS device
EOF
exit 0
}

args=$(getopt -a -o ho:f:P: --long help,outdir:,output-prefix:,policy-file:,dry-run -- "$@")

if [[ $? -gt 0 ]]; then
  usage
fi

eval set -- ${args}

while :
do
  case $1 in
    -h | --help)            help                    ;;
    -o | --outdir)          outdir=$2               ; shift 2 ;;
    -f | --output-prefix)   output_log_prefix=$2    ; shift 2 ;;
    -P | --policy-file)     policy_file=$2          ; shift 2 ;;
    --dry-run)              dry_run=true            ; shift 1 ;;
    --) shift; break ;;
    *) >&2 echo Unsupported option: $1
       usage ;;
  esac
done

if [[ $# -eq 0 ]]; then
  usage
fi

device="$1"

# Ensure device is specified
if [[ -z "${device}" ]]; then
    echo "Error: Specify either the name of a fileset or a directory path"
    usage
fi

# set default output_log_prefix if not specified in the arguments
if [[ -z "${output_log_prefix}" ]]; then
    modified_device=$(echo "${device}" | sed -e 's|^/||' -e 's|/$||' -e 's|/|-|g')
    output_log_prefix="list-policy_${modified_device}"
fi

# create temporary working directory for list aggregation
tmpglobal="${outdir}/slurm-tmp-${SLURM_JOBID}"
tmpscratch="${outdir}/slurm-tmp-${SLURM_JOBID}"

nodes=$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//')
cores="${SLURM_CPUS_PER_TASK}"

DATESTR=$(date +'%Y-%m-%dT%H:%M:%S')

policy=$(basename ${policy_file})
filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}"

cmd="mmapplypolicy ${device} -I defer \
  -P ${policy_file} \
  -g ${tmpglobal} \
  -s ${tmpscratch} \
  -f ${outdir}/list-${SLURM_JOBID} \
  -M FILEPATH=${device} \
  -M JOBID=${SLURM_JOBID} \
  -M LIST_OUTPUT_FILE=${output_log_prefix} \
  -N ${nodes} -n ${cores} -m ${cores}"

if [[ ! ${dry_run} ]]; then
    mkdir -p ${tmpglobal}
    
    # run policy command
    ${cmd}

    log_name="${output_log_prefix}_${filetag}"
    log_dir="${outdir}/${log_name}"

    mkdir -p ${log_dir}/raw
    chmod 1770 ${log_dir}

    # tag output file with run metadata
    raw_log_file=$(find ${outdir} -maxdepth 1 -name "list-${SLURM_JOBID}*" -type f | head -1)
    if [[ "$raw_log_file" != "" ]]; then
        mv -n ${raw_log_file} ${log_dir}/raw/${log_name}
        gzip ${log_dir}/raw/${log_name}

        chmod 440 ${log_dir}/raw/${log_name}.gz
        chmod 550 ${log_dir}/raw
    fi

    chown -R ${USER}:atlab ${log_dir}
    
    rmdir ${tmpglobal}
fi