Update main to rel_v0.1.1-1

Update main to current release. See !19 (merged) for more details

src/run-policy/run-mmpol.sh

@@ -4,43 +4,143 @@ set -euxo pipefail
 
 # run an mmapply policy across the cluster via slurm
 
-# gather info to map mmapplypolicy to runtime configuration
-# arguments passed via job env and runtime context
+############################################################
+# Default Values                                           #
+############################################################
 
-filesystem=${FILESYSTEM:-scratch}
-policyfile=$POLICYFILE
-tmpglobal=$DIR/slurm-tmp-${SLURM_JOBID}
-tmpscratch=$DIR/slurm-tmp-${SLURM_JOBID}
-mkdir -p $tmpglobal
+outdir="/data/rc/gpfs-policy/data"
+policy_file="./policy-def/list-path-external"
+output_log_prefix=""
+dry_run=false
 
-nodes=`scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//'`
+############################################################
+# Help                                                     #
+############################################################
+
+usage()
+{
+>&2 cat << EOF
+Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file] device
+EOF
+exit 1
+}
+
+help()
+{
+>&2 cat << EOF
+Runs mmapplypolicy on the specified device/fileset. The policy file dictates the actions performed including list, delete, add, etc. This is most often called by the submit-pol-job wrapper instead of invoked directly
+
+Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file ] device
+
+options:
+    -h|--help           Print this Help.
+    --dry-run           Do not run the policy command or any command that modifies any files or directories such as 
+                            mkdir
+
+Required:
+    device              GPFS fileset/directory apply the policy to. Can be 
+                            specified as either the name of the fileset or the 
+                            full path to the directory 
+                            (Examples: scratch, /data/user/[username])
+
+Path:   
+    -o|--outdir         Parent directory to save policy output to 
+                            (default: /data/rc/gpfs-policy/data)
+    -f|--output-prefix  Prefix of the policy output file. Appended with a metadata string containing the policy name, 
+                            job ID, and date
+
+Policy Options:
+    -P|--policy-file    Path to policy file to apply to the given GPFS device
+EOF
+exit 0
+}
+
+args=$(getopt -a -o ho:f:P: --long help,outdir:,output-prefix:,policy-file:,dry-run -- "$@")
+
+if [[ $? -gt 0 ]]; then
+  usage
+fi
+
+eval set -- ${args}
+
+while :
+do
+  case $1 in
+    -h | --help)            help                    ;;
+    -o | --outdir)          outdir=$2               ; shift 2 ;;
+    -f | --output-prefix)   output_log_prefix=$2    ; shift 2 ;;
+    -P | --policy-file)     policy_file=$2          ; shift 2 ;;
+    --dry-run)              dry_run=true            ; shift 1 ;;
+    --) shift; break ;;
+    *) >&2 echo Unsupported option: $1
+       usage ;;
+  esac
+done
+
+if [[ $# -eq 0 ]]; then
+  usage
+fi
+
+device="$1"
+
+# Ensure device is specified
+if [[ -z "${device}" ]]; then
+    echo "Error: Specify either the name of a fileset or a directory path"
+    usage
+fi
+
+# set default output_log_prefix if not specified in the arguments
+if [[ -z "${output_log_prefix}" ]]; then
+    modified_device=$(echo "${device}" | sed -e 's|^/||' -e 's|/$||' -e 's|/|-|g')
+    output_log_prefix="list-policy_${modified_device}"
+fi
+
+# create temporary working directory for list aggregation
+tmpglobal="${outdir}/slurm-tmp-${SLURM_JOBID}"
+tmpscratch="${outdir}/slurm-tmp-${SLURM_JOBID}"
+
+nodes=$(scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//')
 cores="${SLURM_CPUS_PER_TASK}"
 
-DATESTR=`date +'%Y-%m-%d-%H:%M:%S'`
+DATESTR=$(date +'%Y-%m-%dT%H:%M:%S')
 
-policy=`basename $policyfile`
+policy=$(basename ${policy_file})
 filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}"
 
-cmd="mmapplypolicy ${filesystem} -I defer \
-  -P $policyfile \
-  -g $tmpglobal \
-  -s $tmpscratch \
-  -f ${DIR}/list-${SLURM_JOBID} \
-  -M FILEPATH=${filesystem} \
+cmd="mmapplypolicy ${device} -I defer \
+  -P ${policy_file} \
+  -g ${tmpglobal} \
+  -s ${tmpscratch} \
+  -f ${outdir}/list-${SLURM_JOBID} \
+  -M FILEPATH=${device} \
   -M JOBID=${SLURM_JOBID} \
-  -M LIST_OUTPUT_FILE=${OUTFILE:-/tmp/gpfs-list-policy}
+  -M LIST_OUTPUT_FILE=${output_log_prefix} \
   -N ${nodes} -n ${cores} -m ${cores}"
 
-# report final command in job log
-echo $cmd
+if [[ ! ${dry_run} ]]; then
+    mkdir -p ${tmpglobal}
+    
+    # run policy command
+    ${cmd}
+
+    log_name="${output_log_prefix}_${filetag}"
+    log_dir="${outdir}/${log_name}"
 
-# run policy command
-$cmd
+    mkdir -p ${log_dir}/raw
+    chmod 1770 ${log_dir}
 
-# tag output file with run metadata
-outfile=`ls -t $tmpglobal | head -1`
-if [[ "$outfile" != "" ]]
-then
-   mv -n $tmpglobal/$outfile $tmpglobal/../${outfile}_$filetag
+    # tag output file with run metadata
+    raw_log_file=$(find ${outdir} -maxdepth 1 -name "list-${SLURM_JOBID}*" -type f | head -1)
+    if [[ "$raw_log_file" != "" ]]; then
+        mv -n ${raw_log_file} ${log_dir}/raw/${log_name}
+        gzip ${log_dir}/raw/${log_name}
+
+        chmod 440 ${log_dir}/raw/${log_name}.gz
+        chmod 550 ${log_dir}/raw
+    fi
+
+    chown -R ${USER}:atlab ${log_dir}
+    
+    rmdir ${tmpglobal}
 fi
-rmdir $tmpglobal
+