Merge branch 'enh-add-getopt-to-run-mmpol' into 'rel_v0.1.1-1'

ENH: Add getopt to run-mmpol and remove reliance on inherited environment variables

See merge request !17

src/run-policy/run-mmpol.sh

@@ -4,13 +4,97 @@ set -euxo pipefail
 
 # run an mmapply policy across the cluster via slurm
 
-# gather info to map mmapplypolicy to runtime configuration
-# arguments passed via job env and runtime context
+############################################################
+# Default Values                                           #
+############################################################
 
-filesystem=${FILESYSTEM:-scratch}
-policyfile=$POLICYFILE
-tmpglobal=$DIR/slurm-tmp-${SLURM_JOBID}
-tmpscratch=$DIR/slurm-tmp-${SLURM_JOBID}
+outdir="/data/rc/gpfs-policy/data"
+policy_file="./policy-def/list-path-external"
+device="scratch"
+output_log_prefix=""
+
+############################################################
+# Help                                                     #
+############################################################
+
+usage()
+{
+>&2 cat << EOF
+Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [-P | --policy-file] device
+EOF
+exit 1
+}
+
+help()
+{
+>&2 cat << EOF
+Runs mmapplypolicy on the specified device/fileset. The policy file dictates the actions performed including list, delete, add, etc. This is most often called by the submit-pol-job wrapper instead of invoked directly
+
+Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file ] device
+
+options:
+    -h|--help       Print this Help.
+
+Required:
+    device              GPFS fileset/directory apply the policy to. Can be 
+                            specified as either the name of the fileset or the 
+                            full path to the directory 
+                            (Examples: scratch, /data/user/[username])
+
+Path:   
+    -o|--outdir         Parent directory to save policy output to 
+                            (default: /data/rc/gpfs-policy/data)
+    -f|--output-prefix  Prefix of the policy output file. Appended with a metadata string containing the policy name, 
+                            job ID, and date
+
+Policy Options:
+    -P|--policy-file    Path to policy file to apply to the given GPFS device
+EOF
+exit 0
+}
+
+args=$(getopt -a -o ho:f:P: --long help,outdir:,output-prefix:,policy-file: -- "$@")
+
+if [[ $? -gt 0 ]]; then
+  usage
+fi
+
+eval set -- ${args}
+
+while :
+do
+  case $1 in
+    -h | --help)            help                    ;;
+    -o | --outdir)          outdir=$2               ; shift 2 ;;
+    -f | --output-prefix)   output_log_prefix=$2    ; shift 2 ;;
+    -P | --policy-file)     policy_file=$2           ; shift 2 ;;
+    --) shift; break ;;
+    *) >&2 echo Unsupported option: $1
+       usage ;;
+  esac
+done
+
+if [[ $# -eq 0 ]]; then
+  usage
+fi
+
+device="$1"
+
+# Ensure device is specified
+if [[ -z "$device" ]]; then
+    echo "Error: Specify either the name of a fileset or a directory path"
+    usage
+fi
+
+# create default output_log_prefix if not specified in the arguments
+if [[ -z "$output_log_prefix"]]; then
+    modified_device=$(echo "$device" | sed -e 's|^/||' -e 's|/$||' -e 's|/|-|g')
+    output_log_prefix="list-policy_${modified_device}"
+fi
+
+# create temporary working directory for list aggregation
+tmpglobal=$outdir/slurm-tmp-${SLURM_JOBID}
+tmpscratch=$outdir/slurm-tmp-${SLURM_JOBID}
 mkdir -p $tmpglobal
 
 nodes=`scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//'`
@@ -18,17 +102,17 @@ cores="${SLURM_CPUS_PER_TASK}"
 
 DATESTR=`date +'%Y-%m-%d-%H:%M:%S'`
 
-policy=`basename $policyfile`
+policy=`basename $policy_file`
 filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}"
 
-cmd="mmapplypolicy ${filesystem} -I defer \
-  -P $policyfile \
+cmd="mmapplypolicy ${device} -I defer \
+  -P $policy_file \
   -g $tmpglobal \
   -s $tmpscratch \
-  -f ${DIR}/list-${SLURM_JOBID} \
-  -M FILEPATH=${filesystem} \
+  -f ${outdir}/list-${SLURM_JOBID} \
+  -M FILEPATH=${device} \
   -M JOBID=${SLURM_JOBID} \
-  -M LIST_OUTPUT_FILE=${OUTFILE:-/tmp/gpfs-list-policy}
+  -M LIST_OUTPUT_FILE=${output_prefix} \
   -N ${nodes} -n ${cores} -m ${cores}"
 
 # report final command in job log
@@ -41,6 +125,6 @@ $cmd
 outfile=`ls -t $tmpglobal | head -1`
 if [[ "$outfile" != "" ]]
 then
-   mv -n $tmpglobal/$outfile $tmpglobal/../${outfile}_$filetag
+   mv -n $tmpglobal/$outfile $tmpglobal/../${output_log_prefix}_$filetag
 fi
 rmdir $tmpglobal

src/run-policy/submit-pol-job

@@ -110,8 +110,8 @@ fi
 slurm_out="out/pol-%A-$(basename ${policy})-$(basename ${device}).out"
 mkdir -p out
 
-DIR=$outdir POLICYFILE=$policy FILESYSTEM=${device} OUTFILE=${outfile} && \
-DIR=$DIR POLICYFILE=$POLICYFILE FILESYSTEM=${FILESYSTEM} OUTFILE=${OUTFILE} \
+run_mmpol_cmd="./run-mmpol.sh -o ${outdir} -f ${outfile} -P ${policy} ${device}"
+
 sbatch \
    -N $nodes \
    -c $cores \
@@ -119,4 +119,4 @@ sbatch \
    --mem-per-cpu=$mem_per_cpu \
    -p $partition \
    -o ${slurm_out} \
-   ./run-mmpol.sh
+   "${run_mmpol_cmd}"