Skip to content
Snippets Groups Projects

ENH: Add getopt to run-mmpol and remove reliance on inherited environment variables

Merged Matthew K Defenderfer requested to merge enh-add-getopt-to-run-mmpol into rel_v0.1.1-1
2 files
+ 100
16
Compare changes
  • Side-by-side
  • Inline
Files
2
+ 97
13
@@ -4,13 +4,97 @@ set -euxo pipefail
@@ -4,13 +4,97 @@ set -euxo pipefail
# run an mmapply policy across the cluster via slurm
# run an mmapply policy across the cluster via slurm
# gather info to map mmapplypolicy to runtime configuration
############################################################
# arguments passed via job env and runtime context
# Default Values #
 
############################################################
filesystem=${FILESYSTEM:-scratch}
outdir="/data/rc/gpfs-policy/data"
policyfile=$POLICYFILE
policy_file="./policy-def/list-path-external"
tmpglobal=$DIR/slurm-tmp-${SLURM_JOBID}
device="scratch"
tmpscratch=$DIR/slurm-tmp-${SLURM_JOBID}
output_log_prefix=""
 
 
############################################################
 
# Help #
 
############################################################
 
 
usage()
 
{
 
>&2 cat << EOF
 
Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [-P | --policy-file] device
 
EOF
 
exit 1
 
}
 
 
help()
 
{
 
>&2 cat << EOF
 
Runs mmapplypolicy on the specified device/fileset. The policy file dictates the actions performed including list, delete, add, etc. This is most often called by the submit-pol-job wrapper instead of invoked directly
 
 
Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file ] device
 
 
options:
 
-h|--help Print this Help.
 
 
Required:
 
device GPFS fileset/directory apply the policy to. Can be
 
specified as either the name of the fileset or the
 
full path to the directory
 
(Examples: scratch, /data/user/[username])
 
 
Path:
 
-o|--outdir Parent directory to save policy output to
 
(default: /data/rc/gpfs-policy/data)
 
-f|--output-prefix Prefix of the policy output file. Appended with a metadata string containing the policy name,
 
job ID, and date
 
 
Policy Options:
 
-P|--policy-file Path to policy file to apply to the given GPFS device
 
EOF
 
exit 0
 
}
 
 
args=$(getopt -a -o ho:f:P: --long help,outdir:,output-prefix:,policy-file: -- "$@")
 
 
if [[ $? -gt 0 ]]; then
 
usage
 
fi
 
 
eval set -- ${args}
 
 
while :
 
do
 
case $1 in
 
-h | --help) help ;;
 
-o | --outdir) outdir=$2 ; shift 2 ;;
 
-f | --output-prefix) output_log_prefix=$2 ; shift 2 ;;
 
-P | --policy-file) policy_file=$2 ; shift 2 ;;
 
--) shift; break ;;
 
*) >&2 echo Unsupported option: $1
 
usage ;;
 
esac
 
done
 
 
if [[ $# -eq 0 ]]; then
 
usage
 
fi
 
 
device="$1"
 
 
# Ensure device is specified
 
if [[ -z "$device" ]]; then
 
echo "Error: Specify either the name of a fileset or a directory path"
 
usage
 
fi
 
 
# create default output_log_prefix if not specified in the arguments
 
if [[ -z "$output_log_prefix"]]; then
 
modified_device=$(echo "$device" | sed -e 's|^/||' -e 's|/$||' -e 's|/|-|g')
 
output_log_prefix="list-policy_${modified_device}"
 
fi
 
 
# create temporary working directory for list aggregation
 
tmpglobal=$outdir/slurm-tmp-${SLURM_JOBID}
 
tmpscratch=$outdir/slurm-tmp-${SLURM_JOBID}
mkdir -p $tmpglobal
mkdir -p $tmpglobal
nodes=`scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//'`
nodes=`scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//'`
@@ -18,17 +102,17 @@ cores="${SLURM_CPUS_PER_TASK}"
@@ -18,17 +102,17 @@ cores="${SLURM_CPUS_PER_TASK}"
DATESTR=`date +'%Y-%m-%d-%H:%M:%S'`
DATESTR=`date +'%Y-%m-%d-%H:%M:%S'`
policy=`basename $policyfile`
policy=`basename $policy_file`
filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}"
filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}"
cmd="mmapplypolicy ${filesystem} -I defer \
cmd="mmapplypolicy ${device} -I defer \
-P $policyfile \
-P $policy_file \
-g $tmpglobal \
-g $tmpglobal \
-s $tmpscratch \
-s $tmpscratch \
-f ${DIR}/list-${SLURM_JOBID} \
-f ${outdir}/list-${SLURM_JOBID} \
-M FILEPATH=${filesystem} \
-M FILEPATH=${device} \
-M JOBID=${SLURM_JOBID} \
-M JOBID=${SLURM_JOBID} \
-M LIST_OUTPUT_FILE=${OUTFILE:-/tmp/gpfs-list-policy}
-M LIST_OUTPUT_FILE=${output_prefix} \
-N ${nodes} -n ${cores} -m ${cores}"
-N ${nodes} -n ${cores} -m ${cores}"
# report final command in job log
# report final command in job log
@@ -41,6 +125,6 @@ $cmd
@@ -41,6 +125,6 @@ $cmd
outfile=`ls -t $tmpglobal | head -1`
outfile=`ls -t $tmpglobal | head -1`
if [[ "$outfile" != "" ]]
if [[ "$outfile" != "" ]]
then
then
mv -n $tmpglobal/$outfile $tmpglobal/../${outfile}_$filetag
mv -n $tmpglobal/$outfile $tmpglobal/../${output_log_prefix}_$filetag
fi
fi
rmdir $tmpglobal
rmdir $tmpglobal
Loading