Skip to content
Snippets Groups Projects
Commit 4de49ccf authored by Matthew K Defenderfer's avatar Matthew K Defenderfer
Browse files

added getopt and changed some variable names to match those in submit-pol-job

parent c155f8a8
No related branches found
No related tags found
7 merge requests!24Release v0.2.0,!22Update main to rel_v0.1.1-1,!21Fix some merge errors,!20Refactoring run-mmpol scripts,!19BUGFIX: Change ownership to executing USER and group to atlab. Also automatically gzip the log output,!18ENH: Change log storage organization and correctly apply log prefix and metadata to file name,!17ENH: Add getopt to run-mmpol and remove reliance on inherited environment variables
...@@ -4,13 +4,97 @@ set -euxo pipefail ...@@ -4,13 +4,97 @@ set -euxo pipefail
# run an mmapply policy across the cluster via slurm # run an mmapply policy across the cluster via slurm
# gather info to map mmapplypolicy to runtime configuration ############################################################
# arguments passed via job env and runtime context # Default Values #
############################################################
filesystem=${FILESYSTEM:-scratch} outdir="/data/rc/gpfs-policy/data"
policyfile=$POLICYFILE policy_file="./policy-def/list-path-external"
tmpglobal=$DIR/slurm-tmp-${SLURM_JOBID} device="scratch"
tmpscratch=$DIR/slurm-tmp-${SLURM_JOBID} output_log_prefix=""
############################################################
# Help #
############################################################
usage()
{
>&2 cat << EOF
Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [-P | --policy-file] device
EOF
exit 1
}
help()
{
>&2 cat << EOF
Runs mmapplypolicy on the specified device/fileset. The policy file dictates the actions performed including list, delete, add, etc. This is most often called by the submit-pol-job wrapper instead of invoked directly
Usage: $0 [ -h ] [ -o | --outdir ] [ -f | --output-prefix ] [ -P | --policy-file ] device
options:
-h|--help Print this Help.
Required:
device GPFS fileset/directory apply the policy to. Can be
specified as either the name of the fileset or the
full path to the directory
(Examples: scratch, /data/user/[username])
Path:
-o|--outdir Parent directory to save policy output to
(default: /data/rc/gpfs-policy/data)
-f|--output-prefix Prefix of the policy output file. Appended with a metadata string containing the policy name,
job ID, and date
Policy Options:
-P|--policy-file Path to policy file to apply to the given GPFS device
EOF
exit 0
}
args=$(getopt -a -o ho:f:P: --long help,outdir:,output-prefix:,policy-file: -- "$@")
if [[ $? -gt 0 ]]; then
usage
fi
eval set -- ${args}
while :
do
case $1 in
-h | --help) help ;;
-o | --outdir) outdir=$2 ; shift 2 ;;
-f | --output-prefix) output_log_prefix=$2 ; shift 2 ;;
-P | --policy-file) policy_file=$2 ; shift 2 ;;
--) shift; break ;;
*) >&2 echo Unsupported option: $1
usage ;;
esac
done
if [[ $# -eq 0 ]]; then
usage
fi
device="$1"
# Ensure device is specified
if [[ -z "$device" ]]; then
echo "Error: Specify either the name of a fileset or a directory path"
usage
fi
# create default output_log_prefix if not specified in the arguments
if [[ -z "$output_log_prefix"]]; then
modified_device=$(echo "$device" | sed -e 's|^/||' -e 's|/$||' -e 's|/|-|g')
output_log_prefix="list-policy_${modified_device}"
fi
# create temporary working directory for list aggregation
tmpglobal=$outdir/slurm-tmp-${SLURM_JOBID}
tmpscratch=$outdir/slurm-tmp-${SLURM_JOBID}
mkdir -p $tmpglobal mkdir -p $tmpglobal
nodes=`scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//'` nodes=`scontrol show hostnames "${SLURM_JOB_NODELIST}" | tr '\n' ',' | sed -e 's/,$//'`
...@@ -18,17 +102,17 @@ cores="${SLURM_CPUS_PER_TASK}" ...@@ -18,17 +102,17 @@ cores="${SLURM_CPUS_PER_TASK}"
DATESTR=`date +'%Y-%m-%d-%H:%M:%S'` DATESTR=`date +'%Y-%m-%d-%H:%M:%S'`
policy=`basename $policyfile` policy=`basename $policy_file`
filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}" filetag="${policy}_slurm-${SLURM_JOBID}_${DATESTR}"
cmd="mmapplypolicy ${filesystem} -I defer \ cmd="mmapplypolicy ${device} -I defer \
-P $policyfile \ -P $policy_file \
-g $tmpglobal \ -g $tmpglobal \
-s $tmpscratch \ -s $tmpscratch \
-f ${DIR}/list-${SLURM_JOBID} \ -f ${outdir}/list-${SLURM_JOBID} \
-M FILEPATH=${filesystem} \ -M FILEPATH=${device} \
-M JOBID=${SLURM_JOBID} \ -M JOBID=${SLURM_JOBID} \
-M LIST_OUTPUT_FILE=${OUTFILE:-/tmp/gpfs-list-policy} -M LIST_OUTPUT_FILE=${output_prefix} \
-N ${nodes} -n ${cores} -m ${cores}" -N ${nodes} -n ${cores} -m ${cores}"
# report final command in job log # report final command in job log
...@@ -41,6 +125,6 @@ $cmd ...@@ -41,6 +125,6 @@ $cmd
outfile=`ls -t $tmpglobal | head -1` outfile=`ls -t $tmpglobal | head -1`
if [[ "$outfile" != "" ]] if [[ "$outfile" != "" ]]
then then
mv -n $tmpglobal/$outfile $tmpglobal/../${outfile}_$filetag mv -n $tmpglobal/$outfile $tmpglobal/../${output_log_prefix}_$filetag
fi fi
rmdir $tmpglobal rmdir $tmpglobal
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment