Newer
Older
Matthew K Defenderfer
committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
#!/bin/bash
set -euo pipefail
############################################################
# Default Values #
############################################################
ntasks=4
mem="16G"
time="12:00:00"
partition="amd-hdr100"
lines=5000000
############################################################
# Help #
############################################################
usage()
{
>&2 cat << EOF
Usage: $0 [ -h ] [ -n | --ntasks ] [ -p | --partition] [ -t | --time ] [ -m | --mem ]
[ -l | --lines ] log
EOF
}
help()
{
# Display Help
>&2 cat << EOF
Splits a GPFS policy log into multiple parts for batch array processing
Usage: $0 [ -h ] [ -n | --ntasks ] [ -p | --partition ] [ -t | --time ] [ -m | --mem ]
[ -l | --lines ] log
General:
-h|--help Print this Help.
Required:
log Path to the log file to split
File Partitioning:
-l|--lines Max number of records to save in each split (default: 5000000)
Job Parameters:
-n|--ntasks Number of job tasks (default: 4)
-p|--partition Partition to submit tasks to (default: amd-hdr100)
-t|--time Max walltime (default: 12:00:00)
-m|--mem Memory (default: 16G)
EOF
exit 0
}
args=$(getopt -a -o hn:p:t:m:l: --long help,ntasks:,partition:,time:,mem:,lines: -- "$@")
if [[ $? -gt 0 ]]; then
usage
fi
eval set -- ${args}
while :
do
case $1 in
-h | --help) help ;;
-n | --ntasks) ntasks=$2 ; shift 2 ;;
-p | --partition) partition=$2 ; shift 2 ;;
-t | --time) time=$2 ; shift 2 ;;
-m | --mem) mem=$2 ; shift 2 ;;
-l | --lines) lines=$2 ; shift 2 ;;
--) shift; break ;;
*) >&2 echo Unsupported option: $1
usage ;;
esac
done
if [[ $# -eq 0 ]]; then
usage
fi
log=$1
dirname="$(basename ${log} .gz).d"
prefix=${dirname}/list-
split_cmd="cat ${log} | split -a 3 -d -l ${lines} - ${prefix}"
zip_cmd="ls ${prefix}* | xargs -i -P 0 bash -c 'gzip {} && echo {} done'"
if [[ $(file -b --mime-type ${log}) == *'gzip'* ]]; then
split_cmd="z${split_cmd}"
fi
>&2 cat << EOF
--------------------------------------------------------------------------------
GPFS log: ${log}
Lines per File: ${lines}
ntasks: ${ntasks}
partition: ${partition}
time: ${time}
mem: ${mem}
split cmd: ${split_cmd}
zip cmd: ${zip_cmd}
--------------------------------------------------------------------------------
EOF
mkdir -p ${dirname}
mkdir -p out
mkdir -p err
############################################################
# Create Array Job Script #
############################################################
{ cat | sbatch; } << EOF
#!/bin/bash
#
#SBATCH --job-name=split-gpfs-log
#SBATCH --ntasks=${ntasks}
#SBATCH --partition=${partition}
#SBATCH --time=${time}
#SBATCH --mem=${mem}
#SBATCH --output=out/%A.out
#SBATCH --error=err/%A.err
${split_cmd}
${zip_cmd}
EOF