From 1553fc8acf5d7216035fd6c06cd92120f8086bf4 Mon Sep 17 00:00:00 2001
From: Matthew K Defenderfer <mdefende@uab.edu>
Date: Wed, 30 Apr 2025 13:50:14 -0500
Subject: [PATCH 1/2] add try-except around sbatch to capture any potential
errors during child job submission
---
src/rc_gpfs/cli/convert_flat_to_hive.py | 5 ++++-
src/rc_gpfs/cli/convert_to_parquet.py | 5 ++++-
src/rc_gpfs/cli/split_log.py | 5 ++++-
3 files changed, 12 insertions(+), 3 deletions(-)
diff --git a/src/rc_gpfs/cli/convert_flat_to_hive.py b/src/rc_gpfs/cli/convert_flat_to_hive.py
index 0fe6463..a2459e0 100644
--- a/src/rc_gpfs/cli/convert_flat_to_hive.py
+++ b/src/rc_gpfs/cli/convert_flat_to_hive.py
@@ -96,7 +96,10 @@ def submit_batch(**kwargs):
# delay should fix that
time.sleep(random.uniform(1,5))
- subprocess.run(['sbatch'],input=script,shell=True,text=True)
+ try:
+ subprocess.run(['sbatch'],input=script,shell=True,text=True,check=True,capture_output=True)
+ except subprocess.CalledProcessError as e:
+ print(f"Error message: {e.stderr.decode()}")
pass
def submit_tld_groups(df,grp_dir,args):
diff --git a/src/rc_gpfs/cli/convert_to_parquet.py b/src/rc_gpfs/cli/convert_to_parquet.py
index ac84448..80469b5 100644
--- a/src/rc_gpfs/cli/convert_to_parquet.py
+++ b/src/rc_gpfs/cli/convert_to_parquet.py
@@ -80,7 +80,10 @@ def submit_batch(**kwargs):
# fix that
time.sleep(random.uniform(1, 5))
- subprocess.run(['sbatch'],input=script,shell=True,text=True)
+ try:
+ subprocess.run(['sbatch'],input=script,shell=True,text=True,check=True,capture_output=True)
+ except subprocess.CalledProcessError as e:
+ print(f"Error message: {e.stderr.decode()}")
pass
def _find_sequential_indexes(idxs):
diff --git a/src/rc_gpfs/cli/split_log.py b/src/rc_gpfs/cli/split_log.py
index 44c8461..56fb413 100644
--- a/src/rc_gpfs/cli/split_log.py
+++ b/src/rc_gpfs/cli/split_log.py
@@ -51,7 +51,10 @@ def submit_batch(**kwargs):
script = BATCH_SCRIPT.format(**kwargs)
- subprocess.run(['sbatch'],input=script,shell=True,text=True)
+ try:
+ subprocess.run(['sbatch'],input=script,shell=True,text=True,check=True,capture_output=True)
+ except subprocess.CalledProcessError as e:
+ print(f"Error message: {e.stderr.decode()}")
pass
def split_log():
--
GitLab
From 2687ed8f557051f4953b7f22adaa88eeca2f7466 Mon Sep 17 00:00:00 2001
From: Matthew K Defenderfer <mdefende@uab.edu>
Date: Wed, 30 Apr 2025 13:53:19 -0500
Subject: [PATCH 2/2] change to raise error instead of just printing message
and exiting normally
---
src/rc_gpfs/cli/convert_flat_to_hive.py | 2 +-
src/rc_gpfs/cli/convert_to_parquet.py | 2 +-
src/rc_gpfs/cli/split_log.py | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/rc_gpfs/cli/convert_flat_to_hive.py b/src/rc_gpfs/cli/convert_flat_to_hive.py
index a2459e0..678646e 100644
--- a/src/rc_gpfs/cli/convert_flat_to_hive.py
+++ b/src/rc_gpfs/cli/convert_flat_to_hive.py
@@ -99,7 +99,7 @@ def submit_batch(**kwargs):
try:
subprocess.run(['sbatch'],input=script,shell=True,text=True,check=True,capture_output=True)
except subprocess.CalledProcessError as e:
- print(f"Error message: {e.stderr.decode()}")
+ raise subprocess.CalledProcessError(e.stderr.decode())
pass
def submit_tld_groups(df,grp_dir,args):
diff --git a/src/rc_gpfs/cli/convert_to_parquet.py b/src/rc_gpfs/cli/convert_to_parquet.py
index 80469b5..5c94df4 100644
--- a/src/rc_gpfs/cli/convert_to_parquet.py
+++ b/src/rc_gpfs/cli/convert_to_parquet.py
@@ -83,7 +83,7 @@ def submit_batch(**kwargs):
try:
subprocess.run(['sbatch'],input=script,shell=True,text=True,check=True,capture_output=True)
except subprocess.CalledProcessError as e:
- print(f"Error message: {e.stderr.decode()}")
+ raise subprocess.CalledProcessError(e.stderr.decode())
pass
def _find_sequential_indexes(idxs):
diff --git a/src/rc_gpfs/cli/split_log.py b/src/rc_gpfs/cli/split_log.py
index 56fb413..a535893 100644
--- a/src/rc_gpfs/cli/split_log.py
+++ b/src/rc_gpfs/cli/split_log.py
@@ -54,7 +54,7 @@ def submit_batch(**kwargs):
try:
subprocess.run(['sbatch'],input=script,shell=True,text=True,check=True,capture_output=True)
except subprocess.CalledProcessError as e:
- print(f"Error message: {e.stderr.decode()}")
+ raise subprocess.CalledProcessError(e.stderr.decode())
pass
def split_log():
--
GitLab