From 7fd2308b85282d94fe541f6b55234d96e748c376 Mon Sep 17 00:00:00 2001
From: Matthew K Defenderfer <mdefende@uab.edu>
Date: Fri, 30 Aug 2024 09:04:35 -0500
Subject: [PATCH] acheck to see if mode exists in the dataset in the first
place
---
transfer-gpfs-with-s5cmd/fpart-db.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/transfer-gpfs-with-s5cmd/fpart-db.py b/transfer-gpfs-with-s5cmd/fpart-db.py
index 11ecc9e..26cf757 100755
--- a/transfer-gpfs-with-s5cmd/fpart-db.py
+++ b/transfer-gpfs-with-s5cmd/fpart-db.py
@@ -34,7 +34,8 @@ def main():
ddf = dd.read_parquet(input_parquet)
ddf = ddf.loc[ddf['path'].str.startswith(filter)].sort_values('path')
- ddf = ddf.loc[~ddf['mode'].str.startswith('d')]
+ if 'mode' in ddf.columns:
+ ddf = ddf.loc[~ddf['mode'].str.startswith('d')]
ddf['cmd'] = ddf['path'].map(lambda x: create_sync_cmd(x, filter=filter, dest=dest), meta=str)
--
GitLab