From bf1078878d3d213f996b1ba810fa67223443ff37 Mon Sep 17 00:00:00 2001
From: Matthew K Defenderfer <mdefende@uab.edu>
Date: Fri, 2 May 2025 17:49:28 -0500
Subject: [PATCH] Write parquet with rust backend to use full statistics option
and better optimize row groups
---
src/rc_gpfs/policy/convert.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/rc_gpfs/policy/convert.py b/src/rc_gpfs/policy/convert.py
index 0873e9a..4890bd6 100755
--- a/src/rc_gpfs/policy/convert.py
+++ b/src/rc_gpfs/policy/convert.py
@@ -88,4 +88,4 @@ def convert(
.sort('path')
)
- df.write_parquet(output_path,use_pyarrow=True)
\ No newline at end of file
+ df.write_parquet(output_path,statistics='full')
\ No newline at end of file
--
GitLab