From 3851bf0cf65df9bb72449746c6ecc9566cac29e1 Mon Sep 17 00:00:00 2001
From: jackstarkmackenzie <v1jmac33@ed.ac.uk>
Date: Tue, 10 Dec 2024 16:18:04 +0000
Subject: [PATCH] parq-csv conversion script, data added to gitignore

---
 .gitignore               |  3 ++-
 parquet_output_to_csv.py | 21 +++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 parquet_output_to_csv.py

diff --git a/.gitignore b/.gitignore
index c8d5609..003c4b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 .Rhistory
 .RData
 .Ruserdata
-test_data/*
+data-output/test_data/*
 example_rolling/*
 .DS_Store
+data-output/test_data_output/*
\ No newline at end of file
diff --git a/parquet_output_to_csv.py b/parquet_output_to_csv.py
new file mode 100644
index 0000000..9044e07
--- /dev/null
+++ b/parquet_output_to_csv.py
@@ -0,0 +1,21 @@
+import pandas as pd
+import os
+from pyarrow import parquet
+
+input_dir = "data-output/test_data"
+output_dir = "data-output/test_data_output"
+
+for partition in os.listdir(input_dir):
+    if partition.startswith("end_of_this_period="): 
+        partition_path = os.path.join(input_dir, partition)
+        date_value = partition.split("=")[-1] 
+        
+        for file in os.listdir(partition_path):
+            if file.endswith(".parquet"):
+                file_path = os.path.join(partition_path, file)
+                df = pd.read_parquet(file_path, engine='pyarrow')
+                
+                df['end_of_this_period'] = date_value
+                
+                output_file = os.path.join(output_dir, f"{date_value}.csv")
+                df.to_csv(output_file, index=False)