From 8965b2be894076edfedd157da5015237932f6b23 Mon Sep 17 00:00:00 2001
From: prisar <prisar230@gmail.com>
Date: Fri, 14 Jul 2023 17:01:37 +0530
Subject: [PATCH] merge for sub folders

---
 scripts/merge_csv.py | 74 +++++++++++++++++++++++++++-----------------
 1 file changed, 46 insertions(+), 28 deletions(-)

diff --git a/scripts/merge_csv.py b/scripts/merge_csv.py
index 91ab85c..2120a28 100644
--- a/scripts/merge_csv.py
+++ b/scripts/merge_csv.py
@@ -1,37 +1,55 @@
 import glob
 import pandas as pd
 import openpyxl
- 
-# list all csv files only
-csv_files = glob.glob('*.{}'.format('csv'))
-# print(csv_files)
+import os
 
-df_csv_append = pd.DataFrame()
- 
-first = True
+def consolidate(curdir):
 
-# merge the CSV files
-for file in csv_files:
-    if first:
-        df_csv_append = pd.read_csv(file)
-        colname = file.split('.')[0]
-        df_csv_append.rename(columns={'ca': colname}, inplace = True)
-        df_csv_append = df_csv_append.drop(['1'], axis=1)
-        first = False
-    else:
-        df = pd.read_csv(file)
-        colname = file.split('.')[0]
-        df.rename(columns={'ca': colname}, inplace = True)
-        df = df.drop(['1'], axis=1)
-        df_csv_append = df_csv_append.merge(df, on='tv')
- 
-print(df_csv_append[df_csv_append['tv'].between(300, 700)])
+    # list all csv files only  
+    csv_files = glob.glob(curdir + '/*.{}'.format('csv'))
+    # print(csv_files)
 
-df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
+    df_csv_append = pd.DataFrame()
+    
+    first = True
 
-df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
+    # merge the CSV files
+    for file in csv_files:
+        # print(file)
+        if first:
+            df_csv_append = pd.read_csv(file)
+            colname = file.split('.')[0].split('/')[-1] #file.split('.')[0]
+            # print(colname)
+            df_csv_append.rename(columns={'ca': colname}, inplace = True)
+            df_csv_append = df_csv_append.drop(['1'], axis=1)
+            first = False
+        else:
+            df = pd.read_csv(file)
+            colname = file.split('.')[0].split('/')[-1] #file.split('.')[0]
+            # print(colname)
+            df.rename(columns={'ca': colname}, inplace = True)
+            df = df.drop(['1'], axis=1)
+            df_csv_append = df_csv_append.merge(df, on='tv')
+    
+    # print(df_csv_append[df_csv_append['tv'].between(300, 700)])
 
-df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
+    df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
 
-df_csv_append.to_excel("D27 Merged.xlsx", index=False)
-df_csv_append.to_csv("D27 Merged.csv", index=False)
+    df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
+
+    df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
+
+    outfile = curdir + "/D" + os.path.split(curdir)[1].split()[1] + "_Merged.xlsx"
+
+    df_csv_append.to_excel(outfile, index=False)
+
+    # df_csv_append.to_csv("D8 Merged.csv", index=False)
+
+
+rootdir = os.getcwd()
+for file in os.listdir(rootdir):
+    d = os.path.join(rootdir, file)
+    if os.path.isdir(d):
+        # print(d)
+        # os.chdir(d)
+        consolidate(d)