hpos-web/scripts/merge_csv.py

import glob
import pandas as pd
import openpyxl
import os

rootdir = os.getcwd()

def consolidate(curdir):

    print(os.path.split(curdir)[1])
    
    # list all csv files only  
    csv_files = glob.glob(curdir + '/*.{}'.format('csv'))
    # print(csv_files)

    df_csv_append = pd.DataFrame()
    
    first = True

    # merge the CSV files
    for file in csv_files:
        # print(file)
        if first:
            df_csv_append = pd.read_csv(file)
            colname = file.split('.')[0].split('/')[-1] #file.split('.')[0]
            # print(colname)
            df_csv_append.rename(columns={'ca': colname}, inplace = True)
            df_csv_append = df_csv_append.drop(['1'], axis=1)
            first = False
        else:
            df = pd.read_csv(file)
            colname = file.split('.')[0].split('/')[-1] #file.split('.')[0]
            # print(colname)
            df.rename(columns={'ca': colname}, inplace = True)
            df = df.drop(['1'], axis=1)
            df_csv_append = df_csv_append.merge(df, on='tv')
    
    # print(df_csv_append[df_csv_append['tv'].between(300, 700)])

    df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]

    df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)

    df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)

    outfile = rootdir + "/" + os.path.split(curdir)[1] + "_Merged.xlsx"

    df_csv_append.to_excel(outfile, index=False)

    # df_csv_append.to_csv("D8 Merged.csv", index=False)


for file in os.listdir(rootdir):
    d = os.path.join(rootdir, file)
    if os.path.isdir(d):
        # print(d)
        # os.chdir(d)
        consolidate(d)