smiuser/hpos-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge for sub folders

Browse Source
This commit is contained in:
prisar
2023-07-14 17:01:37 +05:30
parent 1f65c08ed9
commit 8965b2be89
1 changed files with 46 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
scripts/merge_csv.py
View File

@@ -1,37 +1,55 @@
import glob
import pandas as pd
import openpyxl
# list all csv files only
csv_files = glob.glob('*.{}'.format('csv'))
# print(csv_files)
import os
df_csv_append = pd.DataFrame()
first = True
def consolidate(curdir):
# merge the CSV files
for file in csv_files:
if first:
df_csv_append = pd.read_csv(file)
colname = file.split('.')[0]
df_csv_append.rename(columns={'ca': colname}, inplace = True)
df_csv_append = df_csv_append.drop(['1'], axis=1)
first = False
else:
df = pd.read_csv(file)
colname = file.split('.')[0]
df.rename(columns={'ca': colname}, inplace = True)
df = df.drop(['1'], axis=1)
df_csv_append = df_csv_append.merge(df, on='tv')
print(df_csv_append[df_csv_append['tv'].between(300, 700)])
# list all csv files only
csv_files = glob.glob(curdir + '/*.{}'.format('csv'))
# print(csv_files)
df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
df_csv_append = pd.DataFrame()
first = True
df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
# merge the CSV files
for file in csv_files:
# print(file)
if first:
df_csv_append = pd.read_csv(file)
colname = file.split('.')[0].split('/')[-1] #file.split('.')[0]
# print(colname)
df_csv_append.rename(columns={'ca': colname}, inplace = True)
df_csv_append = df_csv_append.drop(['1'], axis=1)
first = False
else:
df = pd.read_csv(file)
colname = file.split('.')[0].split('/')[-1] #file.split('.')[0]
# print(colname)
df.rename(columns={'ca': colname}, inplace = True)
df = df.drop(['1'], axis=1)
df_csv_append = df_csv_append.merge(df, on='tv')
# print(df_csv_append[df_csv_append['tv'].between(300, 700)])
df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
df_csv_append.to_excel("D27 Merged.xlsx", index=False)
df_csv_append.to_csv("D27 Merged.csv", index=False)
df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
outfile = curdir + "/D" + os.path.split(curdir)[1].split()[1] + "_Merged.xlsx"
df_csv_append.to_excel(outfile, index=False)
# df_csv_append.to_csv("D8 Merged.csv", index=False)
rootdir = os.getcwd()
for file in os.listdir(rootdir):
d = os.path.join(rootdir, file)
if os.path.isdir(d):
# print(d)
# os.chdir(d)
consolidate(d)