From 1b65914b480fcfe5e7cb6df72679d4b257f8006d Mon Sep 17 00:00:00 2001 From: Pritimay Sarkar Date: Thu, 27 Jul 2023 15:05:57 +0530 Subject: [PATCH] add excel merge code into two sheets --- .gitignore | 1 + scripts/merge_csv.py | 105 +++++++++++++++++++++++++++++++++---------- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 2364368..c9aa47f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ yarn-error.log* src/components/testData.json public/world-110m.json +public/india-states.json diff --git a/scripts/merge_csv.py b/scripts/merge_csv.py index 66db587..4551284 100644 --- a/scripts/merge_csv.py +++ b/scripts/merge_csv.py @@ -2,16 +2,18 @@ import glob import pandas as pd import openpyxl import os -from platform import system +from tqdm import tqdm +import platform +from PyQt5.QtWidgets import QApplication, QWidget, QFileDialog -rootdir = os.getcwd() - -def consolidate(curdir): +def consolidate_and_perform_calculations(curdir, rootdir, path_delim, validation_file): print(os.path.split(curdir)[1]) # list all csv files only - csv_files = glob.glob(curdir + '/*.{}'.format('csv')) + csv_files = glob.glob(curdir + path_delim + '/*.{}'.format('csv')) + if len(csv_files) == 0: + print("no csv files in the sub folder") # print(csv_files) df_csv_append = pd.DataFrame() @@ -20,45 +22,100 @@ def consolidate(curdir): # merge the CSV files for file in csv_files: - # print(file) if first: df_csv_append = pd.read_csv(file) - colname = file.split('.')[0].split('/')[-1] #file.split('.')[0] - # print(colname) + colname = file.split('.')[0].split(path_delim)[-1] #file.split('.')[0] df_csv_append.rename(columns={'ca': colname}, inplace = True) df_csv_append = df_csv_append.drop(['1'], axis=1) first = False else: df = pd.read_csv(file) - colname = file.split('.')[0].split('/')[-1] #file.split('.')[0] - # print(colname) + colname = file.split('.')[0].split(path_delim)[-1] #file.split('.')[0] df.rename(columns={'ca': colname}, inplace = True) df = df.drop(['1'], axis=1) df_csv_append = df_csv_append.merge(df, on='tv') - # print(df_csv_append[df_csv_append['tv'].between(300, 700)]) - df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)] - df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True) + wavelength_col = "123_tv" # to make sorting columns simpler + + df_csv_append.rename(columns={'tv': wavelength_col}, inplace = True) df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1) - outfile = rootdir + "/" + os.path.split(curdir)[1] + "_Merged.xlsx" + outfile = rootdir + path_delim + os.path.split(curdir)[1] + "_analysis.xlsx" # df_csv_append.to_excel(outfile, sheet_name="merged_data", index=False) - # df_csv_append.to_csv("D8 Merged.csv", index=False) - writer = pd.ExcelWriter(outfile, engine = 'xlsxwriter') - df_csv_append.to_excel(writer, sheet_name = 'x1') - df_csv_append.to_excel(writer, sheet_name = 'x2') + # calculations + df_427 = df_csv_append.loc[(df_csv_append[wavelength_col] >= 427) & (df_csv_append[wavelength_col] < 428)] + + df_555 = df_csv_append.loc[(df_csv_append[wavelength_col] >= 555) & (df_csv_append[wavelength_col] < 556)] + + df_validation = pd.read_excel(validation_file) + + df_analysis = df_427.iloc[0] + df_555.iloc[0] + # print(df_analysis) + # print(min(df_csv_append[0:5])) + + midpoint1 = 427 + midpoint2 = 555 + bandwidth1 = 25 + bandwidth2 = 10 + + df = df_analysis.rename(columns = {"NM":"Wavelength","CA":"Absorbance"}, inplace = True) + + # 427 nm range + df1 = df[ (df['Wavelength'] > (midpoint1-bandwidth1)) & (df['Wavelength'] < (midpoint1+bandwidth1)) ] + + # 555 nm range + df2 = df[ (df['Wavelength'] > (midpoint2-bandwidth2)) & (df['Wavelength'] < (midpoint2+bandwidth2)) ] + + procData.append({"Sample ID": sampleID, + "max_427": round(df1["Absorbance"].max() , 3), + "wvmax_427": round(df1.at[df1["Absorbance"].idxmax(),"Wavelength"], 3), + "avg_427": round(df1["Absorbance"].mean(), 3), + "max_555": round(df2["Absorbance"].max(), 3), + "wvmax_555": round(df2.at[df2["Absorbance"].idxmax(),"Wavelength"], 3), + "avg_555": round(df2["Absorbance"].mean(), 3), + "ratio_max": round(df2["Absorbance"].max()/df1["Absorbance"].max(), 3), + "ratio_avg": round(df2["Absorbance"].mean()/df1["Absorbance"].mean(), 3) + }) + + writer = pd.ExcelWriter(outfile, engine = 'openpyxl') + df_analysis.to_excel(writer, sheet_name = 'analysis', index=False) + df_csv_append.to_excel(writer, sheet_name = 'merged_data', index=False) writer.close() + +sampleID = 1 +allDF = pd.DataFrame() +procData = [] + + if __name__ == "__main__": - for file in os.listdir(rootdir): - curdir = os.path.join(rootdir, file) - if os.path.isdir(d): - # os.chdir(d) - consolidate(curdir) - \ No newline at end of file + environment = "dev" # dev, QC + rootdir = os.getcwd() + validation_file = rootdir + '/validation.xlsx' + if environment != "dev": + app = QApplication([]) + + # window = QWidget() + # layout = QVBoxLayout(window) + rootdir = QFileDialog.getExistingDirectory(None, 'Select main folder') + choose_validation_file = QFileDialog.getOpenFileName(None, "Select validation excel") + validation_file = choose_validation_file[0] + + path_delim = '' + if platform.system() == 'Darwin': + path_delim = '/' + else: + path_delim = '\\' + + for file in tqdm(os.listdir(rootdir)): + curdir = os.path.join(rootdir, file) + if os.path.isdir(curdir): + # os.chdir(d) + consolidate_and_perform_calculations(curdir, rootdir, path_delim, validation_file) + \ No newline at end of file