script for merging csv

This commit is contained in:
prisar
2023-07-14 16:56:28 +05:30
parent 521eb235d5
commit 1f65c08ed9
3 changed files with 135 additions and 7 deletions

37
scripts/merge_csv.py Normal file
View File

@@ -0,0 +1,37 @@
import glob
import pandas as pd
import openpyxl
# list all csv files only
csv_files = glob.glob('*.{}'.format('csv'))
# print(csv_files)
df_csv_append = pd.DataFrame()
first = True
# merge the CSV files
for file in csv_files:
if first:
df_csv_append = pd.read_csv(file)
colname = file.split('.')[0]
df_csv_append.rename(columns={'ca': colname}, inplace = True)
df_csv_append = df_csv_append.drop(['1'], axis=1)
first = False
else:
df = pd.read_csv(file)
colname = file.split('.')[0]
df.rename(columns={'ca': colname}, inplace = True)
df = df.drop(['1'], axis=1)
df_csv_append = df_csv_append.merge(df, on='tv')
print(df_csv_append[df_csv_append['tv'].between(300, 700)])
df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
df_csv_append.to_excel("D27 Merged.xlsx", index=False)
df_csv_append.to_csv("D27 Merged.csv", index=False)

78
scripts/merge_csv_k.py Normal file
View File

@@ -0,0 +1,78 @@
import csv
import os
from tkinter import Tk, filedialog
def merge_csv_files(input_files, output_file):
file_contents = {}
header = ['First Column']
# Read all selected CSV files
for filepath in input_files:
filename = os.path.basename(filepath)
with open(filepath, 'r') as file:
csv_reader = csv.reader(file)
next(csv_reader) # Skip the header row
for row in csv_reader:
try:
value = float(row[0])
if 300 <= value <= 700: # Check if the value is within the range
if row[0] not in file_contents:
file_contents[row[0]] = {}
file_contents[row[0]][filename] = row[1]
if filename not in header:
header.append(filename)
except ValueError:
pass
# Write the merged contents into the output file
with open(output_file, 'w', newline='') as file:
csv_writer = csv.writer(file)
csv_writer.writerow(header)
for key, values in file_contents.items():
row = [key]
for filename in header[1:]:
row.append(values.get(filename, ''))
csv_writer.writerow(row)
print(f'Merged CSV files saved to: {output_file}')
def extract_data_from_merged_file(merged_file):
data = []
with open(merged_file, 'r') as file:
csv_reader = csv.reader(file)
header = next(csv_reader) # Get the header row
for i, row in enumerate(csv_reader, start=1):
if i in (365, 750):
extracted_row = []
for j, value in enumerate(row[1:], start=1):
column_name = header[j]
extracted_row.append((column_name, value))
data.append(extracted_row)
return data
if __name__ == "__main__":
# Create a file dialog to select multiple CSV files
root = Tk()
root.withdraw()
input_files = filedialog.askopenfilenames(title='Select CSV files', filetypes=(('CSV files', '*.csv'),))
# Ensure at least one file is selected
if input_files:
# Specify the output file path
output_file = 'merged.csv'
merge_csv_files(input_files, output_file)
extracted_data = extract_data_from_merged_file(output_file)
for row in extracted_data:
print(row)
else:
print('No files selected. Program will exit.')

View File

@@ -7,14 +7,27 @@ import { getAnalytics } from "firebase/analytics";
import { getFirestore, collection, getDocs, doc, onSnapshot, query } from "firebase/firestore"; import { getFirestore, collection, getDocs, doc, onSnapshot, query } from "firebase/firestore";
import Card from './components/Card'; import Card from './components/Card';
// dev and test
// const firebaseConfig = {
// apiKey: "AIzaSyBFl_YkJ5PMIvMY3G70313SyrqemjFnUv8",
// authDomain: "hpos-af3cc.firebaseapp.com",
// projectId: "hpos-af3cc",
// storageBucket: "hpos-af3cc.appspot.com",
// messagingSenderId: "650071678820",
// appId: "1:650071678820:web:ce50538bf46632d46c6471",
// measurementId: "G-6JHQ47S9DD"
// };
// prod
const firebaseConfig = { const firebaseConfig = {
apiKey: "AIzaSyBFl_YkJ5PMIvMY3G70313SyrqemjFnUv8", apiKey: "AIzaSyBF4pLZReAedU4dWX1eRJNFNIaOIz2xk4s",
authDomain: "hpos-af3cc.firebaseapp.com", authDomain: "hpos-prod.firebaseapp.com",
projectId: "hpos-af3cc", projectId: "hpos-prod",
storageBucket: "hpos-af3cc.appspot.com", storageBucket: "hpos-prod.appspot.com",
messagingSenderId: "650071678820", messagingSenderId: "1012719870714",
appId: "1:650071678820:web:ce50538bf46632d46c6471", appId: "1:1012719870714:web:fcd00acef93f6612de5f43",
measurementId: "G-6JHQ47S9DD" measurementId: "G-B0NZ7DEF69"
}; };
const app = initializeApp(firebaseConfig); const app = initializeApp(firebaseConfig);