script for merging csv
This commit is contained in:
37
scripts/merge_csv.py
Normal file
37
scripts/merge_csv.py
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
import glob
|
||||||
|
import pandas as pd
|
||||||
|
import openpyxl
|
||||||
|
|
||||||
|
# list all csv files only
|
||||||
|
csv_files = glob.glob('*.{}'.format('csv'))
|
||||||
|
# print(csv_files)
|
||||||
|
|
||||||
|
df_csv_append = pd.DataFrame()
|
||||||
|
|
||||||
|
first = True
|
||||||
|
|
||||||
|
# merge the CSV files
|
||||||
|
for file in csv_files:
|
||||||
|
if first:
|
||||||
|
df_csv_append = pd.read_csv(file)
|
||||||
|
colname = file.split('.')[0]
|
||||||
|
df_csv_append.rename(columns={'ca': colname}, inplace = True)
|
||||||
|
df_csv_append = df_csv_append.drop(['1'], axis=1)
|
||||||
|
first = False
|
||||||
|
else:
|
||||||
|
df = pd.read_csv(file)
|
||||||
|
colname = file.split('.')[0]
|
||||||
|
df.rename(columns={'ca': colname}, inplace = True)
|
||||||
|
df = df.drop(['1'], axis=1)
|
||||||
|
df_csv_append = df_csv_append.merge(df, on='tv')
|
||||||
|
|
||||||
|
print(df_csv_append[df_csv_append['tv'].between(300, 700)])
|
||||||
|
|
||||||
|
df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
|
||||||
|
|
||||||
|
df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
|
||||||
|
|
||||||
|
df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
|
||||||
|
|
||||||
|
df_csv_append.to_excel("D27 Merged.xlsx", index=False)
|
||||||
|
df_csv_append.to_csv("D27 Merged.csv", index=False)
|
||||||
78
scripts/merge_csv_k.py
Normal file
78
scripts/merge_csv_k.py
Normal file
@@ -0,0 +1,78 @@
|
|||||||
|
import csv
|
||||||
|
import os
|
||||||
|
from tkinter import Tk, filedialog
|
||||||
|
|
||||||
|
def merge_csv_files(input_files, output_file):
|
||||||
|
file_contents = {}
|
||||||
|
header = ['First Column']
|
||||||
|
|
||||||
|
# Read all selected CSV files
|
||||||
|
for filepath in input_files:
|
||||||
|
filename = os.path.basename(filepath)
|
||||||
|
with open(filepath, 'r') as file:
|
||||||
|
csv_reader = csv.reader(file)
|
||||||
|
next(csv_reader) # Skip the header row
|
||||||
|
for row in csv_reader:
|
||||||
|
try:
|
||||||
|
value = float(row[0])
|
||||||
|
if 300 <= value <= 700: # Check if the value is within the range
|
||||||
|
if row[0] not in file_contents:
|
||||||
|
file_contents[row[0]] = {}
|
||||||
|
|
||||||
|
file_contents[row[0]][filename] = row[1]
|
||||||
|
|
||||||
|
if filename not in header:
|
||||||
|
header.append(filename)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Write the merged contents into the output file
|
||||||
|
with open(output_file, 'w', newline='') as file:
|
||||||
|
csv_writer = csv.writer(file)
|
||||||
|
csv_writer.writerow(header)
|
||||||
|
|
||||||
|
for key, values in file_contents.items():
|
||||||
|
row = [key]
|
||||||
|
|
||||||
|
for filename in header[1:]:
|
||||||
|
row.append(values.get(filename, ''))
|
||||||
|
|
||||||
|
csv_writer.writerow(row)
|
||||||
|
|
||||||
|
print(f'Merged CSV files saved to: {output_file}')
|
||||||
|
|
||||||
|
|
||||||
|
def extract_data_from_merged_file(merged_file):
|
||||||
|
data = []
|
||||||
|
with open(merged_file, 'r') as file:
|
||||||
|
csv_reader = csv.reader(file)
|
||||||
|
header = next(csv_reader) # Get the header row
|
||||||
|
for i, row in enumerate(csv_reader, start=1):
|
||||||
|
if i in (365, 750):
|
||||||
|
extracted_row = []
|
||||||
|
for j, value in enumerate(row[1:], start=1):
|
||||||
|
column_name = header[j]
|
||||||
|
extracted_row.append((column_name, value))
|
||||||
|
data.append(extracted_row)
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# Create a file dialog to select multiple CSV files
|
||||||
|
root = Tk()
|
||||||
|
root.withdraw()
|
||||||
|
input_files = filedialog.askopenfilenames(title='Select CSV files', filetypes=(('CSV files', '*.csv'),))
|
||||||
|
|
||||||
|
# Ensure at least one file is selected
|
||||||
|
if input_files:
|
||||||
|
# Specify the output file path
|
||||||
|
output_file = 'merged.csv'
|
||||||
|
|
||||||
|
merge_csv_files(input_files, output_file)
|
||||||
|
|
||||||
|
extracted_data = extract_data_from_merged_file(output_file)
|
||||||
|
|
||||||
|
for row in extracted_data:
|
||||||
|
print(row)
|
||||||
|
else:
|
||||||
|
print('No files selected. Program will exit.')
|
||||||
27
src/App.js
27
src/App.js
@@ -7,14 +7,27 @@ import { getAnalytics } from "firebase/analytics";
|
|||||||
import { getFirestore, collection, getDocs, doc, onSnapshot, query } from "firebase/firestore";
|
import { getFirestore, collection, getDocs, doc, onSnapshot, query } from "firebase/firestore";
|
||||||
import Card from './components/Card';
|
import Card from './components/Card';
|
||||||
|
|
||||||
|
// dev and test
|
||||||
|
// const firebaseConfig = {
|
||||||
|
// apiKey: "AIzaSyBFl_YkJ5PMIvMY3G70313SyrqemjFnUv8",
|
||||||
|
// authDomain: "hpos-af3cc.firebaseapp.com",
|
||||||
|
// projectId: "hpos-af3cc",
|
||||||
|
// storageBucket: "hpos-af3cc.appspot.com",
|
||||||
|
// messagingSenderId: "650071678820",
|
||||||
|
// appId: "1:650071678820:web:ce50538bf46632d46c6471",
|
||||||
|
// measurementId: "G-6JHQ47S9DD"
|
||||||
|
// };
|
||||||
|
|
||||||
|
|
||||||
|
// prod
|
||||||
const firebaseConfig = {
|
const firebaseConfig = {
|
||||||
apiKey: "AIzaSyBFl_YkJ5PMIvMY3G70313SyrqemjFnUv8",
|
apiKey: "AIzaSyBF4pLZReAedU4dWX1eRJNFNIaOIz2xk4s",
|
||||||
authDomain: "hpos-af3cc.firebaseapp.com",
|
authDomain: "hpos-prod.firebaseapp.com",
|
||||||
projectId: "hpos-af3cc",
|
projectId: "hpos-prod",
|
||||||
storageBucket: "hpos-af3cc.appspot.com",
|
storageBucket: "hpos-prod.appspot.com",
|
||||||
messagingSenderId: "650071678820",
|
messagingSenderId: "1012719870714",
|
||||||
appId: "1:650071678820:web:ce50538bf46632d46c6471",
|
appId: "1:1012719870714:web:fcd00acef93f6612de5f43",
|
||||||
measurementId: "G-6JHQ47S9DD"
|
measurementId: "G-B0NZ7DEF69"
|
||||||
};
|
};
|
||||||
|
|
||||||
const app = initializeApp(firebaseConfig);
|
const app = initializeApp(firebaseConfig);
|
||||||
|
|||||||
Reference in New Issue
Block a user