script for merging csv
This commit is contained in:
37
scripts/merge_csv.py
Normal file
37
scripts/merge_csv.py
Normal file
@@ -0,0 +1,37 @@
|
||||
import glob
|
||||
import pandas as pd
|
||||
import openpyxl
|
||||
|
||||
# list all csv files only
|
||||
csv_files = glob.glob('*.{}'.format('csv'))
|
||||
# print(csv_files)
|
||||
|
||||
df_csv_append = pd.DataFrame()
|
||||
|
||||
first = True
|
||||
|
||||
# merge the CSV files
|
||||
for file in csv_files:
|
||||
if first:
|
||||
df_csv_append = pd.read_csv(file)
|
||||
colname = file.split('.')[0]
|
||||
df_csv_append.rename(columns={'ca': colname}, inplace = True)
|
||||
df_csv_append = df_csv_append.drop(['1'], axis=1)
|
||||
first = False
|
||||
else:
|
||||
df = pd.read_csv(file)
|
||||
colname = file.split('.')[0]
|
||||
df.rename(columns={'ca': colname}, inplace = True)
|
||||
df = df.drop(['1'], axis=1)
|
||||
df_csv_append = df_csv_append.merge(df, on='tv')
|
||||
|
||||
print(df_csv_append[df_csv_append['tv'].between(300, 700)])
|
||||
|
||||
df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
|
||||
|
||||
df_csv_append.rename(columns={'tv': "123_tv"}, inplace = True)
|
||||
|
||||
df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
|
||||
|
||||
df_csv_append.to_excel("D27 Merged.xlsx", index=False)
|
||||
df_csv_append.to_csv("D27 Merged.csv", index=False)
|
||||
78
scripts/merge_csv_k.py
Normal file
78
scripts/merge_csv_k.py
Normal file
@@ -0,0 +1,78 @@
|
||||
import csv
|
||||
import os
|
||||
from tkinter import Tk, filedialog
|
||||
|
||||
def merge_csv_files(input_files, output_file):
|
||||
file_contents = {}
|
||||
header = ['First Column']
|
||||
|
||||
# Read all selected CSV files
|
||||
for filepath in input_files:
|
||||
filename = os.path.basename(filepath)
|
||||
with open(filepath, 'r') as file:
|
||||
csv_reader = csv.reader(file)
|
||||
next(csv_reader) # Skip the header row
|
||||
for row in csv_reader:
|
||||
try:
|
||||
value = float(row[0])
|
||||
if 300 <= value <= 700: # Check if the value is within the range
|
||||
if row[0] not in file_contents:
|
||||
file_contents[row[0]] = {}
|
||||
|
||||
file_contents[row[0]][filename] = row[1]
|
||||
|
||||
if filename not in header:
|
||||
header.append(filename)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# Write the merged contents into the output file
|
||||
with open(output_file, 'w', newline='') as file:
|
||||
csv_writer = csv.writer(file)
|
||||
csv_writer.writerow(header)
|
||||
|
||||
for key, values in file_contents.items():
|
||||
row = [key]
|
||||
|
||||
for filename in header[1:]:
|
||||
row.append(values.get(filename, ''))
|
||||
|
||||
csv_writer.writerow(row)
|
||||
|
||||
print(f'Merged CSV files saved to: {output_file}')
|
||||
|
||||
|
||||
def extract_data_from_merged_file(merged_file):
|
||||
data = []
|
||||
with open(merged_file, 'r') as file:
|
||||
csv_reader = csv.reader(file)
|
||||
header = next(csv_reader) # Get the header row
|
||||
for i, row in enumerate(csv_reader, start=1):
|
||||
if i in (365, 750):
|
||||
extracted_row = []
|
||||
for j, value in enumerate(row[1:], start=1):
|
||||
column_name = header[j]
|
||||
extracted_row.append((column_name, value))
|
||||
data.append(extracted_row)
|
||||
return data
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Create a file dialog to select multiple CSV files
|
||||
root = Tk()
|
||||
root.withdraw()
|
||||
input_files = filedialog.askopenfilenames(title='Select CSV files', filetypes=(('CSV files', '*.csv'),))
|
||||
|
||||
# Ensure at least one file is selected
|
||||
if input_files:
|
||||
# Specify the output file path
|
||||
output_file = 'merged.csv'
|
||||
|
||||
merge_csv_files(input_files, output_file)
|
||||
|
||||
extracted_data = extract_data_from_merged_file(output_file)
|
||||
|
||||
for row in extracted_data:
|
||||
print(row)
|
||||
else:
|
||||
print('No files selected. Program will exit.')
|
||||
27
src/App.js
27
src/App.js
@@ -7,14 +7,27 @@ import { getAnalytics } from "firebase/analytics";
|
||||
import { getFirestore, collection, getDocs, doc, onSnapshot, query } from "firebase/firestore";
|
||||
import Card from './components/Card';
|
||||
|
||||
// dev and test
|
||||
// const firebaseConfig = {
|
||||
// apiKey: "AIzaSyBFl_YkJ5PMIvMY3G70313SyrqemjFnUv8",
|
||||
// authDomain: "hpos-af3cc.firebaseapp.com",
|
||||
// projectId: "hpos-af3cc",
|
||||
// storageBucket: "hpos-af3cc.appspot.com",
|
||||
// messagingSenderId: "650071678820",
|
||||
// appId: "1:650071678820:web:ce50538bf46632d46c6471",
|
||||
// measurementId: "G-6JHQ47S9DD"
|
||||
// };
|
||||
|
||||
|
||||
// prod
|
||||
const firebaseConfig = {
|
||||
apiKey: "AIzaSyBFl_YkJ5PMIvMY3G70313SyrqemjFnUv8",
|
||||
authDomain: "hpos-af3cc.firebaseapp.com",
|
||||
projectId: "hpos-af3cc",
|
||||
storageBucket: "hpos-af3cc.appspot.com",
|
||||
messagingSenderId: "650071678820",
|
||||
appId: "1:650071678820:web:ce50538bf46632d46c6471",
|
||||
measurementId: "G-6JHQ47S9DD"
|
||||
apiKey: "AIzaSyBF4pLZReAedU4dWX1eRJNFNIaOIz2xk4s",
|
||||
authDomain: "hpos-prod.firebaseapp.com",
|
||||
projectId: "hpos-prod",
|
||||
storageBucket: "hpos-prod.appspot.com",
|
||||
messagingSenderId: "1012719870714",
|
||||
appId: "1:1012719870714:web:fcd00acef93f6612de5f43",
|
||||
measurementId: "G-B0NZ7DEF69"
|
||||
};
|
||||
|
||||
const app = initializeApp(firebaseConfig);
|
||||
|
||||
Reference in New Issue
Block a user