csv data analysis for single file and pie chart
This commit is contained in:
prisar
125
scripts/single_csv.py
Normal file
125
scripts/single_csv.py
Normal file
@@ -0,0 +1,125 @@
|
||||
import glob
|
||||
import pandas as pd
|
||||
import openpyxl
|
||||
import os
|
||||
from tqdm import tqdm
|
||||
import platform
|
||||
from PyQt5.QtWidgets import QApplication, QWidget, QFileDialog
|
||||
|
||||
def consolidate_and_perform_calculations(curdir, rootdir, path_delim, validation_file):
|
||||
|
||||
print(os.path.split(curdir)[1])
|
||||
|
||||
# list all csv files only
|
||||
csv_files = glob.glob(curdir + path_delim + '/*.{}'.format('csv'))
|
||||
if len(csv_files) == 0:
|
||||
print("no csv files in the sub folder")
|
||||
# print(csv_files)
|
||||
|
||||
df_csv_append = pd.DataFrame()
|
||||
|
||||
first = True
|
||||
|
||||
# merge the CSV files
|
||||
for file in csv_files:
|
||||
if first:
|
||||
df_csv_append = pd.read_csv(file)
|
||||
colname = file.split('.')[0].split(path_delim)[-1] #file.split('.')[0]
|
||||
df_csv_append.rename(columns={'ca': colname}, inplace = True)
|
||||
df_csv_append = df_csv_append.drop(['1'], axis=1)
|
||||
first = False
|
||||
else:
|
||||
df = pd.read_csv(file)
|
||||
colname = file.split('.')[0].split(path_delim)[-1] #file.split('.')[0]
|
||||
df.rename(columns={'ca': colname}, inplace = True)
|
||||
df = df.drop(['1'], axis=1)
|
||||
df_csv_append = df_csv_append.merge(df, on='tv')
|
||||
|
||||
df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
|
||||
|
||||
wavelength_col = "123_tv" # to make sorting columns simpler
|
||||
|
||||
df_csv_append.rename(columns={'tv': wavelength_col}, inplace = True)
|
||||
|
||||
df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
|
||||
|
||||
outfile = rootdir + path_delim + os.path.split(curdir)[1] + "_analysis.xlsx"
|
||||
|
||||
# df_csv_append.to_excel(outfile, sheet_name="merged_data", index=False)
|
||||
# df_csv_append.to_csv("D8 Merged.csv", index=False)
|
||||
|
||||
# calculations
|
||||
df_427 = df_csv_append.loc[(df_csv_append[wavelength_col] >= 427) & (df_csv_append[wavelength_col] < 428)]
|
||||
|
||||
df_555 = df_csv_append.loc[(df_csv_append[wavelength_col] >= 555) & (df_csv_append[wavelength_col] < 556)]
|
||||
|
||||
df_validation = pd.read_excel(validation_file)
|
||||
|
||||
df_analysis = df_427.iloc[0] + df_555.iloc[0]
|
||||
# print(df_analysis)
|
||||
# print(min(df_csv_append[0:5]))
|
||||
|
||||
midpoint1 = 427
|
||||
midpoint2 = 555
|
||||
bandwidth1 = 25
|
||||
bandwidth2 = 10
|
||||
|
||||
df = df_analysis.rename(columns = {"NM":"Wavelength","CA":"Absorbance"}, inplace = True)
|
||||
|
||||
# 427 nm range
|
||||
df1 = df[ (df['Wavelength'] > (midpoint1-bandwidth1)) & (df['Wavelength'] < (midpoint1+bandwidth1)) ]
|
||||
|
||||
# 555 nm range
|
||||
df2 = df[ (df['Wavelength'] > (midpoint2-bandwidth2)) & (df['Wavelength'] < (midpoint2+bandwidth2)) ]
|
||||
|
||||
procData.append({"Sample ID": sampleID,
|
||||
"max_427": round(df1["Absorbance"].max() , 3),
|
||||
"wvmax_427": round(df1.at[df1["Absorbance"].idxmax(),"Wavelength"], 3),
|
||||
"avg_427": round(df1["Absorbance"].mean(), 3),
|
||||
"max_555": round(df2["Absorbance"].max(), 3),
|
||||
"wvmax_555": round(df2.at[df2["Absorbance"].idxmax(),"Wavelength"], 3),
|
||||
"avg_555": round(df2["Absorbance"].mean(), 3),
|
||||
"ratio_max": round(df2["Absorbance"].max()/df1["Absorbance"].max(), 3),
|
||||
"ratio_avg": round(df2["Absorbance"].mean()/df1["Absorbance"].mean(), 3)
|
||||
})
|
||||
|
||||
writer = pd.ExcelWriter(outfile, engine = 'openpyxl')
|
||||
df_analysis.to_excel(writer, sheet_name = 'analysis', index=False)
|
||||
df_csv_append.to_excel(writer, sheet_name = 'merged_data', index=False)
|
||||
writer.close()
|
||||
|
||||
|
||||
sampleID = 1
|
||||
allDF = pd.DataFrame()
|
||||
procData = []
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
file = os.getcwd() + "/D26-100umol-1.csv"
|
||||
df = pd.read_csv(file)
|
||||
|
||||
midpoint1 = 427
|
||||
midpoint2 = 555
|
||||
bandwidth1 = 25
|
||||
bandwidth2 = 10
|
||||
|
||||
df.rename(columns = {"tv":"Wavelength","ca":"Absorbance"}, inplace = True)
|
||||
|
||||
# 427 nm range
|
||||
df1 = df[ (df['Wavelength'] > (midpoint1-bandwidth1)) & (df['Wavelength'] < (midpoint1+bandwidth1)) ]
|
||||
|
||||
# 555 nm range
|
||||
df2 = df[ (df['Wavelength'] > (midpoint2-bandwidth2)) & (df['Wavelength'] < (midpoint2+bandwidth2)) ]
|
||||
|
||||
procData.append({"Sample ID": sampleID,
|
||||
"max_427": round(df1["Absorbance"].max() , 3),
|
||||
"wvmax_427": round(df1.at[df1["Absorbance"].idxmax(),"Wavelength"], 3),
|
||||
"avg_427": round(df1["Absorbance"].mean(), 3),
|
||||
"max_555": round(df2["Absorbance"].max(), 3),
|
||||
"wvmax_555": round(df2.at[df2["Absorbance"].idxmax(),"Wavelength"], 3),
|
||||
"avg_555": round(df2["Absorbance"].mean(), 3),
|
||||
"ratio_max": round(df2["Absorbance"].max()/df1["Absorbance"].max(), 3),
|
||||
"ratio_avg": round(df2["Absorbance"].mean()/df1["Absorbance"].mean(), 3)
|
||||
})
|
||||
print(procData)
|
||||
6
src/components/TestAnalytics.css
Normal file
6
src/components/TestAnalytics.css
Normal file
@@ -0,0 +1,6 @@
|
||||
.container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import Select from 'react-select';
|
||||
|
||||
import DeviceGraph from "./DeviceGraph";
|
||||
import db from '../firebase';
|
||||
import './Devices.css';
|
||||
import './TestAnalytics.css';
|
||||
import TestAnalyticsGraph from './TestAnalyticsGraph';
|
||||
import TestAnalyticsPie from './TestAnalyticsPie';
|
||||
|
||||
@@ -43,10 +43,10 @@ const TestAnalytics = () => {
|
||||
const [allTests, setAllTests] = useState(null);
|
||||
const [showGraph, setShowGraph] = useState(false);
|
||||
const [dailyResults, setDailyResults] = useState(null);
|
||||
const [curDayTests, setCurDayTests] = useState(null);
|
||||
const [dailyCategoryCounts, setDailyCategoryCounts] = useState(null);
|
||||
|
||||
const changeDevice = ({ value }) => {
|
||||
// console.log(event);
|
||||
|
||||
setDevice(value);
|
||||
setDays([...new Set(allTests
|
||||
.filter((x) => {
|
||||
@@ -101,14 +101,11 @@ const TestAnalytics = () => {
|
||||
hposTests.push(hposTestData);
|
||||
const { deviceSerialNumber } = hposTestData;
|
||||
testDevices.add(deviceSerialNumber);
|
||||
// console.log(document.data());
|
||||
});
|
||||
const groups = [...testDevices];
|
||||
// console.log(groups);
|
||||
setDevices(groups);
|
||||
setAllTests(hposTests);
|
||||
|
||||
// let rsult;
|
||||
const dailyCounts = hposTests.reduce(function (result, test) {
|
||||
const day = moment(test.testTime).format("YYYY-MM-DD");
|
||||
if (!result[day]) {
|
||||
@@ -125,6 +122,20 @@ const TestAnalytics = () => {
|
||||
return moment(test.testTime).startOf('day').format();
|
||||
});
|
||||
// setDailyResults(rss);
|
||||
const todaysTests = hposTests.filter((x) => moment(x.testTime).isBetween(moment().startOf('day'), moment().endOf('day')));
|
||||
setCurDayTests(todaysTests);
|
||||
const dailyCategoryCounts = todaysTests.reduce(function (result, test) {
|
||||
const category = test.result;
|
||||
if (!result[category]) {
|
||||
result[category] = 0;
|
||||
}
|
||||
result[category]++;
|
||||
return result;
|
||||
}, {});
|
||||
|
||||
setDailyCategoryCounts(Object.entries(dailyCategoryCounts).sort().map(x => {
|
||||
return { category: x[0], count: x[1] };
|
||||
}));
|
||||
});
|
||||
|
||||
return () => unsub;
|
||||
@@ -142,7 +153,10 @@ const TestAnalytics = () => {
|
||||
<button className='clear-button' onClick={toggleShowGraph}><img className="graph-icon" src={require('./graph_icon.png')}></img>{curTestData ? 'Clear' : 'Show'}</button>
|
||||
</div> */}
|
||||
|
||||
{dailyResults && <TestAnalyticsGraph data={dailyResults} />}
|
||||
<div className='container'>
|
||||
{dailyCategoryCounts && <TestAnalyticsPie data={dailyCategoryCounts} />}
|
||||
{dailyResults && <TestAnalyticsGraph data={dailyResults} />}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
17
src/components/TestAnalyticsGraph.css
Normal file
17
src/components/TestAnalyticsGraph.css
Normal file
@@ -0,0 +1,17 @@
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #009EDC;
|
||||
stroke-width: 1px;
|
||||
}
|
||||
|
||||
.tests-count-card {
|
||||
width: 80vw;
|
||||
background-color: white;
|
||||
margin: 10px;
|
||||
}
|
||||
|
||||
.graph-title {
|
||||
text-align: center;
|
||||
font-weight: 600;
|
||||
padding: 15px;
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import React, { Component } from 'react';
|
||||
import * as d3 from 'd3';
|
||||
import moment from 'moment';
|
||||
|
||||
import './DeviceGraph.css';
|
||||
import './TestAnalyticsGraph.css';
|
||||
|
||||
class TestAnalyticsGraph extends Component {
|
||||
constructor(props) {
|
||||
@@ -28,7 +28,7 @@ class TestAnalyticsGraph extends Component {
|
||||
.x(function (d) { return x(moment(d.date)); })
|
||||
.y(function (d) { return y(d.count); });
|
||||
|
||||
var svg = d3.select("#result-graph").append("svg")
|
||||
var svg = d3.select("#tests-count-graph").append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g").attr("transform",
|
||||
@@ -84,8 +84,8 @@ class TestAnalyticsGraph extends Component {
|
||||
render() {
|
||||
return (
|
||||
<div>
|
||||
<div className='graph-card'>
|
||||
<div id="result-graph" style={{ margin: '1em' }}>
|
||||
<div className='tests-count-card'>
|
||||
<div id="tests-count-graph" style={{ margin: '1em' }}>
|
||||
</div>
|
||||
<div className='graph-title'>test counts per day</div>
|
||||
</div>
|
||||
|
||||
23
src/components/TestAnalyticsPie.css
Normal file
23
src/components/TestAnalyticsPie.css
Normal file
@@ -0,0 +1,23 @@
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #009EDC;
|
||||
stroke-width: 1px;
|
||||
}
|
||||
|
||||
.graph-card {
|
||||
width: 80vw;
|
||||
height: 80vh;
|
||||
background-color: white;
|
||||
margin: 10px;
|
||||
}
|
||||
|
||||
.graph-title {
|
||||
text-align: center;
|
||||
font-weight: 600;
|
||||
padding: 15px;
|
||||
}
|
||||
|
||||
.title {
|
||||
fill: teal;
|
||||
font-weight: bold;
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { sgg } from 'ml-savitzky-golay-generalized';
|
||||
import axios from 'axios';
|
||||
import moment from 'moment';
|
||||
|
||||
import './DeviceGraph.css';
|
||||
import './TestAnalyticsPie.css';
|
||||
|
||||
class TestAnalyticsPie extends Component {
|
||||
constructor(props) {
|
||||
@@ -15,128 +15,73 @@ class TestAnalyticsPie extends Component {
|
||||
this.drawChart(this.state.data);
|
||||
}
|
||||
|
||||
drawChart(data2) {
|
||||
// if (!data) return;
|
||||
// var margin = { top: 20, right: 20, bottom: 30, left: 50 },
|
||||
// width = 960 - margin.left - margin.right,
|
||||
// height = 500 - margin.top - margin.bottom;
|
||||
drawChart(data) {
|
||||
if (!data) return;
|
||||
|
||||
var margin = { top: 60, right: 20, bottom: 30, left: 50 },
|
||||
width = 960 - margin.left - margin.right,
|
||||
height = 500 - margin.top - margin.bottom,
|
||||
radius = Math.min(width, height) / 2;
|
||||
|
||||
// // set the ranges
|
||||
// var x = d3.scaleTime().range([0, width]);
|
||||
// var y = d3.scaleLinear().range([height, 0]);
|
||||
|
||||
// // define the line
|
||||
// var valueline = d3.line()
|
||||
// .x(function (d) { return x(moment(d.date)); })
|
||||
// .y(function (d) { return y(d.count); });
|
||||
|
||||
// var svg = d3.select("#result-graph").append("svg")
|
||||
// .attr("width", width + margin.left + margin.right)
|
||||
// .attr("height", height + margin.top + margin.bottom)
|
||||
// .append("g").attr("transform",
|
||||
// "translate(" + margin.left + "," + margin.top + ")");
|
||||
// var g = svg.append("g").attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
// // const options = {
|
||||
// // windowSize: 5,
|
||||
// // derivative: 0,
|
||||
// // polynomial: 3,
|
||||
// // };
|
||||
// // const sggResult = sgg(data.map(x => x.CA), (Math.PI * 2) / data.length, options);
|
||||
|
||||
// // data.forEach(function (d, index) {
|
||||
// // d.NM = d.NM;
|
||||
// // d.CA = sggResult[index];
|
||||
// // });
|
||||
// // data.forEach(function (d) {
|
||||
// // d.date = d.date;
|
||||
// // d.count = +d.count;
|
||||
// // });
|
||||
|
||||
// x.domain(d3.extent(data, function (d) { return moment(d.date); }));
|
||||
// y.domain([0, d3.max(data, function (d) { return d.count; })]);
|
||||
|
||||
// svg.append("path")
|
||||
// .data([data])
|
||||
// .attr("class", "line")
|
||||
// .attr("d", valueline);
|
||||
|
||||
// svg.append("g")
|
||||
// .attr("transform", "translate(0," + height + ")")
|
||||
// .call(d3.axisBottom(x))
|
||||
// .append("text")
|
||||
// // .attr("transform", "rotate(-90)")
|
||||
// .attr("x", 400)
|
||||
// .attr("y", 30)
|
||||
// .attr("dx", "0.71em")
|
||||
// .attr("fill", "#000")
|
||||
// .text("Test time");
|
||||
|
||||
// svg.append("g")
|
||||
// .call(d3.axisLeft(y))
|
||||
// .append("text")
|
||||
// .attr("transform", "rotate(-90)")
|
||||
// .attr("x", -160)
|
||||
// .attr("y", -36)
|
||||
// .attr("dy", "0.71em")
|
||||
// .attr("fill", "#000")
|
||||
// .text("Tests Count");
|
||||
|
||||
|
||||
|
||||
var data = [2, 4, 8, 10];
|
||||
|
||||
var margin = { top: 50, right: 10, bottom: 15, left: 50 },
|
||||
width = 400 - margin.left - margin.right,
|
||||
height = 200 - margin.top - margin.bottom;
|
||||
var radius = Math.min(width, height) / 2;
|
||||
|
||||
|
||||
// var svg = d3.select("#result-graph").append("svg"),
|
||||
// width = svg.attr("width"),
|
||||
// height = svg.attr("height"),
|
||||
// radius = Math.min(width, height) / 2,
|
||||
// g = svg.append("g").attr("transform", "translate(" + width / 2 + "," + height / 2 + ")");
|
||||
|
||||
var svg = d3.select("#result-graph").append("svg")
|
||||
var svg = d3.select("#category-diagram").append('svg')
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g").attr("transform",
|
||||
"translate(" + margin.left + "," + margin.top + ")");
|
||||
var g = svg.append("g").attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
var g = svg.append("g")
|
||||
.attr("transform", "translate(" + width / 2 + "," + height / 2 + ")");
|
||||
|
||||
var color = d3.scaleOrdinal(['#4daf4a', '#377eb8', '#ff7f00', '#984ea3', '#e41a1c']);
|
||||
|
||||
// Generate the pie
|
||||
var pie = d3.pie();
|
||||
var pie = d3.pie().value(function (d) {
|
||||
return d.count;
|
||||
});
|
||||
|
||||
// Generate the arcs
|
||||
var arc = d3.arc()
|
||||
.innerRadius(0)
|
||||
.outerRadius(radius);
|
||||
var path = d3.arc()
|
||||
.outerRadius(radius - 10)
|
||||
// .innerRadius(0);
|
||||
.innerRadius(100);
|
||||
|
||||
//Generate groups
|
||||
var arcs = g.selectAll("arc")
|
||||
var label = d3.arc()
|
||||
.outerRadius(radius)
|
||||
.innerRadius(radius - 80);
|
||||
|
||||
// d3.csv("browseruse.csv", function(error, data) {
|
||||
// if (error) {
|
||||
// throw error;
|
||||
// }
|
||||
var arc = g.selectAll(".arc")
|
||||
.data(pie(data))
|
||||
.enter()
|
||||
.append("g")
|
||||
.attr("class", "arc")
|
||||
.enter().append("g")
|
||||
.attr("class", "arc");
|
||||
|
||||
//Draw arc paths
|
||||
arcs.append("path")
|
||||
.attr("fill", function (d, i) {
|
||||
return color(i);
|
||||
arc.append("path")
|
||||
.attr("d", path)
|
||||
.attr("fill", function (d) { return color(d.data.category); });
|
||||
|
||||
console.log(arc)
|
||||
|
||||
arc.append("text")
|
||||
.attr("transform", function (d) {
|
||||
return "translate(" + label.centroid(d) + ")";
|
||||
})
|
||||
.attr("d", arc);
|
||||
.text(function (d) { return `${d.data.category} (${d.data.count})`; });
|
||||
|
||||
// svg.append("g")
|
||||
// .attr("transform", "translate(" + (width / 2 - 170) + "," + 1 + ")")
|
||||
// .append("text")
|
||||
// .text("category counts per day")
|
||||
// .attr("class", "title")
|
||||
}
|
||||
|
||||
render() {
|
||||
return (
|
||||
<div>
|
||||
<div className='graph-card'>
|
||||
<div id="result-graph" style={{ margin: '1em' }}>
|
||||
<div id="category-diagram" style={{ margin: '1em' }}>
|
||||
</div>
|
||||
<div className='graph-title'>test counts per day</div>
|
||||
<div className='graph-title'>category data on daily basis</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user