csv data analysis for single file and pie chart

scripts/single_csv.py

@@ -0,0 +1,125 @@
+import glob
+import pandas as pd
+import openpyxl
+import os
+from tqdm import tqdm
+import platform
+from PyQt5.QtWidgets import QApplication, QWidget, QFileDialog
+
+def consolidate_and_perform_calculations(curdir, rootdir, path_delim, validation_file):
+
+    print(os.path.split(curdir)[1])
+    
+    # list all csv files only  
+    csv_files = glob.glob(curdir + path_delim + '/*.{}'.format('csv'))
+    if len(csv_files) == 0:
+        print("no csv files in the sub folder")
+    # print(csv_files)
+
+    df_csv_append = pd.DataFrame()
+    
+    first = True
+
+    # merge the CSV files
+    for file in csv_files:
+        if first:
+            df_csv_append = pd.read_csv(file)
+            colname = file.split('.')[0].split(path_delim)[-1] #file.split('.')[0]
+            df_csv_append.rename(columns={'ca': colname}, inplace = True)
+            df_csv_append = df_csv_append.drop(['1'], axis=1)
+            first = False
+        else:
+            df = pd.read_csv(file)
+            colname = file.split('.')[0].split(path_delim)[-1] #file.split('.')[0]
+            df.rename(columns={'ca': colname}, inplace = True)
+            df = df.drop(['1'], axis=1)
+            df_csv_append = df_csv_append.merge(df, on='tv')
+    
+    df_csv_append = df_csv_append[df_csv_append['tv'].between(300, 700)]
+
+    wavelength_col = "123_tv" # to make sorting columns simpler
+
+    df_csv_append.rename(columns={'tv': wavelength_col}, inplace = True)
+
+    df_csv_append = df_csv_append.reindex(sorted(df_csv_append.columns), axis=1)
+
+    outfile = rootdir + path_delim + os.path.split(curdir)[1] + "_analysis.xlsx"
+
+    # df_csv_append.to_excel(outfile, sheet_name="merged_data", index=False)
+    # df_csv_append.to_csv("D8 Merged.csv", index=False)
+
+    # calculations
+    df_427 = df_csv_append.loc[(df_csv_append[wavelength_col] >= 427) & (df_csv_append[wavelength_col] < 428)]
+
+    df_555 = df_csv_append.loc[(df_csv_append[wavelength_col] >= 555) & (df_csv_append[wavelength_col] < 556)]
+
+    df_validation = pd.read_excel(validation_file)
+
+    df_analysis = df_427.iloc[0] + df_555.iloc[0]
+    # print(df_analysis)
+    # print(min(df_csv_append[0:5]))
+
+    midpoint1 = 427
+    midpoint2 = 555
+    bandwidth1 = 25
+    bandwidth2 = 10
+
+    df = df_analysis.rename(columns = {"NM":"Wavelength","CA":"Absorbance"}, inplace = True)
+
+    # 427 nm range
+    df1 = df[ (df['Wavelength'] > (midpoint1-bandwidth1)) & (df['Wavelength'] < (midpoint1+bandwidth1)) ]
+
+    # 555 nm range
+    df2 = df[ (df['Wavelength'] > (midpoint2-bandwidth2)) & (df['Wavelength'] < (midpoint2+bandwidth2)) ]
+    
+    procData.append({"Sample ID": sampleID, 
+                     "max_427": round(df1["Absorbance"].max() , 3),
+                     "wvmax_427": round(df1.at[df1["Absorbance"].idxmax(),"Wavelength"], 3),
+                     "avg_427": round(df1["Absorbance"].mean(), 3),
+                     "max_555": round(df2["Absorbance"].max(), 3),
+                     "wvmax_555": round(df2.at[df2["Absorbance"].idxmax(),"Wavelength"], 3),
+                     "avg_555": round(df2["Absorbance"].mean(), 3),
+                     "ratio_max": round(df2["Absorbance"].max()/df1["Absorbance"].max(), 3),
+                     "ratio_avg": round(df2["Absorbance"].mean()/df1["Absorbance"].mean(), 3)
+                    })
+
+    writer = pd.ExcelWriter(outfile, engine = 'openpyxl')
+    df_analysis.to_excel(writer, sheet_name = 'analysis', index=False)
+    df_csv_append.to_excel(writer, sheet_name = 'merged_data', index=False)
+    writer.close()
+
+
+sampleID = 1
+allDF = pd.DataFrame()
+procData = []
+
+
+if __name__ == "__main__":
+
+    file = os.getcwd() + "/D26-100umol-1.csv"
+    df = pd.read_csv(file)
+    
+    midpoint1 = 427
+    midpoint2 = 555
+    bandwidth1 = 25
+    bandwidth2 = 10
+
+    df.rename(columns = {"tv":"Wavelength","ca":"Absorbance"}, inplace = True)
+
+    # 427 nm range
+    df1 = df[ (df['Wavelength'] > (midpoint1-bandwidth1)) & (df['Wavelength'] < (midpoint1+bandwidth1)) ]
+
+    # 555 nm range
+    df2 = df[ (df['Wavelength'] > (midpoint2-bandwidth2)) & (df['Wavelength'] < (midpoint2+bandwidth2)) ]
+    
+    procData.append({"Sample ID": sampleID, 
+                     "max_427": round(df1["Absorbance"].max() , 3),
+                     "wvmax_427": round(df1.at[df1["Absorbance"].idxmax(),"Wavelength"], 3),
+                     "avg_427": round(df1["Absorbance"].mean(), 3),
+                     "max_555": round(df2["Absorbance"].max(), 3),
+                     "wvmax_555": round(df2.at[df2["Absorbance"].idxmax(),"Wavelength"], 3),
+                     "avg_555": round(df2["Absorbance"].mean(), 3),
+                     "ratio_max": round(df2["Absorbance"].max()/df1["Absorbance"].max(), 3),
+                     "ratio_avg": round(df2["Absorbance"].mean()/df1["Absorbance"].mean(), 3)
+                    })
+    print(procData)

src/components/TestAnalytics.css

@@ -0,0 +1,6 @@
+.container {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    justify-content: center;
+}

src/components/TestAnalytics.js

@@ -8,7 +8,7 @@ import Select from 'react-select';
 
 import DeviceGraph from "./DeviceGraph";
 import db from '../firebase';
-import './Devices.css';
+import './TestAnalytics.css';
 import TestAnalyticsGraph from './TestAnalyticsGraph';
 import TestAnalyticsPie from './TestAnalyticsPie';
 
@@ -43,10 +43,10 @@ const TestAnalytics = () => {
     const [allTests, setAllTests] = useState(null);
     const [showGraph, setShowGraph] = useState(false);
     const [dailyResults, setDailyResults] = useState(null);
+    const [curDayTests, setCurDayTests] = useState(null);
+    const [dailyCategoryCounts, setDailyCategoryCounts] = useState(null);
 
     const changeDevice = ({ value }) => {
-        // console.log(event);
-
         setDevice(value);
         setDays([...new Set(allTests
             .filter((x) => {
@@ -101,14 +101,11 @@ const TestAnalytics = () => {
                     hposTests.push(hposTestData);
                     const { deviceSerialNumber } = hposTestData;
                     testDevices.add(deviceSerialNumber);
-                    // console.log(document.data());
                 });
                 const groups = [...testDevices];
-                // console.log(groups);
                 setDevices(groups);
                 setAllTests(hposTests);
 
-                // let rsult;
                 const dailyCounts = hposTests.reduce(function (result, test) {
                     const day = moment(test.testTime).format("YYYY-MM-DD");
                     if (!result[day]) {
@@ -125,6 +122,20 @@ const TestAnalytics = () => {
                     return moment(test.testTime).startOf('day').format();
                 });
                 // setDailyResults(rss);
+                const todaysTests = hposTests.filter((x) => moment(x.testTime).isBetween(moment().startOf('day'), moment().endOf('day')));
+                setCurDayTests(todaysTests);
+                const dailyCategoryCounts = todaysTests.reduce(function (result, test) {
+                    const category = test.result;
+                    if (!result[category]) {
+                        result[category] = 0;
+                    }
+                    result[category]++;
+                    return result;
+                }, {});
+
+                setDailyCategoryCounts(Object.entries(dailyCategoryCounts).sort().map(x => {
+                    return { category: x[0], count: x[1] };
+                }));
             });
 
             return () => unsub;
@@ -142,7 +153,10 @@ const TestAnalytics = () => {
                 <button className='clear-button' onClick={toggleShowGraph}><img className="graph-icon" src={require('./graph_icon.png')}></img>{curTestData ? 'Clear' : 'Show'}</button>
             </div> */}
 
-            {dailyResults && <TestAnalyticsGraph data={dailyResults} />}
+            <div className='container'>
+                {dailyCategoryCounts && <TestAnalyticsPie data={dailyCategoryCounts} />}
+                {dailyResults && <TestAnalyticsGraph data={dailyResults} />}
+            </div>
         </div>
     )
 }

src/components/TestAnalyticsGraph.css

@@ -0,0 +1,17 @@
+.line {
+    fill: none;
+    stroke: #009EDC;
+    stroke-width: 1px;
+ }
+
+ .tests-count-card {
+    width: 80vw;
+    background-color: white;
+    margin: 10px;
+ }
+
+ .graph-title {
+    text-align: center;
+    font-weight: 600;
+    padding: 15px;
+ }

src/components/TestAnalyticsGraph.js

@@ -2,7 +2,7 @@ import React, { Component } from 'react';
 import * as d3 from 'd3';
 import moment from 'moment';
 
-import './DeviceGraph.css';
+import './TestAnalyticsGraph.css';
 
 class TestAnalyticsGraph extends Component {
     constructor(props) {
@@ -28,7 +28,7 @@ class TestAnalyticsGraph extends Component {
             .x(function (d) { return x(moment(d.date)); })
             .y(function (d) { return y(d.count); });
 
-        var svg = d3.select("#result-graph").append("svg")
+        var svg = d3.select("#tests-count-graph").append("svg")
             .attr("width", width + margin.left + margin.right)
             .attr("height", height + margin.top + margin.bottom)
             .append("g").attr("transform",
@@ -84,8 +84,8 @@ class TestAnalyticsGraph extends Component {
     render() {
         return (
             <div>
-                <div className='graph-card'>
+                <div className='tests-count-card'>
-                    <div id="result-graph" style={{ margin: '1em' }}>
+                    <div id="tests-count-graph" style={{ margin: '1em' }}>
                     </div>
                     <div className='graph-title'>test counts per day</div>
                 </div>

src/components/TestAnalyticsPie.css

@@ -0,0 +1,23 @@
+.line {
+    fill: none;
+    stroke: #009EDC;
+    stroke-width: 1px;
+ }
+
+ .graph-card {
+    width: 80vw;
+    height: 80vh;
+    background-color: white;
+    margin: 10px;
+ }
+
+ .graph-title {
+    text-align: center;
+    font-weight: 600;
+    padding: 15px;
+ }
+
+ .title {
+   fill: teal;
+   font-weight: bold;
+}

src/components/TestAnalyticsPie.js

@@ -4,7 +4,7 @@ import { sgg } from 'ml-savitzky-golay-generalized';
 import axios from 'axios';
 import moment from 'moment';
 
-import './DeviceGraph.css';
+import './TestAnalyticsPie.css';
 
 class TestAnalyticsPie extends Component {
     constructor(props) {
@@ -15,128 +15,73 @@ class TestAnalyticsPie extends Component {
         this.drawChart(this.state.data);
     }
 
-    drawChart(data2) {
+    drawChart(data) {
-        // if (!data) return;
+        if (!data) return;
-        // var margin = { top: 20, right: 20, bottom: 30, left: 50 },
+        
-        //     width = 960 - margin.left - margin.right,
+        var margin = { top: 60, right: 20, bottom: 30, left: 50 },
-        //     height = 500 - margin.top - margin.bottom;
+            width = 960 - margin.left - margin.right,
+            height = 500 - margin.top - margin.bottom,
+            radius = Math.min(width, height) / 2;
 
-        // // set the ranges
+        var svg = d3.select("#category-diagram").append('svg')
-        // var x = d3.scaleTime().range([0, width]);
-        // var y = d3.scaleLinear().range([height, 0]);
-
-        // // define the line
-        // var valueline = d3.line()
-        //     .x(function (d) { return x(moment(d.date)); })
-        //     .y(function (d) { return y(d.count); });
-
-        // var svg = d3.select("#result-graph").append("svg")
-        //     .attr("width", width + margin.left + margin.right)
-        //     .attr("height", height + margin.top + margin.bottom)
-        //     .append("g").attr("transform",
-        //         "translate(" + margin.left + "," + margin.top + ")");
-        // var g = svg.append("g").attr("transform", "translate(" + margin.left + "," + margin.top + ")");
-
-        // // const options = {
-        // //     windowSize: 5,
-        // //     derivative: 0,
-        // //     polynomial: 3,
-        // // };
-        // // const sggResult = sgg(data.map(x => x.CA), (Math.PI * 2) / data.length, options);
-
-        // // data.forEach(function (d, index) {
-        // //     d.NM = d.NM;
-        // //     d.CA = sggResult[index];
-        // // });
-        // // data.forEach(function (d) {
-        // //     d.date = d.date;
-        // //     d.count = +d.count;
-        // // });
-
-        // x.domain(d3.extent(data, function (d) { return moment(d.date); }));
-        // y.domain([0, d3.max(data, function (d) { return d.count; })]);
-
-        // svg.append("path")
-        //     .data([data])
-        //     .attr("class", "line")
-        //     .attr("d", valueline);
-
-        // svg.append("g")
-        //     .attr("transform", "translate(0," + height + ")")
-        //     .call(d3.axisBottom(x))
-        //     .append("text")
-        //     // .attr("transform", "rotate(-90)")
-        //     .attr("x", 400)
-        //     .attr("y", 30)
-        //     .attr("dx", "0.71em")
-        //     .attr("fill", "#000")
-        //     .text("Test time");
-
-        // svg.append("g")
-        //     .call(d3.axisLeft(y))
-        //     .append("text")
-        //     .attr("transform", "rotate(-90)")
-        //     .attr("x", -160)
-        //     .attr("y", -36)
-        //     .attr("dy", "0.71em")
-        //     .attr("fill", "#000")
-        //     .text("Tests Count");
-
-
-
-        var data = [2, 4, 8, 10];
-
-        var margin = { top: 50, right: 10, bottom: 15, left: 50 },
-            width = 400 - margin.left - margin.right,
-            height = 200 - margin.top - margin.bottom;
-        var radius = Math.min(width, height) / 2;
-
-
-        // var svg = d3.select("#result-graph").append("svg"),
-        //     width = svg.attr("width"),
-        //     height = svg.attr("height"),
-        //     radius = Math.min(width, height) / 2,
-        //     g = svg.append("g").attr("transform", "translate(" + width / 2 + "," + height / 2 + ")");
-
-        var svg = d3.select("#result-graph").append("svg")
             .attr("width", width + margin.left + margin.right)
             .attr("height", height + margin.top + margin.bottom)
             .append("g").attr("transform",
                 "translate(" + margin.left + "," + margin.top + ")");
-        var g = svg.append("g").attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+        var g = svg.append("g")
+            .attr("transform", "translate(" + width / 2 + "," + height / 2 + ")");
 
         var color = d3.scaleOrdinal(['#4daf4a', '#377eb8', '#ff7f00', '#984ea3', '#e41a1c']);
 
-        // Generate the pie
+        var pie = d3.pie().value(function (d) {
-        var pie = d3.pie();
+            return d.count;
+        });
 
-        // Generate the arcs
+        var path = d3.arc()
-        var arc = d3.arc()
+            .outerRadius(radius - 10)
-            .innerRadius(0)
+            // .innerRadius(0);
-            .outerRadius(radius);
+            .innerRadius(100);
 
-        //Generate groups
+        var label = d3.arc()
-        var arcs = g.selectAll("arc")
+            .outerRadius(radius)
+            .innerRadius(radius - 80);
+
+        // d3.csv("browseruse.csv", function(error, data) {
+        //     if (error) {
+        //         throw error;
+        //     }
+        var arc = g.selectAll(".arc")
             .data(pie(data))
-            .enter()
+            .enter().append("g")
-            .append("g")
+            .attr("class", "arc");
-            .attr("class", "arc")
 
-        //Draw arc paths
+        arc.append("path")
-        arcs.append("path")
+            .attr("d", path)
-            .attr("fill", function (d, i) {
+            .attr("fill", function (d) { return color(d.data.category); });
-                return color(i);
+
+        console.log(arc)
+
+        arc.append("text")
+            .attr("transform", function (d) {
+                return "translate(" + label.centroid(d) + ")";
             })
-            .attr("d", arc);
+            .text(function (d) { return `${d.data.category} (${d.data.count})`; });
+
+        // svg.append("g")
+        //     .attr("transform", "translate(" + (width / 2 - 170) + "," + 1 + ")")
+        //     .append("text")
+        //     .text("category counts per day")
+        //     .attr("class", "title")
     }
 
     render() {
         return (
             <div>
                 <div className='graph-card'>
-                    <div id="result-graph" style={{ margin: '1em' }}>
+                    <div id="category-diagram" style={{ margin: '1em' }}>
                     </div>
-                    <div className='graph-title'>test counts per day</div>
+                    <div className='graph-title'>category data on daily basis</div>
                 </div>
             </div>
         )