diff --git a/cloud-functions/python/functions/performance/main.py b/cloud-functions/python/functions/performance/main.py new file mode 100644 index 0000000..a7dd67c --- /dev/null +++ b/cloud-functions/python/functions/performance/main.py @@ -0,0 +1,276 @@ +import functions_framework +from firebase_functions import https_fn +# import firebase_admin +from firebase_admin import initialize_app, firestore +import pandas as pd +import numpy as np +import datetime +import sys +import platform +import os +import io +from flask import send_file +from datetime import datetime +import re +import xlsxwriter +import math +import matplotlib.pyplot as plt + +initialize_app() + +@functions_framework.http +def performance(request): + """HTTP Cloud Function. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + + if request.method == 'OPTIONS': + headers = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET', + 'Access-Control-Allow-Headers': 'Content-Type', + # 'Access-Control-Max-Age': '3600' + } + + return ('', 204, headers) + + headers = { + 'Access-Control-Allow-Origin': '*' + } + + request_json = request.get_json(silent=True) + request_args = request.args + + if request_json and 'start_date' in request_json and 'end_date' in request_json: + start_date = request_json['start_date'] + end_date = request_json['end_date'] + elif request_args and 'start_date' in request_args and 'end_date' in request_args: + start_date = request_args['start_date'] + end_date = request_args['end_date'] + else: + start_date = "2023-12-04" + end_date = "2023-12-05" + + path_delim = "/" + + db = firestore.client() + + test_collection = db.collection("testData") + + data = [] + + test_docs = test_collection.stream() + for test_doc in test_docs: + test_data = test_doc.to_dict() + data.append(test_data) + + df = pd.DataFrame(data) + + df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)] + df = df.sort_values(by=['testTime'], ascending=False) + # df = df.drop(['resultData', "reportUploadTime", "userImageURL", "testType", "birthYear", "testStatus", "reportPath", "createdBy", "csvPath", "result", "mobileId", "resultRatio", "localFlag", "led2", "led1"], axis=1) + df = df[["_id", "classificationResult", "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "abs3", "led3Average", "led3Buffer", "led3Sample", "abs4", "led4Average", "led4Buffer", "led4Sample", "deviceId", "deviceSerialNumber", "name", "testTime"]] + df.rename(columns={'deviceSerialNumber': "loginId"}, inplace = True) + # df = df.reindex(sorted(df.columns), axis=1) + + df = df.drop_duplicates() + + curdir = os.getcwd() + path_delim = '/' + + precision_threshold = 0.009 + accuracy_threshold = 0.02 + + denovix_reference_values = { + "Tartrazine": { + "30": 0.187191676, + "50": 0.187447015, + "60": 0.187191676, + "75": 0.294739334, + "90": 0.187191676, + "100": 0.410113264, + "120": 0.187191676, + "125": 0.512526022, + "150": 0.622883833, + "175": 0.721095085, + "180": 0.187191676, + "200": 0.824065454, + "210": 0.187191676, + "225": 0.931385567, + "250": 1.002224884, + "275": 1.091996882, + "300": 1.16558307, + }, + "KMnO4": { + "145": 0.187191676, + "250": 0.077846397, + "290": 0.359288533, + "350": 0.111884606, + "435": 0.187191676, + "450": 0.138031952, + "550": 0.175111257, + "580": 0.628039375, + "650": 0.208337398, + "720": 0.187191676, + "750": 0.238773222, + "850": 0.267574903, + "950": 0.299568449, + "1050": 0.329271864, + "1150": 0.365827844, + "1160": 0.388015579, + "1250": 0.388015579, + "1350": 0.417304075, + "1450": 0.447634285, + "1550": 0.478220085, + }, + } + df_reference_device = pd.DataFrame(denovix_reference_values).T + df_reference_device.index.name = 'Solution' + df_reference_device.columns.name = 'Wavelength' + + texts_to_check = ['Tar', 'KM'] + df['solution'] = df['name'].str.extract(f"({'|'.join(texts_to_check)})", flags=re.IGNORECASE) + df['solution'] = df['solution'].replace({'Tar': 'Tartrazine', 'KM': 'KMnO4'}, regex=True) + + def extract_numbers(s): + match = re.match(r'\d+', s) + if match: + return int(match.group()) + else: + return None + + df["concentration"] = df['name'].apply(extract_numbers) + df['absorbance'] = df.apply(lambda row: row['led1Average'] if row['solution'] == 'Tartrazine' else row['led3Average'], axis=1) + + fig, ax = plt.subplots() + + unique_solutions = df['solution'].unique() + + for solution in unique_solutions: + solution_data = df[df['solution'] == solution] + + concentration_means = solution_data.groupby('concentration')['absorbance'].mean() + + ax.plot(concentration_means.index, concentration_means.values, marker='o', linestyle='-', label=f'Mean Absorbance for {solution}') + + ax.set_title('Mean Absorbance for Each Solution') + ax.set_xlabel('Concentration') + ax.set_ylabel('Mean Absorbance') + ax.legend() + + df_precision_acc = df[["deviceId", "solution", "concentration", "led1Average", "led3Average", "absorbance"]].groupby(["deviceId", "solution", "concentration"]).describe()["absorbance"][["count", "min", "max", "mean"]] + df_precision_acc['precision'] = df_precision_acc['max'] - df_precision_acc['min'] + + df_precision_acc['precision_result'] = ['Fail' if diff > precision_threshold else 'Pass' for diff in df_precision_acc['precision']] + + df_precision_acc.reset_index(inplace=True) + df_precision_acc.set_index(["deviceId", "solution", "concentration"], inplace=True) + + device_precision_results = {} + for index, group_df in df_precision_acc.groupby(level=[0, 1, 2]): + soln = index[1] + concen = index[2] + result_values = group_df['precision_result'].values + + if index[0] not in device_precision_results: + device_precision_results[index[0]] = "Pass" + else: + result = 'Fail' if 'Fail' in result_values else 'Pass' + if result == 'Fail': + device_precision_results[index[0]] = 'Fail' + + df_device_precicion_results = pd.DataFrame(list(device_precision_results.items()), columns=['Device', 'Result']) + + df_precision_acc = df_precision_acc.assign(accuracy='', accuracy_result='') + + device_accuracy_results = {} + + for idx, row in df_precision_acc.iterrows(): + device_id = row.name[0] + solution = row.name[1] + concentration = str(int(row.name[2])) + reference_value = denovix_reference_values[solution][concentration] + + try: + reference_value = denovix_reference_values[solution][concentration] + accuracy = math.fabs(row['mean'] - reference_value) + df_precision_acc.at[idx, 'accuracy'] = accuracy + + accuracy_result = 'Pass' if accuracy < accuracy_threshold else 'Fail' + df_precision_acc.at[idx, 'accuracy_result'] = accuracy_result + if device_id not in device_accuracy_results: + device_accuracy_results[device_id] = accuracy_result + else: + if accuracy_result == 'Fail': + device_accuracy_results[device_id] = 'Fail' + except KeyError: + df_precision_acc.at[idx, 'accuracy'] = np.nan + df_precision_acc.at[idx, 'accuracy_result'] = 'Fail' # Assume 'Fail' for missing data + + df_device_accuracy_results = pd.DataFrame(list(device_accuracy_results.items()), columns=['Device', 'Result']) + + device_results = {} + + df_device_results = pd.merge(df_device_precicion_results, df_device_accuracy_results, on='Device', how='outer', suffixes=('_precision', '_accuracy')) + df_device_results['Result'] = np.where((df_device_results['Result_precision'] == 'Fail') | (df_device_results['Result_accuracy'] == 'Fail'), 'Fail', 'Pass') + + output_filename = f'/tmp/accuracy_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx' + output_path = os.getcwd() + path_delim + output_filename + print('output_path', output_path) + writer = pd.ExcelWriter(output_filename, engine = 'xlsxwriter') + df_device_results.to_excel(writer, sheet_name="device_results") + # df_device_precicion_results.to_excel(writer, sheet_name="device_precision") + # df_device_accuracy_results.to_excel(writer, sheet_name="device_accuracy") + df_precision_acc.to_excel(writer, sheet_name="concentration") + df.to_excel(writer, sheet_name="in") + df_reference_device.to_excel(writer, sheet_name="reference_device") + + workbook = writer.book + worksheet_device_results = writer.sheets['device_results'] + + red_format = workbook.add_format({'bg_color': '#FFC7CE', 'font_color': '#9C0006'}) + green_format = workbook.add_format({'bg_color': '#C6EFCE', 'font_color': '#006100'}) + + worksheet_device_results.conditional_format('E2:E{}'.format(df_device_results.shape[0] + 1), {'type': 'text', + 'criteria': 'containing', + 'value': 'Fail', + 'format': red_format}) + + worksheet_device_results.conditional_format('E2:E{}'.format(df_device_results.shape[0] + 1), {'type': 'text', + 'criteria': 'containing', + 'value': 'Pass', + 'format': green_format}) + + wks1 = workbook.add_worksheet('abs_plot') + wks1.write(0,0,'Abs Plot') + + imgdata=io.BytesIO() + fig.savefig(imgdata, format='png') + wks1.insert_image(2,2, '', {'image_data': imgdata}) + + + ### debug each row + # # Merge df and df_device_precicion_results on deviceId + # df_merged = pd.merge(df, df_device_precicion_results, left_on='deviceId', right_on='Device', how='left') + + # # Drop the duplicate "Device" column + # df_merged.drop(columns=['Device'], inplace=True) + + # # Save the merged DataFrame to the Excel file + # df_merged.to_excel(writer, sheet_name="merged_results") + + writer.close() + + # firebase_admin.delete_app(firebase_admin.get_app()) + + with open(output_filename,'rb') as f: + file_data = io.BytesIO(f.read()) + + # application/vnd.ms-excel + return send_file(file_data, download_name= output_filename, mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") \ No newline at end of file diff --git a/cloud-functions/python/functions/performance/requirements.txt b/cloud-functions/python/functions/performance/requirements.txt new file mode 100644 index 0000000..7fa0417 --- /dev/null +++ b/cloud-functions/python/functions/performance/requirements.txt @@ -0,0 +1,7 @@ +functions-framework==3.* +firebase_functions~=0.1.0 +pandas==2.0.3 +openpyxl==3.1.2 +firebase-admin==6.2.0 +XlsxWriter==3.1.2 +matplotlib==3.8.0 \ No newline at end of file