rename test data files

scripts/testdata_collection.py

@@ -0,0 +1,76 @@
+import os
+import firebase_admin
+from firebase_admin import credentials
+from firebase_admin import firestore
+from google.cloud.firestore_v1.base_query import FieldFilter
+import pandas as pd
+from datetime import datetime
+import sys
+import matplotlib.pyplot as plt
+
+cred = credentials.Certificate(os.getcwd() + '/' + 'keys/hpos-prod-firebase-adminsdk.json')
+firebase_admin.initialize_app(cred)
+
+db = firestore.client()
+
+patient_collection = db.collection("patientData")
+test_collection = db.collection("testData")
+start_date = sys.argv[1]
+end_date = sys.argv[2] 
+
+query = test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date))
+docs = query.stream()
+
+data = []
+for doc in docs:
+    doc_data = doc.to_dict()
+    data.append(doc_data)
+
+df = pd.DataFrame(data)
+print(df)
+print(df.size)
+
+
+column_names = ["_id", "name", "classificationResult",  "prdClassification", "slopeRatioClass", "predictedDenovixRatio", "slopeRatio", "calculatedRatio", "deviceRatio", "kitSerial", "led1Gain1", "led1Gain2", "led1Gain4", "abs1", "led1Average", "led1Buffer", "led1Sample", "led2Gain1", "led2Gain2", "led2Gain4", "abs2", "led2Average", "led2Buffer", "led2Sample", "led3Gain1", "led3Gain2", "led3Gain4", "hb3", "abs3", "led3Average", "led3Buffer", "led3Sample", "led4Gain1", "led4Gain2", "led4Gain4", "hb4", "abs4", "led4Average", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "appVersion", "testTime"]
+common_columns = [col for col in column_names if col in df.columns]
+df = df[common_columns]
+
+
+print("duplicates", len(df['_id']) - len(df['_id'].drop_duplicates()))
+
+df = df.drop_duplicates()
+
+# df = df.groupby(["classificationResult"]).describe()
+df_count = df.groupby(["classificationResult"]).describe()["calculatedRatio"]["count"]
+print(df.groupby(["classificationResult"]).describe()["calculatedRatio"]["count"])
+print(df.groupby(["deviceId"]).describe()["calculatedRatio"]["count"])
+print(df.groupby(["deviceSerialNumber"]).describe()["calculatedRatio"]["count"])
+# print(df.groupby(["kitSerial", "classificationResult"]).describe())
+
+# df_device = df[df["deviceId"] == "HCV1003"]
+# df.plot(kind = 'scatter', x = 'testTime', y = 'led2Buffer')
+# plt.show()
+
+# df[df["deviceId"] == "HCV2013"]['led2Sample'].plot.box()
+# plt.show()
+
+print(df.groupby(["kitSerial"]).describe()["calculatedRatio"]["count"])
+
+output_filename = f'tests_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx'
+
+downloads_dir = os.path.join(os.path.expanduser("~"), "Downloads")
+output_path = os.path.join(downloads_dir, output_filename)
+writer = pd.ExcelWriter(output_path, engine = 'openpyxl')
+df.to_excel(writer, sheet_name = 'data', index=False)
+df_count.to_excel(writer, sheet_name = "hc_count")
+df.groupby(["result"]).describe()["resultRatio"]["count"].to_excel(writer, sheet_name = "tr_count")
+df.groupby(["deviceId"]).describe().to_excel(writer, sheet_name = "stats")
+df.groupby(["kitSerial"]).describe()["calculatedRatio"]["count"].to_excel(writer, sheet_name = "kit_count")
+df.groupby(["kitSerial", "classificationResult"]).describe()["calculatedRatio"]["count"].to_excel(writer, sheet_name = "kit_class")
+df.groupby(["deviceId", "classificationResult"]).describe()["calculatedRatio"]["count"].to_excel(writer, sheet_name = "device_class")
+writer.close()
+
+print(f"Data saved to '{output_path}'")
+
+firebase_admin.delete_app(firebase_admin.get_app())
+