diff --git a/README.md b/README.md index 7ccff1f..36760af 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,8 @@ Repo for all data related work. gcloud functions deploy my_function --runtime=python311 --region=asia-south1 --trigger-http --allow-unauthenticated - gcloud functions deploy curvefit --runtime=python311 --region=asia-south1 --trigger-http --memory=512MB + gcloud functions deploy clearusers --runtime=python311 --region=asia-south1 --trigger-http --memory=512MB ### function log - gcloud functions logs read curvefit --region=asia-south1 + gcloud functions logs read performance --region=asia-south1 --limit=100 diff --git a/cloud-functions/nodejs/functions/index.js b/cloud-functions/nodejs/functions/index.js index 2fdfd24..9232015 100644 --- a/cloud-functions/nodejs/functions/index.js +++ b/cloud-functions/nodejs/functions/index.js @@ -137,7 +137,7 @@ exports.kitAlert = async (event, context) => { console.log('Function triggered by change to: ' + resource); // now log the full event object console.log(JSON.stringify(event)); - const { kitSerial, classificationResult, _id, led1Average, led3Average } = event.value.fields; + const { kitSerial, classificationResult, _id, led1Average, led3Average, deviceRatio } = event.value.fields; console.log('kitSerial', kitSerial); console.log('classificationResult', classificationResult); // if (classificationResult?.stringValue?.includes("Trait")) { @@ -204,17 +204,37 @@ exports.kitAlert = async (event, context) => { classification = "Sickle Cell Disease"; } + // device ratio class + const devRatio = deviceRatio?.doubleValue; + let devRatioClass = "INVALID"; + if (devRatio > 0 && devRatio <= 0.55) { + devRatioClass = "Normal"; + } + if (devRatio > 0.55 && devRatio <= 0.575) { + devRatioClass = "Negative Borderline"; + } + if (devRatio > 0.575 && devRatio <= 0.85) { + devRatioClass = "Sickle Cell Trait"; + } + if (devRatio > 0.85 && devRatio <= 0.9) { + devRatioClass = "Positive Borderline"; + } + if (devRatio > 0.9 && devRatio <= 1) { + devRatioClass = "Sickle Cell Disease"; + } + console.log(classification); const docs = await admin.firestore().collection("testData").where("_id", "==", _id.stringValue).get(); docs?.forEach(async (doc) => { - // await admin.firestore().collection("testData").doc(doc.id).update({ + await admin.firestore().collection("testData").doc(doc.id).update({ // abs427: Abs427, // abs555: Abs555, // predictedDenovixRatio: pdr, // prdClassification: classification - // // incubationTime: admin.firestore.FieldValue.delete() - // }); + // incubationTime: admin.firestore.FieldValue.delete() + deviceRatioClass: devRatioClass, + }); }) }; diff --git a/cloud-functions/python/functions/clearusers/main.py b/cloud-functions/python/functions/clearusers/main.py new file mode 100644 index 0000000..0175542 --- /dev/null +++ b/cloud-functions/python/functions/clearusers/main.py @@ -0,0 +1,56 @@ +import functions_framework +from google.cloud.firestore_v1.base_query import FieldFilter +from firebase_admin import initialize_app, credentials, firestore +import os +from datetime import datetime + +initialize_app() + +@functions_framework.http +def clearusers(request): + """HTTP Cloud Function. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + if request.method == 'OPTIONS': + headers = { + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': 'GET', + 'Access-Control-Allow-Headers': 'Content-Type', + # 'Access-Control-Max-Age': '3600' + } + + return ('', 204, headers) + + headers = { + 'Access-Control-Allow-Origin': '*' + } + + db = firestore.client() + source_collection = "patientData" + + # Get all documents from the source collection + source_docs = db.collection(source_collection).where(filter=FieldFilter("createdAt", ">=", datetime.today().strftime("%Y-%m-%d"))).stream() + count = 0 + for doc in source_docs: + # Extract the document ID + doc_id = doc.id + print(doc_id) + + # Get the document data + doc_data = doc.to_dict() + + try: + # Delete the document from the source collection + db.collection(source_collection).document(doc_id).delete() + print(f"Document with ID '{doc_id}' updated") + count = count + 1 + except Exception as e: + print(f"Error deleting document with ID '{doc_id}' from the source collection: {e}") + + return ('cleared users: {}!'.format(count), 200, headers) diff --git a/cloud-functions/python/functions/clearusers/requirements.txt b/cloud-functions/python/functions/clearusers/requirements.txt new file mode 100644 index 0000000..df9b31b --- /dev/null +++ b/cloud-functions/python/functions/clearusers/requirements.txt @@ -0,0 +1,6 @@ +functions-framework==3.* +firebase_functions~=0.1.0 +pandas==2.0.3 +openpyxl==3.1.2 +firebase-admin==6.2.0 +scikit-learn==1.3.1 \ No newline at end of file diff --git a/scripts/allow_retest_backdated.py b/scripts/allow_retest_backdated.py index a6f709a..f9e649d 100644 --- a/scripts/allow_retest_backdated.py +++ b/scripts/allow_retest_backdated.py @@ -17,7 +17,7 @@ db = firestore.client() patient_collection = db.collection("patientData") test_collection = db.collection("testData") -query = patient_collection.where(filter=FieldFilter("createdAt", ">=", "2023-10-20")).where(filter=FieldFilter("createdAt", "<", "2023-10-21")) +query = patient_collection.where(filter=FieldFilter("createdAt", ">=", "2023-12-13")).where(filter=FieldFilter("createdAt", "<", "2023-12-14")) #.where(filter=FieldFilter("registrationCenterName", "==", "SCS high school")) patient_docs = query.stream() @@ -27,12 +27,12 @@ data = [] for patient_doc in patient_docs: patient_data = patient_doc.to_dict() data.append(patient_data) - if "BHI" in patient_data['_id']: + if "sar" in patient_data['_id']: print(patient_data['_id']) doc_ref = db.collection("patientData").document(patient_doc.id) delete_field_name = 'incubationTime' - batch.update(doc_ref, {"_idSearch": '20231209' + patient_data['_id'], delete_field_name: firestore.DELETE_FIELD, "allowFreshTest": True, "testStatus": False}) + batch.update(doc_ref, {"_idSearch": '20231214' + patient_data['_id'], delete_field_name: firestore.DELETE_FIELD, "allowFreshTest": True, "testStatus": False}) batch.commit() diff --git a/scripts/combine62.py b/scripts/combine62.py new file mode 100644 index 0000000..4738bb6 --- /dev/null +++ b/scripts/combine62.py @@ -0,0 +1,60 @@ +import pandas as pd +import os +import numpy as np +import matplotlib.pyplot as plt + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/users_14_12_2023_18_42.xlsx", sheet_name="Sheet1") +df2 = pd.read_excel(curdir + path_delim + "data/tests_14_12_2023_18_36.xlsx", sheet_name="data") + +df2 = df2.sort_values('testTime') + +variance_column = df2["led2Buffer"].var(ddof=0) +print(variance_column) + +df = df2.merge(df1, on="_id", how="inner") +print(df.columns) +# # df = pd.concat([df1, df3], ignore_index=True) +df['Age'] = 2023 - df['birthYear'] + +df['deviceId'].hist() +plt.show() + +df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]] +print(df) + +df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True) +df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1) + +# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y')) + +# df = df.sort_values(by=['Date'], ascending=True) + +df["Test Result"].fillna("NOTEST", inplace = True) +print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"])) +df = df[df["Test Result"] != "NOTEST"] + +df["Blood Group"].fillna("NOBLOODGROUP", inplace = True) +print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"])) + +df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last') + +df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)' +df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline' +df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline' +df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE' +df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE' +print(df_final.groupby(["Test Result"]).describe()) + +df_count = df_final.groupby(["Test Result"]).describe()["ABHA ID"]["count"] +print(df_final.groupby(["Test Result"]).describe()["ABHA ID"]["count"]) + +writer = pd.ExcelWriter(curdir + path_delim + "data/Dec14.xlsx", engine = 'openpyxl') +df_final.to_excel(writer, sheet_name = 'op', index=False) +df_count.to_excel(writer, sheet_name = "count") +writer.close() \ No newline at end of file diff --git a/scripts/consolidated_data.py b/scripts/consolidated_data.py index 27dc63b..0cd9c34 100644 --- a/scripts/consolidated_data.py +++ b/scripts/consolidated_data.py @@ -9,7 +9,7 @@ import sys import platform from datetime import datetime, timedelta -environment = "preprod" +environment = "qa" if __name__ == "__main__": @@ -82,7 +82,7 @@ if __name__ == "__main__": df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)] df = df.sort_values(by=['testTime'], ascending=False) - df = df[["_id", "errorMessages", "classificationResult", "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "abs3", "led3Average", "led3Buffer", "led3Sample", "abs4", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "name", "testTime"]] + df = df[["_id", "errorMessages", "classificationResult", "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "hb3", "abs3", "led3Average", "led3Buffer", "led3Sample", "hb4", "abs4", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "name", "testTime"]] # df.rename(columns={'deviceSerialNumber': "login_id", "calculatedRatio": "calibrated_ratio", "led1Buffer": "427_buffer_intensity", "led2Buffer": "555_buffer_intensity", "led1Sample": "427_sample_intensity", "led2Sample": "555_sample_intensity", "led1Average": "427_absorbance", "led2Average": "555_absorbance"}, inplace = True) # df = df.reindex(sorted(df.columns), axis=1) diff --git a/scripts/dataset04.py b/scripts/dataset04.py new file mode 100644 index 0000000..7090ca1 --- /dev/null +++ b/scripts/dataset04.py @@ -0,0 +1,35 @@ +import pandas as pd +import os + +curdir = os.getcwd() +path_delim = '/' +df = pd.read_csv(curdir + path_delim + 'data/bquxjob_6ab822ae_18c5f7cb3c7.csv') +df = df.dropna() +print(df) + +print(df["finalResult"].unique()) + +df['testResult'] = df['finalResult'] +df.loc[df['testResult'] == 'Normal', 'testResult'] = 'Normal' +df.loc[df['testResult'] == 'Normal (HbA)', 'testResult'] = 'Normal' +df.loc[df['testResult'] == 'Sickle Cell Trait', 'testResult'] = 'SCT' +df.loc[df['testResult'] == 'Sickle Cell Trait (HbAS)', 'testResult'] = 'SCT' +df.loc[df['testResult'] == 'Sickle cell Trait (HbAS)', 'testResult'] = 'SCT' +df.loc[df['testResult'] == 'Sickle Cell Disease', 'testResult'] = 'SCD' +df.loc[df['testResult'] == 'Sickle Cell Disease (HbAS)', 'testResult'] = 'SCD' +df.loc[df['testResult'] == 'Sickle cell Disease (HbSS)', 'testResult'] = 'SCD' +df.loc[df['testResult'] == 'Positive for Sickle Cell. HPLC for Confirmation', 'testResult'] = 'Inconclusive' +df.loc[df['testResult'] == 'Inconclusive. Very low Absorbance - Repeat test with Higher Blood Volume', 'testResult'] = 'Inconclusive' +df.loc[df['testResult'] == 'Negative Borderline. Repeat Test', 'testResult'] = 'Inconclusive' +df.loc[df['testResult'] == 'Inconclusive. Very low Absorbance - Repeat test with Higher Blood Volume', 'testResult'] = 'Inconclusive' +df.loc[df['testResult'] == 'Inconclusive. Repeat with test with lower volume of blood', 'testResult'] = 'Inconclusive' + +df = df.drop_duplicates() + +print(df.groupby(["testResult"]).describe()) + +# writer = pd.ExcelWriter(curdir + path_delim + "data/dataset4.xlsx", engine = 'openpyxl') +# df.to_excel(writer, sheet_name = 'op', index=False) +df.to_csv(curdir + path_delim + "data/dataset04.csv", index=False) +# df_count.to_excel(writer, sheet_name = "count") +# writer.close() \ No newline at end of file diff --git a/scripts/denovix_absorbance.py b/scripts/denovix_absorbance.py new file mode 100644 index 0000000..0e96701 --- /dev/null +++ b/scripts/denovix_absorbance.py @@ -0,0 +1,4 @@ +import pandas as pd + +df = pd.read_csv("data/denovix-04-09-23.csv") +print(df) \ No newline at end of file diff --git a/scripts/deploy_preprod1_app.sh b/scripts/deploy_preprod1_app.sh new file mode 100644 index 0000000..76f66dd --- /dev/null +++ b/scripts/deploy_preprod1_app.sh @@ -0,0 +1,3 @@ +firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \ + --app 1:121176529204:android:e6841fdac57bdc95bbed61 \ + --release-notes "coeffs update for device 4" --groups "smi-group" diff --git a/scripts/deploy_preprod_app.sh b/scripts/deploy_preprod_app.sh index 772a676..ee961d8 100644 --- a/scripts/deploy_preprod_app.sh +++ b/scripts/deploy_preprod_app.sh @@ -1,3 +1,3 @@ firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \ --app 1:121176529204:android:30b1bdb8db18ba72bbed61 \ - --release-notes "new changes" --groups "smi-group" + --release-notes "coeffs update for device 4" --groups "smi-group" diff --git a/scripts/deploy_regapp_dev.sh b/scripts/deploy_regapp_dev.sh new file mode 100644 index 0000000..8c03bd3 --- /dev/null +++ b/scripts/deploy_regapp_dev.sh @@ -0,0 +1,3 @@ +firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \ + --app 1:650071678820:android:7569c1cad4fc99916c6471 \ + --release-notes "new changes" --groups "smi-group" diff --git a/scripts/deploy_regapp_preprod.sh b/scripts/deploy_regapp_preprod.sh new file mode 100644 index 0000000..ae4a46c --- /dev/null +++ b/scripts/deploy_regapp_preprod.sh @@ -0,0 +1,3 @@ +firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \ + --app 1:121176529204:android:d7d51c13b0de5a6cbbed61 \ + --release-notes "new changes" --groups "smi-group" diff --git a/scripts/deploy_testingapp_dev.sh b/scripts/deploy_testingapp_dev.sh new file mode 100644 index 0000000..a3048de --- /dev/null +++ b/scripts/deploy_testingapp_dev.sh @@ -0,0 +1,3 @@ +firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \ + --app 1:650071678820:android:f96be19e5d43102b6c6471 \ + --release-notes "new changes" --groups "smi-group" diff --git a/scripts/deploy_testingapp_qa.sh b/scripts/deploy_testingapp_qa.sh new file mode 100644 index 0000000..3fd62c5 --- /dev/null +++ b/scripts/deploy_testingapp_qa.sh @@ -0,0 +1,3 @@ +firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \ + --app 1:1004619739289:android:8397cd1f0357bd89e5c808 \ + --release-notes "add new device for PQ" --groups "smi-group" diff --git a/scripts/image_class_subfolder.py b/scripts/image_class_subfolder.py new file mode 100644 index 0000000..bdee148 --- /dev/null +++ b/scripts/image_class_subfolder.py @@ -0,0 +1,23 @@ +import os +import shutil +import pandas as pd + +# Assuming you have a DataFrame named df with columns 'id', 'class', and 'image_path' +# 'image_path' should contain the path to each image + +# Example DataFrame creation (replace this with your actual data) +data = {'id': [1, 2, 3], + 'class': ['A', 'B', 'A'], + 'image_path': ['/path/to/img1.jpg', '/path/to/img2.jpg', '/path/to/img3.jpg']} +df = pd.DataFrame(data) + +# Iterate through rows and move images +for index, row in df.iterrows(): + class_folder = os.path.join(os.getcwd(), row['class']) + + # Create subfolder if it doesn't exist + if not os.path.exists(class_folder): + os.makedirs(class_folder) + + # Move image to subfolder + shutil.move(row['_id'], os.path.join(class_folder, f"{row['id']}.jpg")) diff --git a/scripts/test_collection.py b/scripts/test_collection.py index 3ba84fb..b98b57b 100644 --- a/scripts/test_collection.py +++ b/scripts/test_collection.py @@ -15,8 +15,8 @@ db = firestore.client() patient_collection = db.collection("patientData") test_collection = db.collection("testData") -start_date = sys.argv[1] # '2023-09-12' #input("Please enter the start date (yyyy-mm-dd): ") -end_date = sys.argv[2] #'2023-07-16' #input("Please enter the end date (yyyy-mm-dd): ") +start_date = sys.argv[1] +end_date = sys.argv[2] query = test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date)) docs = query.stream() diff --git a/scripts/user_collection.py b/scripts/user_collection.py index 489bd54..1bbaf03 100644 --- a/scripts/user_collection.py +++ b/scripts/user_collection.py @@ -15,11 +15,11 @@ firebase_admin.initialize_app(cred) db = firestore.client() patient_collection = db.collection("patientData") -# test_collection = db.collection("testData") +test_collection = db.collection("testData") start_date = sys.argv[1] end_date = sys.argv[2] -query = patient_collection.where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date)) +query = patient_collection #.where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date)) patient_docs = query.stream() data = [] @@ -33,6 +33,8 @@ print(df.size) print("duplicates", len(df['_id']) - len(df['_id'].drop_duplicates())) +df = df.sort_values('bloodGroup', na_position='first', ascending=False).drop_duplicates('_id').sort_index() + output_filename = f'users_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx' df.to_excel(output_filename, index=False)