14 dec data

README.md

@@ -26,8 +26,8 @@ Repo for all data related work.
 
 	gcloud functions deploy my_function --runtime=python311 --region=asia-south1  --trigger-http --allow-unauthenticated
 
-	gcloud functions deploy curvefit --runtime=python311 --region=asia-south1  --trigger-http --memory=512MB
+	gcloud functions deploy clearusers --runtime=python311 --region=asia-south1  --trigger-http --memory=512MB
 
 ### function log
 
-	gcloud functions logs read curvefit --region=asia-south1
+	gcloud functions logs read performance --region=asia-south1 --limit=100

cloud-functions/nodejs/functions/index.js

@@ -137,7 +137,7 @@ exports.kitAlert = async (event, context) => {
     console.log('Function triggered by change to: ' + resource);
     // now log the full event object
     console.log(JSON.stringify(event));
-    const { kitSerial, classificationResult, _id, led1Average, led3Average } = event.value.fields;
+    const { kitSerial, classificationResult, _id, led1Average, led3Average, deviceRatio } = event.value.fields;
     console.log('kitSerial', kitSerial);
     console.log('classificationResult', classificationResult);
     // if (classificationResult?.stringValue?.includes("Trait")) {
@@ -204,17 +204,37 @@ exports.kitAlert = async (event, context) => {
         classification = "Sickle Cell Disease";
     }
 
+    // device ratio class
+    const devRatio = deviceRatio?.doubleValue;
+    let devRatioClass = "INVALID";
+    if (devRatio > 0 && devRatio <= 0.55) {
+        devRatioClass = "Normal";
+    }
+    if (devRatio > 0.55 && devRatio <= 0.575) {
+        devRatioClass = "Negative Borderline";
+    }
+    if (devRatio > 0.575 && devRatio <= 0.85) {
+        devRatioClass = "Sickle Cell Trait";
+    }
+    if (devRatio > 0.85 && devRatio <= 0.9) {
+        devRatioClass = "Positive Borderline";
+    }
+    if (devRatio > 0.9 && devRatio <= 1) {
+        devRatioClass = "Sickle Cell Disease";
+    }
+
     console.log(classification);
 
     const docs = await admin.firestore().collection("testData").where("_id", "==", _id.stringValue).get();
     docs?.forEach(async (doc) => {
-        // await admin.firestore().collection("testData").doc(doc.id).update({
+        await admin.firestore().collection("testData").doc(doc.id).update({
         // abs427: Abs427,
         // abs555: Abs555,
         // predictedDenovixRatio: pdr,
         // prdClassification: classification
-        // // incubationTime: admin.firestore.FieldValue.delete()
+        // incubationTime: admin.firestore.FieldValue.delete()
-        // });
+            deviceRatioClass: devRatioClass,
+        });
     })
 };
 

cloud-functions/python/functions/clearusers/main.py

@@ -0,0 +1,56 @@
+import functions_framework
+from google.cloud.firestore_v1.base_query import FieldFilter
+from firebase_admin import initialize_app, credentials, firestore
+import os
+from datetime import datetime
+
+initialize_app()
+
+@functions_framework.http
+def clearusers(request):
+    """HTTP Cloud Function.
+    Args:
+        request (flask.Request): The request object.
+        <https://flask.palletsprojects.com/en/1.1.x/api/#incoming-request-data>
+    Returns:
+        The response text, or any set of values that can be turned into a
+        Response object using `make_response`
+        <https://flask.palletsprojects.com/en/1.1.x/api/#flask.make_response>.
+    """
+    if request.method == 'OPTIONS':
+        headers = {
+            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Methods': 'GET',
+            'Access-Control-Allow-Headers': 'Content-Type',
+            # 'Access-Control-Max-Age': '3600'
+        }
+
+        return ('', 204, headers)
+
+    headers = {
+        'Access-Control-Allow-Origin': '*'
+    }
+    
+    db = firestore.client()
+    source_collection = "patientData"
+
+    # Get all documents from the source collection
+    source_docs = db.collection(source_collection).where(filter=FieldFilter("createdAt", ">=", datetime.today().strftime("%Y-%m-%d"))).stream()
+    count = 0
+    for doc in source_docs:
+        # Extract the document ID
+        doc_id = doc.id
+        print(doc_id)
+
+        # Get the document data
+        doc_data = doc.to_dict()
+
+        try:
+            # Delete the document from the source collection
+            db.collection(source_collection).document(doc_id).delete()
+            print(f"Document with ID '{doc_id}' updated")
+            count = count + 1
+        except Exception as e:
+            print(f"Error deleting document with ID '{doc_id}' from the source collection: {e}")
+    
+    return ('cleared users: {}!'.format(count), 200, headers)

cloud-functions/python/functions/clearusers/requirements.txt

@@ -0,0 +1,6 @@
+functions-framework==3.*
+firebase_functions~=0.1.0
+pandas==2.0.3
+openpyxl==3.1.2
+firebase-admin==6.2.0
+scikit-learn==1.3.1

scripts/allow_retest_backdated.py

@@ -17,7 +17,7 @@ db = firestore.client()
 patient_collection = db.collection("patientData")
 test_collection = db.collection("testData")
 
-query = patient_collection.where(filter=FieldFilter("createdAt", ">=", "2023-10-20")).where(filter=FieldFilter("createdAt", "<", "2023-10-21"))
+query = patient_collection.where(filter=FieldFilter("createdAt", ">=", "2023-12-13")).where(filter=FieldFilter("createdAt", "<", "2023-12-14"))
 #.where(filter=FieldFilter("registrationCenterName", "==", "SCS high school"))
 patient_docs = query.stream()
 
@@ -27,12 +27,12 @@ data = []
 for patient_doc in patient_docs:
     patient_data = patient_doc.to_dict()
     data.append(patient_data)
-    if "BHI" in patient_data['_id']:
+    if "sar" in patient_data['_id']:
         print(patient_data['_id'])
         doc_ref = db.collection("patientData").document(patient_doc.id)
 
         delete_field_name = 'incubationTime'
 
-        batch.update(doc_ref, {"_idSearch": '20231209' + patient_data['_id'], delete_field_name: firestore.DELETE_FIELD, "allowFreshTest": True, "testStatus": False})
+        batch.update(doc_ref, {"_idSearch": '20231214' + patient_data['_id'], delete_field_name: firestore.DELETE_FIELD, "allowFreshTest": True, "testStatus": False})
 
 batch.commit()

scripts/combine62.py

@@ -0,0 +1,60 @@
+import pandas as pd
+import os
+import numpy as np
+import matplotlib.pyplot as plt
+
+curdir = os.getcwd()
+path_delim = '/'
+df1 = pd.read_excel(curdir + path_delim + "data/users_14_12_2023_18_42.xlsx", sheet_name="Sheet1")
+df2 = pd.read_excel(curdir + path_delim + "data/tests_14_12_2023_18_36.xlsx", sheet_name="data")
+
+df2 = df2.sort_values('testTime')
+
+variance_column = df2["led2Buffer"].var(ddof=0)
+print(variance_column)
+
+df = df2.merge(df1, on="_id", how="inner")
+print(df.columns)
+# # df = pd.concat([df1, df3], ignore_index=True)
+df['Age'] = 2023 - df['birthYear']
+
+df['deviceId'].hist()
+plt.show()
+
+df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]]
+print(df)
+
+df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True)
+df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1)
+
+# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y'))
+
+# df = df.sort_values(by=['Date'], ascending=True)
+
+df["Test Result"].fillna("NOTEST", inplace = True)
+print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"]))
+df = df[df["Test Result"] != "NOTEST"]
+
+df["Blood Group"].fillna("NOBLOODGROUP", inplace = True)
+print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"]))
+
+df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last')
+
+df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)'
+df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
+df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)'
+df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
+df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)'
+df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline'
+df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline'
+df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE'
+df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE'
+print(df_final.groupby(["Test Result"]).describe())
+
+df_count = df_final.groupby(["Test Result"]).describe()["ABHA ID"]["count"]
+print(df_final.groupby(["Test Result"]).describe()["ABHA ID"]["count"])
+
+writer = pd.ExcelWriter(curdir + path_delim + "data/Dec14.xlsx", engine = 'openpyxl')
+df_final.to_excel(writer, sheet_name = 'op', index=False)
+df_count.to_excel(writer, sheet_name = "count")
+writer.close()

scripts/consolidated_data.py

@@ -9,7 +9,7 @@ import sys
 import platform
 from datetime import datetime, timedelta
 
-environment = "preprod"
+environment = "qa"
 
 if __name__ == "__main__":
 
@@ -82,7 +82,7 @@ if __name__ == "__main__":
     df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)]
     df = df.sort_values(by=['testTime'], ascending=False)
     
-    df = df[["_id", "errorMessages", "classificationResult",  "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "abs3", "led3Average", "led3Buffer", "led3Sample", "abs4", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "name", "testTime"]]
+    df = df[["_id", "errorMessages", "classificationResult",  "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "hb3", "abs3", "led3Average", "led3Buffer", "led3Sample", "hb4", "abs4", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "name", "testTime"]]
     # df.rename(columns={'deviceSerialNumber': "login_id", "calculatedRatio": "calibrated_ratio", "led1Buffer": "427_buffer_intensity", "led2Buffer": "555_buffer_intensity", "led1Sample": "427_sample_intensity", "led2Sample": "555_sample_intensity", "led1Average": "427_absorbance", "led2Average": "555_absorbance"}, inplace = True)
     # df = df.reindex(sorted(df.columns), axis=1)
 

scripts/dataset04.py

@@ -0,0 +1,35 @@
+import pandas as pd
+import os
+
+curdir = os.getcwd()
+path_delim = '/'
+df = pd.read_csv(curdir + path_delim + 'data/bquxjob_6ab822ae_18c5f7cb3c7.csv')
+df = df.dropna()
+print(df)
+
+print(df["finalResult"].unique())
+
+df['testResult'] = df['finalResult']
+df.loc[df['testResult'] == 'Normal', 'testResult'] = 'Normal'
+df.loc[df['testResult'] == 'Normal (HbA)', 'testResult'] = 'Normal'
+df.loc[df['testResult'] == 'Sickle Cell Trait', 'testResult'] = 'SCT'
+df.loc[df['testResult'] == 'Sickle Cell Trait (HbAS)', 'testResult'] = 'SCT'
+df.loc[df['testResult'] == 'Sickle cell Trait (HbAS)', 'testResult'] = 'SCT'
+df.loc[df['testResult'] == 'Sickle Cell Disease', 'testResult'] = 'SCD'
+df.loc[df['testResult'] == 'Sickle Cell Disease (HbAS)', 'testResult'] = 'SCD'
+df.loc[df['testResult'] == 'Sickle cell Disease (HbSS)', 'testResult'] = 'SCD'
+df.loc[df['testResult'] == 'Positive for Sickle Cell. HPLC for Confirmation', 'testResult'] = 'Inconclusive'
+df.loc[df['testResult'] == 'Inconclusive. Very low Absorbance - Repeat test with Higher Blood Volume', 'testResult'] = 'Inconclusive'
+df.loc[df['testResult'] == 'Negative Borderline. Repeat Test', 'testResult'] = 'Inconclusive'
+df.loc[df['testResult'] == 'Inconclusive. Very low Absorbance - Repeat test with Higher Blood Volume', 'testResult'] = 'Inconclusive'
+df.loc[df['testResult'] == 'Inconclusive. Repeat with test with lower volume of blood', 'testResult'] = 'Inconclusive'
+
+df = df.drop_duplicates()
+
+print(df.groupby(["testResult"]).describe())
+
+# writer = pd.ExcelWriter(curdir + path_delim + "data/dataset4.xlsx", engine = 'openpyxl')
+# df.to_excel(writer, sheet_name = 'op', index=False)
+df.to_csv(curdir + path_delim + "data/dataset04.csv", index=False)
+# df_count.to_excel(writer, sheet_name = "count")
+# writer.close()

scripts/denovix_absorbance.py

@@ -0,0 +1,4 @@
+import pandas as pd
+
+df = pd.read_csv("data/denovix-04-09-23.csv")
+print(df)

scripts/deploy_preprod1_app.sh

@@ -0,0 +1,3 @@
+firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk  \
+    --app 1:121176529204:android:e6841fdac57bdc95bbed61  \
+    --release-notes "coeffs update for device 4" --groups "smi-group"

scripts/deploy_preprod_app.sh

@@ -1,3 +1,3 @@
 firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk  \
     --app 1:121176529204:android:30b1bdb8db18ba72bbed61  \
-    --release-notes "new changes" --groups "smi-group"
+    --release-notes "coeffs update for device 4" --groups "smi-group"

scripts/deploy_regapp_dev.sh

@@ -0,0 +1,3 @@
+firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk  \
+    --app 1:650071678820:android:7569c1cad4fc99916c6471  \
+    --release-notes "new changes" --groups "smi-group"

scripts/deploy_regapp_preprod.sh

@@ -0,0 +1,3 @@
+firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk  \
+    --app 1:121176529204:android:d7d51c13b0de5a6cbbed61  \
+    --release-notes "new changes" --groups "smi-group"

scripts/deploy_testingapp_dev.sh

@@ -0,0 +1,3 @@
+firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk  \
+    --app 1:650071678820:android:f96be19e5d43102b6c6471  \
+    --release-notes "new changes" --groups "smi-group"

scripts/deploy_testingapp_qa.sh

@@ -0,0 +1,3 @@
+firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk  \
+    --app 1:1004619739289:android:8397cd1f0357bd89e5c808  \
+    --release-notes "add new device for PQ" --groups "smi-group"

scripts/image_class_subfolder.py

@@ -0,0 +1,23 @@
+import os
+import shutil
+import pandas as pd
+
+# Assuming you have a DataFrame named df with columns 'id', 'class', and 'image_path'
+# 'image_path' should contain the path to each image
+
+# Example DataFrame creation (replace this with your actual data)
+data = {'id': [1, 2, 3],
+        'class': ['A', 'B', 'A'],
+        'image_path': ['/path/to/img1.jpg', '/path/to/img2.jpg', '/path/to/img3.jpg']}
+df = pd.DataFrame(data)
+
+# Iterate through rows and move images
+for index, row in df.iterrows():
+    class_folder = os.path.join(os.getcwd(), row['class'])
+    
+    # Create subfolder if it doesn't exist
+    if not os.path.exists(class_folder):
+        os.makedirs(class_folder)
+    
+    # Move image to subfolder
+    shutil.move(row['_id'], os.path.join(class_folder, f"{row['id']}.jpg"))

scripts/test_collection.py

@@ -15,8 +15,8 @@ db = firestore.client()
 
 patient_collection = db.collection("patientData")
 test_collection = db.collection("testData")
-start_date = sys.argv[1] # '2023-09-12' #input("Please enter the start date (yyyy-mm-dd): ")
+start_date = sys.argv[1]
-end_date = sys.argv[2] #'2023-07-16' #input("Please enter the end date (yyyy-mm-dd): ")
+end_date = sys.argv[2] 
 
 query = test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date))
 docs = query.stream()

scripts/user_collection.py

@@ -15,11 +15,11 @@ firebase_admin.initialize_app(cred)
 db = firestore.client()
 
 patient_collection = db.collection("patientData")
-# test_collection = db.collection("testData")
+test_collection = db.collection("testData")
 start_date = sys.argv[1]
 end_date = sys.argv[2]
 
-query = patient_collection.where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date))
+query = patient_collection #.where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date))
 patient_docs = query.stream()
 
 data = []
@@ -33,6 +33,8 @@ print(df.size)
 
 print("duplicates", len(df['_id']) - len(df['_id'].drop_duplicates()))
 
+df = df.sort_values('bloodGroup', na_position='first', ascending=False).drop_duplicates('_id').sort_index()
+
 output_filename =  f'users_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx'
 df.to_excel(output_filename, index=False)