smiuser/hpos-data
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

14 dec data

Browse Source
This commit is contained in:
Pritimay Sarkar
2023-12-14 21:03:09 +05:30
parent 6bd972276f
commit 7bcdbff123
18 changed files with 237 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -26,8 +26,8 @@ Repo for all data related work.
gcloud functions deploy my_function --runtime=python311 --region=asia-south1 --trigger-http --allow-unauthenticated
gcloud functions deploy curvefit --runtime=python311 --region=asia-south1 --trigger-http --memory=512MB
gcloud functions deploy clearusers --runtime=python311 --region=asia-south1 --trigger-http --memory=512MB
### function log
gcloud functions logs read curvefit --region=asia-south1
gcloud functions logs read performance --region=asia-south1 --limit=100

28
cloud-functions/nodejs/functions/index.js
View File

@@ -137,7 +137,7 @@ exports.kitAlert = async (event, context) => {
console.log('Function triggered by change to: ' + resource);
// now log the full event object
console.log(JSON.stringify(event));
const { kitSerial, classificationResult, _id, led1Average, led3Average } = event.value.fields;
const { kitSerial, classificationResult, _id, led1Average, led3Average, deviceRatio } = event.value.fields;
console.log('kitSerial', kitSerial);
console.log('classificationResult', classificationResult);
// if (classificationResult?.stringValue?.includes("Trait")) {
@@ -204,17 +204,37 @@ exports.kitAlert = async (event, context) => {
classification = "Sickle Cell Disease";
}
// device ratio class
const devRatio = deviceRatio?.doubleValue;
let devRatioClass = "INVALID";
if (devRatio > 0 && devRatio <= 0.55) {
devRatioClass = "Normal";
}
if (devRatio > 0.55 && devRatio <= 0.575) {
devRatioClass = "Negative Borderline";
}
if (devRatio > 0.575 && devRatio <= 0.85) {
devRatioClass = "Sickle Cell Trait";
}
if (devRatio > 0.85 && devRatio <= 0.9) {
devRatioClass = "Positive Borderline";
}
if (devRatio > 0.9 && devRatio <= 1) {
devRatioClass = "Sickle Cell Disease";
}
console.log(classification);
const docs = await admin.firestore().collection("testData").where("_id", "==", _id.stringValue).get();
docs?.forEach(async (doc) => {
// await admin.firestore().collection("testData").doc(doc.id).update({
await admin.firestore().collection("testData").doc(doc.id).update({
// abs427: Abs427,
// abs555: Abs555,
// predictedDenovixRatio: pdr,
// prdClassification: classification
// // incubationTime: admin.firestore.FieldValue.delete()
// });
// incubationTime: admin.firestore.FieldValue.delete()
deviceRatioClass: devRatioClass,
});
})
};

56
cloud-functions/python/functions/clearusers/main.py Normal file
View File

@@ -0,0 +1,56 @@
import functions_framework
from google.cloud.firestore_v1.base_query import FieldFilter
from firebase_admin import initialize_app, credentials, firestore
import os
from datetime import datetime
initialize_app()
@functions_framework.http
def clearusers(request):
"""HTTP Cloud Function.
Args:
request (flask.Request): The request object.
<https://flask.palletsprojects.com/en/1.1.x/api/#incoming-request-data>
Returns:
The response text, or any set of values that can be turned into a
Response object using `make_response`
<https://flask.palletsprojects.com/en/1.1.x/api/#flask.make_response>.
"""
if request.method == 'OPTIONS':
headers = {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET',
'Access-Control-Allow-Headers': 'Content-Type',
# 'Access-Control-Max-Age': '3600'
}
return ('', 204, headers)
headers = {
'Access-Control-Allow-Origin': '*'
}
db = firestore.client()
source_collection = "patientData"
# Get all documents from the source collection
source_docs = db.collection(source_collection).where(filter=FieldFilter("createdAt", ">=", datetime.today().strftime("%Y-%m-%d"))).stream()
count = 0
for doc in source_docs:
# Extract the document ID
doc_id = doc.id
print(doc_id)
# Get the document data
doc_data = doc.to_dict()
try:
# Delete the document from the source collection
db.collection(source_collection).document(doc_id).delete()
print(f"Document with ID '{doc_id}' updated")
count = count + 1
except Exception as e:
print(f"Error deleting document with ID '{doc_id}' from the source collection: {e}")
return ('cleared users: {}!'.format(count), 200, headers)

6
cloud-functions/python/functions/clearusers/requirements.txt Normal file
View File

@@ -0,0 +1,6 @@
functions-framework==3.*
firebase_functions~=0.1.0
pandas==2.0.3
openpyxl==3.1.2
firebase-admin==6.2.0
scikit-learn==1.3.1

6
scripts/allow_retest_backdated.py
View File

@@ -17,7 +17,7 @@ db = firestore.client()
patient_collection = db.collection("patientData")
test_collection = db.collection("testData")
query = patient_collection.where(filter=FieldFilter("createdAt", ">=", "2023-10-20")).where(filter=FieldFilter("createdAt", "<", "2023-10-21"))
query = patient_collection.where(filter=FieldFilter("createdAt", ">=", "2023-12-13")).where(filter=FieldFilter("createdAt", "<", "2023-12-14"))
#.where(filter=FieldFilter("registrationCenterName", "==", "SCS high school"))
patient_docs = query.stream()
@@ -27,12 +27,12 @@ data = []
for patient_doc in patient_docs:
patient_data = patient_doc.to_dict()
data.append(patient_data)
if "BHI" in patient_data['_id']:
if "sar" in patient_data['_id']:
print(patient_data['_id'])
doc_ref = db.collection("patientData").document(patient_doc.id)
delete_field_name = 'incubationTime'
batch.update(doc_ref, {"_idSearch": '20231209' + patient_data['_id'], delete_field_name: firestore.DELETE_FIELD, "allowFreshTest": True, "testStatus": False})
batch.update(doc_ref, {"_idSearch": '20231214' + patient_data['_id'], delete_field_name: firestore.DELETE_FIELD, "allowFreshTest": True, "testStatus": False})
batch.commit()

60
scripts/combine62.py Normal file
View File

@@ -0,0 +1,60 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/users_14_12_2023_18_42.xlsx", sheet_name="Sheet1")
df2 = pd.read_excel(curdir + path_delim + "data/tests_14_12_2023_18_36.xlsx", sheet_name="data")
df2 = df2.sort_values('testTime')
variance_column = df2["led2Buffer"].var(ddof=0)
print(variance_column)
df = df2.merge(df1, on="_id", how="inner")
print(df.columns)
# # df = pd.concat([df1, df3], ignore_index=True)
df['Age'] = 2023 - df['birthYear']
df['deviceId'].hist()
plt.show()
df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]]
print(df)
df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True)
df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1)
# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y'))
# df = df.sort_values(by=['Date'], ascending=True)
df["Test Result"].fillna("NOTEST", inplace = True)
print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"]))
df = df[df["Test Result"] != "NOTEST"]
df["Blood Group"].fillna("NOBLOODGROUP", inplace = True)
print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"]))
df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last')
df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)'
df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline'
df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline'
df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE'
df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE'
print(df_final.groupby(["Test Result"]).describe())
df_count = df_final.groupby(["Test Result"]).describe()["ABHA ID"]["count"]
print(df_final.groupby(["Test Result"]).describe()["ABHA ID"]["count"])
writer = pd.ExcelWriter(curdir + path_delim + "data/Dec14.xlsx", engine = 'openpyxl')
df_final.to_excel(writer, sheet_name = 'op', index=False)
df_count.to_excel(writer, sheet_name = "count")
writer.close()

4
scripts/consolidated_data.py
View File

@@ -9,7 +9,7 @@ import sys
import platform
from datetime import datetime, timedelta
environment = "preprod"
environment = "qa"
if __name__ == "__main__":
@@ -82,7 +82,7 @@ if __name__ == "__main__":
df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)]
df = df.sort_values(by=['testTime'], ascending=False)
df = df[["_id", "errorMessages", "classificationResult", "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "abs3", "led3Average", "led3Buffer", "led3Sample", "abs4", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "name", "testTime"]]
df = df[["_id", "errorMessages", "classificationResult", "prdClassification", "predictedDenovixRatio", "calculatedRatio", "deviceRatio", "kitSerial", "abs1", "led1Average", "led1Buffer", "led1Sample", "abs2", "led2Average", "led2Buffer", "led2Sample", "hb3", "abs3", "led3Average", "led3Buffer", "led3Sample", "hb4", "abs4", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "deviceId", "deviceSerialNumber", "name", "testTime"]]
# df.rename(columns={'deviceSerialNumber': "login_id", "calculatedRatio": "calibrated_ratio", "led1Buffer": "427_buffer_intensity", "led2Buffer": "555_buffer_intensity", "led1Sample": "427_sample_intensity", "led2Sample": "555_sample_intensity", "led1Average": "427_absorbance", "led2Average": "555_absorbance"}, inplace = True)
# df = df.reindex(sorted(df.columns), axis=1)

35
scripts/dataset04.py Normal file
View File

@@ -0,0 +1,35 @@
import pandas as pd
import os
curdir = os.getcwd()
path_delim = '/'
df = pd.read_csv(curdir + path_delim + 'data/bquxjob_6ab822ae_18c5f7cb3c7.csv')
df = df.dropna()
print(df)
print(df["finalResult"].unique())
df['testResult'] = df['finalResult']
df.loc[df['testResult'] == 'Normal', 'testResult'] = 'Normal'
df.loc[df['testResult'] == 'Normal (HbA)', 'testResult'] = 'Normal'
df.loc[df['testResult'] == 'Sickle Cell Trait', 'testResult'] = 'SCT'
df.loc[df['testResult'] == 'Sickle Cell Trait (HbAS)', 'testResult'] = 'SCT'
df.loc[df['testResult'] == 'Sickle cell Trait (HbAS)', 'testResult'] = 'SCT'
df.loc[df['testResult'] == 'Sickle Cell Disease', 'testResult'] = 'SCD'
df.loc[df['testResult'] == 'Sickle Cell Disease (HbAS)', 'testResult'] = 'SCD'
df.loc[df['testResult'] == 'Sickle cell Disease (HbSS)', 'testResult'] = 'SCD'
df.loc[df['testResult'] == 'Positive for Sickle Cell. HPLC for Confirmation', 'testResult'] = 'Inconclusive'
df.loc[df['testResult'] == 'Inconclusive. Very low Absorbance - Repeat test with Higher Blood Volume', 'testResult'] = 'Inconclusive'
df.loc[df['testResult'] == 'Negative Borderline. Repeat Test', 'testResult'] = 'Inconclusive'
df.loc[df['testResult'] == 'Inconclusive. Very low Absorbance - Repeat test with Higher Blood Volume', 'testResult'] = 'Inconclusive'
df.loc[df['testResult'] == 'Inconclusive. Repeat with test with lower volume of blood', 'testResult'] = 'Inconclusive'
df = df.drop_duplicates()
print(df.groupby(["testResult"]).describe())
# writer = pd.ExcelWriter(curdir + path_delim + "data/dataset4.xlsx", engine = 'openpyxl')
# df.to_excel(writer, sheet_name = 'op', index=False)
df.to_csv(curdir + path_delim + "data/dataset04.csv", index=False)
# df_count.to_excel(writer, sheet_name = "count")
# writer.close()

4
scripts/denovix_absorbance.py Normal file
View File

@@ -0,0 +1,4 @@
import pandas as pd
df = pd.read_csv("data/denovix-04-09-23.csv")
print(df)

3
scripts/deploy_preprod1_app.sh Normal file
View File

@@ -0,0 +1,3 @@
firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \
--app 1:121176529204:android:e6841fdac57bdc95bbed61 \
--release-notes "coeffs update for device 4" --groups "smi-group"

2
scripts/deploy_preprod_app.sh
View File

@@ -1,3 +1,3 @@
firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \
--app 1:121176529204:android:30b1bdb8db18ba72bbed61 \
--release-notes "new changes" --groups "smi-group"
--release-notes "coeffs update for device 4" --groups "smi-group"

3
scripts/deploy_regapp_dev.sh Normal file
View File

@@ -0,0 +1,3 @@
firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \
--app 1:650071678820:android:7569c1cad4fc99916c6471 \
--release-notes "new changes" --groups "smi-group"

3
scripts/deploy_regapp_preprod.sh Normal file
View File

@@ -0,0 +1,3 @@
firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \
--app 1:121176529204:android:d7d51c13b0de5a6cbbed61 \
--release-notes "new changes" --groups "smi-group"

3
scripts/deploy_testingapp_dev.sh Normal file
View File

@@ -0,0 +1,3 @@
firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \
--app 1:650071678820:android:f96be19e5d43102b6c6471 \
--release-notes "new changes" --groups "smi-group"

3
scripts/deploy_testingapp_qa.sh Normal file
View File

@@ -0,0 +1,3 @@
firebase appdistribution:distribute /Users/apple/Downloads/work/hpos/app/build/outputs/apk/debug/app-debug.apk \
--app 1:1004619739289:android:8397cd1f0357bd89e5c808 \
--release-notes "add new device for PQ" --groups "smi-group"

23
scripts/image_class_subfolder.py Normal file
View File

@@ -0,0 +1,23 @@
import os
import shutil
import pandas as pd
# Assuming you have a DataFrame named df with columns 'id', 'class', and 'image_path'
# 'image_path' should contain the path to each image
# Example DataFrame creation (replace this with your actual data)
data = {'id': [1, 2, 3],
'class': ['A', 'B', 'A'],
'image_path': ['/path/to/img1.jpg', '/path/to/img2.jpg', '/path/to/img3.jpg']}
df = pd.DataFrame(data)
# Iterate through rows and move images
for index, row in df.iterrows():
class_folder = os.path.join(os.getcwd(), row['class'])
# Create subfolder if it doesn't exist
if not os.path.exists(class_folder):
os.makedirs(class_folder)
# Move image to subfolder
shutil.move(row['_id'], os.path.join(class_folder, f"{row['id']}.jpg"))

4
scripts/test_collection.py
View File

@@ -15,8 +15,8 @@ db = firestore.client()
patient_collection = db.collection("patientData")
test_collection = db.collection("testData")
start_date = sys.argv[1] # '2023-09-12' #input("Please enter the start date (yyyy-mm-dd): ")
end_date = sys.argv[2] #'2023-07-16' #input("Please enter the end date (yyyy-mm-dd): ")
start_date = sys.argv[1]
end_date = sys.argv[2]
query = test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date))
docs = query.stream()

6
scripts/user_collection.py
View File

@@ -15,11 +15,11 @@ firebase_admin.initialize_app(cred)
db = firestore.client()
patient_collection = db.collection("patientData")
# test_collection = db.collection("testData")
test_collection = db.collection("testData")
start_date = sys.argv[1]
end_date = sys.argv[2]
query = patient_collection.where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date))
query = patient_collection #.where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date))
patient_docs = query.stream()
data = []
@@ -33,6 +33,8 @@ print(df.size)
print("duplicates", len(df['_id']) - len(df['_id'].drop_duplicates()))
df = df.sort_values('bloodGroup', na_position='first', ascending=False).drop_duplicates('_id').sort_index()
output_filename = f'users_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx'
df.to_excel(output_filename, index=False)