From 55b98e47d37d233e9f897cf371d22688e761ef28 Mon Sep 17 00:00:00 2001 From: Pritimay Sarkar Date: Wed, 22 Nov 2023 21:41:42 +0530 Subject: [PATCH] data for 21 - 22 nov --- scripts/combine44.py | 88 ++++++++++++++++++++++++++++++++++++++++++++ scripts/combine45.py | 60 ++++++++++++++++++++++++++++++ scripts/combine46.py | 60 ++++++++++++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 scripts/combine44.py create mode 100644 scripts/combine45.py create mode 100644 scripts/combine46.py diff --git a/scripts/combine44.py b/scripts/combine44.py new file mode 100644 index 0000000..3439134 --- /dev/null +++ b/scripts/combine44.py @@ -0,0 +1,88 @@ +import pandas as pd +import os +import numpy as np +import matplotlib.pyplot as plt + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/users_19_11_2023_19_38.xlsx", sheet_name="Sheet1") +df2 = pd.read_excel(curdir + path_delim + "data/tests_18_11_2023_14_12.xlsx") + +df2 = df2.sort_values('testTime') + +variance_column = df2["led2Buffer"].var(ddof=0) +print(variance_column) + +# df2.plot(kind = 'scatter', x = 'testTime', y = 'calculatedRatio') +# plt.show() + + +# df2.plot(kind = 'scatter', x = 'testTime', y = 'led1Buffer') +# plt.show() + +# df2.plot(kind = 'scatter', x = 'testTime', y = 'led2Buffer') +# plt.show() + +# df2.plot(kind = 'scatter', x = 'testTime', y = 'led1Sample') +# plt.show() + +# df2.plot(kind = 'scatter', x = 'testTime', y = 'led2Sample') +# plt.show() + +df = df1.merge(df2, on="_id", how="outer") +print(df.columns) +# # df = pd.concat([df1, df3], ignore_index=True) +df['Age'] = 2023 - df['birthYear'] + +df['deviceId'].hist() +plt.show() + +# df['led2Sample'].plot.box() +# plt.show() + +# df['led2Sample'].plot.kde() +# plt.show() + +# df['led2Sample'].plot.density() +# plt.show() + +# df['category'].plot.pie() +# plt.show() + +df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]] +print(df) + +df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True) +df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1) + +# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y')) + +# df = df.sort_values(by=['Date'], ascending=True) + +df["Test Result"].fillna("NOTEST", inplace = True) +print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"])) +df = df[df["Test Result"] != "NOTEST"] + +df["Blood Group"].fillna("NOBLOODGROUP", inplace = True) +print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"])) + +df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last') + +df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)' +df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline' +df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline' +df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE' +df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE' +print(df_final.groupby(["Test Result"]).describe()) + +df_count = df.groupby(["Test Result"]).describe()["ABHA ID"]["count"] +print(df.groupby(["Test Result"]).describe()["ABHA ID"]["count"]) + +writer = pd.ExcelWriter(curdir + path_delim + "data/Nov17.xlsx", engine = 'openpyxl') +df_final.to_excel(writer, sheet_name = 'op', index=False) +df_count.to_excel(writer, sheet_name = "count") +writer.close() \ No newline at end of file diff --git a/scripts/combine45.py b/scripts/combine45.py new file mode 100644 index 0000000..b148696 --- /dev/null +++ b/scripts/combine45.py @@ -0,0 +1,60 @@ +import pandas as pd +import os +import numpy as np +import matplotlib.pyplot as plt + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/users_21_11_2023_20_25.xlsx", sheet_name="Sheet1") +df2 = pd.read_excel(curdir + path_delim + "data/tests_21_11_2023_20_20.xlsx", sheet_name="data") + +df2 = df2.sort_values('testTime') + +variance_column = df2["led2Buffer"].var(ddof=0) +print(variance_column) + +df = df1.merge(df2, on="_id", how="outer") +print(df.columns) +# # df = pd.concat([df1, df3], ignore_index=True) +df['Age'] = 2023 - df['birthYear'] + +df['deviceId'].hist() +plt.show() + +df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]] +print(df) + +df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True) +df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1) + +# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y')) + +# df = df.sort_values(by=['Date'], ascending=True) + +df["Test Result"].fillna("NOTEST", inplace = True) +print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"])) +df = df[df["Test Result"] != "NOTEST"] + +df["Blood Group"].fillna("NOBLOODGROUP", inplace = True) +print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"])) + +df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last') + +df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)' +df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline' +df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline' +df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE' +df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE' +print(df_final.groupby(["Test Result"]).describe()) + +df_count = df.groupby(["Test Result"]).describe()["ABHA ID"]["count"] +print(df.groupby(["Test Result"]).describe()["ABHA ID"]["count"]) + +writer = pd.ExcelWriter(curdir + path_delim + "data/Nov21.xlsx", engine = 'openpyxl') +df_final.to_excel(writer, sheet_name = 'op', index=False) +df_count.to_excel(writer, sheet_name = "count") +writer.close() \ No newline at end of file diff --git a/scripts/combine46.py b/scripts/combine46.py new file mode 100644 index 0000000..6842626 --- /dev/null +++ b/scripts/combine46.py @@ -0,0 +1,60 @@ +import pandas as pd +import os +import numpy as np +import matplotlib.pyplot as plt + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/users_22_11_2023_21_06.xlsx", sheet_name="Sheet1") +df2 = pd.read_excel(curdir + path_delim + "data/2023-11-22_to_2023-11-23.xlsx") + +df2 = df2.sort_values('testTime') + +variance_column = df2["led2Buffer"].var(ddof=0) +print(variance_column) + +df = df1.merge(df2, on="_id", how="outer") +print(df.columns) +# # df = pd.concat([df1, df3], ignore_index=True) +df['Age'] = 2023 - df['birthYear'] + +df['deviceId'].hist() +plt.show() + +df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]] +print(df) + +df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True) +df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1) + +# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y')) + +# df = df.sort_values(by=['Date'], ascending=True) + +df["Test Result"].fillna("NOTEST", inplace = True) +print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"])) +df = df[df["Test Result"] != "NOTEST"] + +df["Blood Group"].fillna("NOBLOODGROUP", inplace = True) +print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"])) + +df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last') + +df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)' +df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline' +df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline' +df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE' +df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE' +print(df_final.groupby(["Test Result"]).describe()) + +df_count = df.groupby(["Test Result"]).describe()["ABHA ID"]["count"] +print(df.groupby(["Test Result"]).describe()["ABHA ID"]["count"]) + +writer = pd.ExcelWriter(curdir + path_delim + "data/Nov22.xlsx", engine = 'openpyxl') +df_final.to_excel(writer, sheet_name = 'op', index=False) +df_count.to_excel(writer, sheet_name = "count") +writer.close() \ No newline at end of file