smiuser/hpos-data
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

uncommon data for dec 1 and 7

Browse Source
This commit is contained in:
Pritimay Sarkar
2023-12-15 10:17:52 +05:30
parent 7bcdbff123
commit 590c7494bd
7 changed files with 256 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
scripts/combine46b.py Normal file
View File

@@ -0,0 +1,60 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/users_14_12_2023_18_42.xlsx", sheet_name="Sheet1")
df2 = pd.read_excel(curdir + path_delim + "data/2023-11-22_to_2023-11-23.xlsx")
df2 = df2.sort_values('testTime')
variance_column = df2["led2Buffer"].var(ddof=0)
print(variance_column)
df = df1.merge(df2, on="_id", how="inner")
print(df.columns)
# # df = pd.concat([df1, df3], ignore_index=True)
df['Age'] = 2023 - df['birthYear']
df['deviceId'].hist()
plt.show()
df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]]
print(df)
df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True)
df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1)
# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y'))
# df = df.sort_values(by=['Date'], ascending=True)
df["Test Result"].fillna("NOTEST", inplace = True)
print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"]))
df = df[df["Test Result"] != "NOTEST"]
df["Blood Group"].fillna("NOBLOODGROUP", inplace = True)
print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"]))
df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last')
df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)'
df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline'
df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline'
df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE'
df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE'
print(df_final.groupby(["Test Result"]).describe())
df_count = df.groupby(["Test Result"]).describe()["ABHA ID"]["count"]
print(df.groupby(["Test Result"]).describe()["ABHA ID"]["count"])
writer = pd.ExcelWriter(curdir + path_delim + "data/Nov22b.xlsx", engine = 'openpyxl')
df_final.to_excel(writer, sheet_name = 'op', index=False)
df_count.to_excel(writer, sheet_name = "count")
writer.close()

22
scripts/combine46c.py Normal file
View File

@@ -0,0 +1,22 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/Nov22b.xlsx", sheet_name="op")
df2 = pd.read_excel(curdir + path_delim + "data/Nov22.xlsx", sheet_name="op")
# df2 = df2.sort_values('testTime')
df = df1.merge(df2, on="Sample ID", how="inner")
uncommon_df = pd.concat([df1, df2, df2]).drop_duplicates(keep=False)
# Display the resulting DataFrame
print(uncommon_df)
writer = pd.ExcelWriter(curdir + path_delim + "data/Nov22b.xlsx", engine = 'openpyxl')
uncommon_df.to_excel(writer, sheet_name = 'op', index=False)
uncommon_df.to_excel(writer, sheet_name = "count")
writer.close()

60
scripts/combine53b.py Normal file
View File

@@ -0,0 +1,60 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/users_14_12_2023_18_42.xlsx", sheet_name="Sheet1")
df2 = pd.read_excel(curdir + path_delim + "data/tests_02_12_2023_00_07.xlsx", sheet_name="data")
df2 = df2.sort_values('testTime')
variance_column = df2["led2Buffer"].var(ddof=0)
print(variance_column)
df = df2.merge(df1, on="_id", how="inner")
print(df.columns)
# # df = pd.concat([df1, df3], ignore_index=True)
df['Age'] = 2023 - df['birthYear']
df['deviceId'].hist()
plt.show()
df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]]
print(df)
df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True)
df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1)
# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y'))
# df = df.sort_values(by=['Date'], ascending=True)
df["Test Result"].fillna("NOTEST", inplace = True)
print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"]))
df = df[df["Test Result"] != "NOTEST"]
df["Blood Group"].fillna("NOBLOODGROUP", inplace = True)
print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"]))
df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last')
df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)'
df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline'
df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline'
df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE'
df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE'
print(df_final.groupby(["Test Result"]).describe())
df_count = df.groupby(["Test Result"]).describe()["ABHA ID"]["count"]
print(df.groupby(["Test Result"]).describe()["ABHA ID"]["count"])
writer = pd.ExcelWriter(curdir + path_delim + "data/Dec01b.xlsx", engine = 'openpyxl')
df_final.to_excel(writer, sheet_name = 'op', index=False)
df_count.to_excel(writer, sheet_name = "count")
writer.close()

22
scripts/combine53c.py Normal file
View File

@@ -0,0 +1,22 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/Dec01b.xlsx", sheet_name="op")
df2 = pd.read_excel(curdir + path_delim + "data/Dec01.xlsx", sheet_name="op")
# df2 = df2.sort_values('testTime')
# df = df1.merge(df2, on="Sample ID", how="inner")
uncommon_df = pd.concat([df1, df2, df2]).drop_duplicates(keep=False)
# Display the resulting DataFrame
print(uncommon_df)
writer = pd.ExcelWriter(curdir + path_delim + "data/Dec01c.xlsx", engine = 'openpyxl')
uncommon_df.to_excel(writer, sheet_name = 'op', index=False)
uncommon_df.to_excel(writer, sheet_name = "count")
writer.close()

60
scripts/combine56b.py Normal file
View File

@@ -0,0 +1,60 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/users_14_12_2023_18_42.xlsx", sheet_name="Sheet1")
df2 = pd.read_excel(curdir + path_delim + "data/tests_08_12_2023_12_46.xlsx", sheet_name="data")
df2 = df2.sort_values('testTime')
variance_column = df2["led2Buffer"].var(ddof=0)
print(variance_column)
df = df2.merge(df1, on="_id", how="inner")
print(df.columns)
# # df = pd.concat([df1, df3], ignore_index=True)
df['Age'] = 2023 - df['birthYear']
df['deviceId'].hist()
plt.show()
df = df[["_id", "name_x", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "classificationResult", "bloodGroup", "testTime", "caste", "registrationCenterName"]]
print(df)
df.rename(columns={'_id': "Sample ID", "name_x": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "testTime": "Date", "classificationResult": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "caste": "Caste", "registrationCenterName": "Center"}, inplace = True)
df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group", "Center"], axis=1)
# df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y'))
# df = df.sort_values(by=['Date'], ascending=True)
df["Test Result"].fillna("NOTEST", inplace = True)
print("NOTEST: ", len(df[df["Test Result"] == "NOTEST"]))
df = df[df["Test Result"] != "NOTEST"]
df["Blood Group"].fillna("NOBLOODGROUP", inplace = True)
print("NO BLOOD GROUP: ", len(df[df["Blood Group"] == "NOBLOODGROUP"]))
df_final = df.sort_values('Date').drop_duplicates('Sample ID', keep='last')
df_final.loc[df_final['Test Result'] == 'Normal', 'Test Result'] = 'Normal (HbA)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Trait', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'Sickle Cell Disease', 'Test Result'] = 'Sickle Cell Disease (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCT', 'Test Result'] = 'Sickle Cell Trait (HbAS)'
df_final.loc[df_final['Test Result'] == 'SCD', 'Test Result'] = 'Sickle cell Disease (HbSS)'
df_final.loc[df_final['Test Result'] == 'PBL', 'Test Result'] = 'Positive Borderline'
df_final.loc[df_final['Test Result'] == 'NBL', 'Test Result'] = 'Negative Borderline'
df_final.loc[df_final['Gender'] == 'Male', 'Gender'] = 'MALE'
df_final.loc[df_final['Gender'] == 'Female', 'Gender'] = 'FEMALE'
print(df_final.groupby(["Test Result"]).describe())
df_count = df.groupby(["Test Result"]).describe()["ABHA ID"]["count"]
print(df.groupby(["Test Result"]).describe()["ABHA ID"]["count"])
writer = pd.ExcelWriter(curdir + path_delim + "data/Dec07b.xlsx", engine = 'openpyxl')
df_final.to_excel(writer, sheet_name = 'op', index=False)
df_count.to_excel(writer, sheet_name = "count")
writer.close()

18
scripts/combine56c.py Normal file
View File

@@ -0,0 +1,18 @@
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
curdir = os.getcwd()
path_delim = '/'
df1 = pd.read_excel(curdir + path_delim + "data/Dec07b.xlsx", sheet_name="op")
df2 = pd.read_excel(curdir + path_delim + "data/Dec07.xlsx", sheet_name="op")
uncommon_df = pd.concat([df1, df2, df2]).drop_duplicates(keep=False)
print(uncommon_df)
writer = pd.ExcelWriter(curdir + path_delim + "data/Dec07c.xlsx", engine = 'openpyxl')
uncommon_df.to_excel(writer, sheet_name = 'op', index=False)
uncommon_df.to_excel(writer, sheet_name = "count")
writer.close()

14
scripts/uncommon.py Normal file
View File

@@ -0,0 +1,14 @@
import pandas as pd
# Sample DataFrames
df1 = pd.DataFrame({'A': [1, 2, 3, 4],
'B': ['a', 'b', 'c', 'd']})
df2 = pd.DataFrame({'A': [3, 4, 5, 6],
'B': ['c', 'd', 'e', 'f']})
# Find uncommon elements
uncommon_df = pd.concat([df1, df2, df2]).drop_duplicates(keep=False)
# Display the resulting DataFrame
print(uncommon_df)