From 46367353e8cabd7366ccb609b5ec1ecc1752a869 Mon Sep 17 00:00:00 2001 From: Pritimay Sarkar Date: Thu, 21 Sep 2023 14:09:02 +0530 Subject: [PATCH] combine5 --- scripts/combine5.py | 31 +++++++++++++++++++++++++++++++ scripts/consolidated_data.py | 4 ++-- scripts/data_diff.py | 33 +++++++++++++++++++++++++++++++++ scripts/describe1.py | 14 ++++++++++++++ scripts/test_collection.py | 11 +++++++---- scripts/untested.py | 0 6 files changed, 87 insertions(+), 6 deletions(-) create mode 100644 scripts/combine5.py create mode 100644 scripts/data_diff.py create mode 100644 scripts/describe1.py create mode 100644 scripts/untested.py diff --git a/scripts/combine5.py b/scripts/combine5.py new file mode 100644 index 0000000..36aef9d --- /dev/null +++ b/scripts/combine5.py @@ -0,0 +1,31 @@ +import pandas as pd +import os +import numpy as np + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/July_Sept3.xlsx", sheet_name="op") +df2 = pd.read_excel(curdir + path_delim + "data/diff_check.xlsx", sheet_name="op") + +print(df2) +df = df2.merge(df1, on="_id", how='outer') +print(df.columns) +# df = pd.concat([df1, df3], ignore_index=True) +df['Age'] = 2023 - df['birthYear'] + + +df = df[["_id", "name", "abhaId", "aadharId", "Age", "gender", "category", "maritalStatus", "house", "district", "state", "pinCode", "phoneNumber", "Test Result_x", "bloodGroup", "createdAt", "Caste"]] +# print(df.columns) + +df.rename(columns={'_id': "Sample ID", "name": "Name", "abhaId": "ABHA ID", "aadharId": "Aadhaar ID", "gender": "Gender", "category": "Category", "maritalStatus": "Marital Status", "house": "Address", "district": "District", "state": "State", "pinCode": "Pincode", "phoneNumber": "Mobile Number", "createdAt": "Date", "Test Result_x": "Test Result", "bloodGroup": "Blood Group", "Age": "Age", "Caste": "Caste"}, inplace = True) +df = df.reindex(["Sample ID", "Name", "ABHA ID", "Aadhaar ID", "Age", "Gender", "Caste", "Category", "Marital Status", "Address", "District", "State", "Pincode", "Mobile Number", "Date", "Test Result", "Blood Group"], axis=1) +df['Date'] = pd.to_datetime(df["Date"].dt.strftime('%d-%m-%Y')) + +df.loc[df['Test Result'] == np.nan, 'Test Result'] = 'Retest' +# df.loc[df['Test Result'] == "Inconclusive", 'Test Result'] = 'Retest' +print(df) + +writer = pd.ExcelWriter(curdir + path_delim + "data/July_Sept4.xlsx", engine = 'openpyxl') +df.to_excel(writer, sheet_name = 'op', index=False) +writer.close() + diff --git a/scripts/consolidated_data.py b/scripts/consolidated_data.py index 8946872..da0b1b3 100644 --- a/scripts/consolidated_data.py +++ b/scripts/consolidated_data.py @@ -16,7 +16,7 @@ if __name__ == "__main__": else: path_delim = '/' - key_file_path = os.getcwd() + path_delim + "keys" + path_delim + "hpos-preprod-firebase-adminsdk-d6jjt-ae42ad0f67.json" + key_file_path = os.getcwd() + path_delim + "keys" + path_delim + "hpos-prod-firebase-adminsdk-bionp-3c041b2300.json" cred = credentials.Certificate(key_file_path) # Replace with your own service account key path firebase_admin.initialize_app(cred) @@ -79,7 +79,7 @@ if __name__ == "__main__": df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)] df = df.sort_values(by=['testTime'], ascending=False) - df = df[["_id", "calculatedRatio", "deviceId", "deviceRatio", "deviceType", "kitSerial", "led1Average", "led1Buffer", "led1Sample", "led2Average", "led2Buffer", "led2Sample", "location", "deviceSerialNumber", "name", "testTime", "classificationResult"]] + df = df[["_id", "calculatedRatio", "deviceId", "deviceRatio", "deviceType", "kitSerial", "led1Average", "led1Buffer", "led1Sample", "led2Average", "led2Buffer", "led2Sample", "location", "deviceSerialNumber", "name", "testTime", "classificationResult", "resultData"]] # df.rename(columns={'deviceSerialNumber': "login_id", "calculatedRatio": "calibrated_ratio", "led1Buffer": "427_buffer_intensity", "led2Buffer": "555_buffer_intensity", "led1Sample": "427_sample_intensity", "led2Sample": "555_sample_intensity", "led1Average": "427_absorbance", "led2Average": "555_absorbance"}, inplace = True) df = df.reindex(sorted(df.columns), axis=1) diff --git a/scripts/data_diff.py b/scripts/data_diff.py new file mode 100644 index 0000000..3d30b93 --- /dev/null +++ b/scripts/data_diff.py @@ -0,0 +1,33 @@ +import pandas as pd +import os + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/all_users_14_07_2023_to_12_09_2023.xlsx", sheet_name="Sheet1") +df2 = pd.read_excel(curdir + path_delim + "data/all_users_14_07_2023_to_12_09_2023.xlsx", sheet_name="Sheet2") +df3 = pd.read_excel(curdir + path_delim + "data/July_Sept2 copy.xlsx", sheet_name="op") + +### merging July +df = df1.merge(df2, on="_id", how='outer') +# df = pd.concat([df1, df3], ignore_index=True) +print(df) + +# sample_ids = df["_id"].tolist() +# print(len(sample_ids)) + +print(len(df['_id']) - len(df['_id'].drop_duplicates())) + +# df_dup = df['_id']-df['_id'].drop_duplicates() +# print(df_dedup) + +# dfut = pd.merge(df1, df2, how='outer', +# left_index=True, right_on=['_id', 'Sample ID'], +# indicator=True) + +# print(dfut) +# dfut.query('_merge != "both"') + +writer = pd.ExcelWriter(curdir + path_delim + "data/diff_check.xlsx", engine = 'openpyxl') +df.to_excel(writer, sheet_name = 'op', index=False) +writer.close() + diff --git a/scripts/describe1.py b/scripts/describe1.py new file mode 100644 index 0000000..07061ed --- /dev/null +++ b/scripts/describe1.py @@ -0,0 +1,14 @@ +import pandas as pd +import os +import numpy as np + +curdir = os.getcwd() +path_delim = '/' +df1 = pd.read_excel(curdir + path_delim + "data/July_Sept4.xlsx", sheet_name="op") + +df1.loc[df1['Test Result'] == 'Sickle cell Trait (HbAS)', 'Test Result'] = 'Sickle Cell Trait (HbAS)' +print(df1.groupby(['Test Result']).describe()) + +writer = pd.ExcelWriter(curdir + path_delim + "data/July_Sept5.xlsx", engine = 'openpyxl') +df1.to_excel(writer, sheet_name = 'op', index=False) +writer.close() \ No newline at end of file diff --git a/scripts/test_collection.py b/scripts/test_collection.py index ae22910..e7e6c3b 100644 --- a/scripts/test_collection.py +++ b/scripts/test_collection.py @@ -4,7 +4,7 @@ from firebase_admin import credentials from firebase_admin import firestore from google.cloud.firestore_v1.base_query import FieldFilter import pandas as pd -import datetime +from datetime import datetime import sys import os @@ -53,13 +53,16 @@ df = pd.DataFrame(data) print(df) print(df.size) +df = df[["_id", "classificationResult", "calculatedRatio", "deviceId", "deviceRatio", "deviceType", "kitSerial", "led1Average", "led1Buffer", "led1Sample", "led2Average", "led2Buffer", "led2Sample", "location", "deviceSerialNumber", "name", "testTime"]] + # Save the DataFrame to a CSV file -output_filename = "data.csv" -df.to_csv(output_filename, index=False) +output_filename = f'tests_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx' downloads_dir = os.path.join(os.path.expanduser("~"), "Downloads") output_path = os.path.join(downloads_dir, output_filename) -df.to_csv(output_path, index=False) +writer = pd.ExcelWriter(output_path, engine = 'openpyxl') +df.to_excel(writer, sheet_name = 'data', index=False) +writer.close() print(f"Data saved to '{output_path}'") diff --git a/scripts/untested.py b/scripts/untested.py new file mode 100644 index 0000000..e69de29