diff --git a/scripts/consolidated_data.py b/scripts/consolidated_data.py index 42439cb..cfb2b29 100644 --- a/scripts/consolidated_data.py +++ b/scripts/consolidated_data.py @@ -1,62 +1,75 @@ import os import firebase_admin -from IPython.core.display import Image -from IPython.core.display_functions import display from firebase_admin import credentials from firebase_admin import firestore +from google.cloud.firestore_v1.base_query import FieldFilter import pandas as pd import datetime +import sys +import platform +if __name__ == "__main__": -# Initialize Firebase Admin SDK -cred = credentials.Certificate(r'C:\Users\smila\PycharmProjects\pythonProject\hpos-prod-firebase-adminsdk-bionp-5486f7becd.json') # Replace with your own service account key path + path_delim = "" + if platform.system() == 'Darwin': + path_delim = '/' + else: + path_delim = '\\' + + key_file_path = os.getcwd() + path_delim + "keys" + path_delim + "hpos-af3cc-firebase-adminsdk-n261k-2bfd463ec0.json" + cred = credentials.Certificate(key_file_path) # Replace with your own service account key path + firebase_admin.initialize_app(cred) -firebase_admin.initialize_app(cred) + # Get a reference to the Firestore database + db = firestore.client() -# Get a reference to the Firestore database -db = firestore.client() + # Specify the collections + patient_collection = db.collection("patientData") + test_collection = db.collection("testData") + start_date = sys.argv[1] # '2023-01-12' #input("Please enter the start date (yyyy-mm-dd): ") + end_date = sys.argv[2] #'2023-07-16' #input("Please enter the end date (yyyy-mm-dd): ") -# Specify the collections -patient_collection = db.collection("patientData") -test_collection = db.collection("testData") -start_date = input("Please enter the start date (yyyy-mm-dd): ") -end_date = input("Please enter the end date (yyyy-mm-dd): ") + query = test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date)) + patient_docs = query.stream() -query = patient_collection.where("createdAt", ">=", start_date).where("createdAt", "<", end_date) -patient_docs = query.stream() + # Prepare data to store in CSV + data = [] + for patient_doc in patient_docs: + print(patient_doc) + patient_data = patient_doc.to_dict() + patient_id = patient_data["_id"] -# Prepare data to store in CSV -data = [] -for patient_doc in patient_docs: - patient_data = patient_doc.to_dict() - patient_id = patient_data["_id"] + # Query the document from testData collection based on the common _id + test_docs = test_collection.where("_id", "==", patient_id).stream() - # Query the document from testData collection based on the common _id - test_docs = test_collection.where("_id", "==", patient_id).stream() + for test_doc in test_docs: + print(test_doc) + test_data = test_doc.to_dict() - for test_doc in test_docs: - test_data = test_doc.to_dict() + # Combine the data from both collections into a single dictionary + combined_data = {**patient_data, **test_data} - # Combine the data from both collections into a single dictionary - combined_data = {**patient_data, **test_data} + # Fill empty fields in test_data with corresponding values from patient_data + for key, value in combined_data.items(): + if value == "" and key in patient_data: + combined_data[key] = patient_data[key] - # Skip if the csvPath is not a valid URL + data.append(combined_data) - data.append(combined_data) + # Convert the data to a DataFrame + df = pd.DataFrame(data) + print(df) + print(df.size) -# Convert the data to a DataFrame -df = pd.DataFrame(data) -print(df.size) + # Save the DataFrame to a CSV file + output_filename = "data.csv" + df.to_csv(output_filename, index=False) -# Save the DataFrame to a CSV file -output_filename = "data.csv" -df.to_csv(output_filename, index=False) + downloads_dir = os.path.join(os.path.expanduser("~"), "Downloads") + output_path = os.path.join(downloads_dir, output_filename) + df.to_csv(output_path, index=False) -downloads_dir = os.path.join(os.path.expanduser("~"), "Downloads") -output_path = os.path.join(downloads_dir, output_filename) -df.to_csv(output_path, index=False) + print(f"Data saved to '{output_path}'") -print(f"Data saved to '{output_path}'") - -# Close the Firebase Admin SDK -firebase_admin.delete_app(firebase_admin.get_app()) + # Close the Firebase Admin SDK + firebase_admin.delete_app(firebase_admin.get_app())