From bddd2a137c9b94054deaa40584dddf3142759216 Mon Sep 17 00:00:00 2001 From: Pritimay Sarkar Date: Mon, 21 Aug 2023 19:14:24 +0530 Subject: [PATCH] add data download api --- .gitignore | 3 +- .../python/functions/consolidation.py | 88 +++++++++++++++++++ cloud-functions/python/functions/main.py | 72 ++++++++++++++- .../python/functions/requirements.txt | 5 +- requirements.txt | 42 +++++++++ scripts/consolidated_data.py | 2 +- 6 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 cloud-functions/python/functions/consolidation.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index afef899..3b81fe3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ env/ .DS_Store *.xlsx *.csv -*.json \ No newline at end of file +*.json +cloud-functions/python/functions/__pycache__/ \ No newline at end of file diff --git a/cloud-functions/python/functions/consolidation.py b/cloud-functions/python/functions/consolidation.py new file mode 100644 index 0000000..7f31911 --- /dev/null +++ b/cloud-functions/python/functions/consolidation.py @@ -0,0 +1,88 @@ +import functions_framework +from firebase_functions import https_fn +# import firebase_admin +from firebase_admin import initialize_app, firestore +import pandas as pd +import datetime +import sys +import platform +import os +import io +from flask import send_file + +initialize_app() + +@functions_framework.http +def consolidation(request): + """HTTP Cloud Function. + Args: + request (flask.Request): The request object. + + Returns: + The response text, or any set of values that can be turned into a + Response object using `make_response` + . + """ + request_json = request.get_json(silent=True) + request_args = request.args + + if request_json and 'start_date' in request_json and 'end_date' in request_json: + start_date = request_json['start_date'] + end_date = request_json['end_date'] + elif request_args and 'start_date' in request_args and 'end_date' in request_args: + start_date = request_args['start_date'] + end_date = request_args['end_date'] + else: + start_date = "2023-08-20" + end_date = "2023-08-21" + + path_delim = "/" + + db = firestore.client() + + patient_collection = db.collection("patientData") + test_collection = db.collection("testData") + + query = test_collection + patient_docs = query.stream() + + data = [] + for patient_doc in patient_docs: + patient_data = patient_doc.to_dict() + patient_id = patient_data["_id"] + + test_docs = test_collection.where("_id", "==", patient_id).stream() + + for test_doc in test_docs: + test_data = test_doc.to_dict() + + combined_data = {**patient_data, **test_data} + + for key, value in combined_data.items(): + if value == "" and key in patient_data: + combined_data[key] = patient_data[key] + + data.append(combined_data) + + df = pd.DataFrame(data) + + output_filename = "data.xlsx" + + output_path = os.getcwd() + path_delim + output_filename + writer = pd.ExcelWriter(output_path, engine = 'openpyxl') + df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)] + df = df.sort_values(by=['testTime'], ascending=False) + df = df.drop(['resultData', "reportUploadTime", "userImageURL", "testType", "birthYear", "testStatus", "reportPath", "createdBy", "csvPath", "result", "mobileId", "resultRatio", "localFlag", "led2", "led1"], axis=1) + df.rename(columns={'deviceSerialNumber': "loginId", "calculatedRatio": "calibratedRatio"}, inplace = True) + df.to_excel(writer, sheet_name = 'data', index=False) + writer.close() + + # firebase_admin.delete_app(firebase_admin.get_app()) + + # return 'df size {}!'.format(df.size) + + with open(output_path,'rb') as f: + file_data = io.BytesIO(f.read()) + + # application/vnd.ms-excel + return send_file(file_data, download_name= "data.xlsx", mimetype="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") \ No newline at end of file diff --git a/cloud-functions/python/functions/main.py b/cloud-functions/python/functions/main.py index 52f04f1..d86b3eb 100644 --- a/cloud-functions/python/functions/main.py +++ b/cloud-functions/python/functions/main.py @@ -1,7 +1,12 @@ # Deploy with `firebase deploy` from firebase_functions import https_fn -from firebase_admin import initialize_app +from firebase_admin import initialize_app, firestore +import pandas as pd +import datetime +import sys +import platform +import os # initialize_app() # @@ -10,8 +15,69 @@ from firebase_admin import initialize_app # def on_request_example(req: https_fn.Request) -> https_fn.Response: # return https_fn.Response("Hello world!") + +# @https_fn.on_request( +# cors=options.CorsOptions( +# cors_origins=[r"firebase\.com$", r"https://flutter\.com"], +# cors_methods=["get", "post"], +# ) +# ) + initialize_app() @https_fn.on_request() -def on_request_example(req: https_fn.Request) -> https_fn.Response: - return https_fn.Response("Hello world!") +def consolidation(req: https_fn.Request) -> https_fn.Response: + + path_delim = "/" + + db = firestore.client() + + patient_collection = db.collection("patientData") + test_collection = db.collection("testData") + + start_date = req.query.start_date + print("start_date", start_date) + end_date = req.query.end_date + print("end_date", end_date) + + + query = test_collection # test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date)) + patient_docs = query.stream() + + data = [] + for patient_doc in patient_docs: + patient_data = patient_doc.to_dict() + patient_id = patient_data["_id"] + + test_docs = test_collection.where("_id", "==", patient_id).stream() + + for test_doc in test_docs: + test_data = test_doc.to_dict() + + combined_data = {**patient_data, **test_data} + + for key, value in combined_data.items(): + if value == "" and key in patient_data: + combined_data[key] = patient_data[key] + + data.append(combined_data) + + df = pd.DataFrame(data) + + output_filename = "data.xlsx" + df.to_csv(output_filename, index=False) + + output_path = os.getcwd() + path_delim + output_filename + writer = pd.ExcelWriter(output_path, engine = 'openpyxl') + df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)] + df = df.sort_values(by=['testTime'], ascending=False) + df = df.drop(['resultData', "reportUploadTime", "userImageURL", "testType", "birthYear", "testStatus", "reportPath", "createdBy", "csvPath", "result", "mobileId", "resultRatio", "localFlag", "led2", "led1"], axis=1) + df.to_excel(writer, sheet_name = 'data', index=False) + writer.close() + + firebase_admin.delete_app(firebase_admin.get_app()) + + + # return https_fn.Response(response = send_file(output_path)) + + return https_fn.Response("Ok!") diff --git a/cloud-functions/python/functions/requirements.txt b/cloud-functions/python/functions/requirements.txt index bcf4c12..1533b06 100644 --- a/cloud-functions/python/functions/requirements.txt +++ b/cloud-functions/python/functions/requirements.txt @@ -1 +1,4 @@ -firebase_functions~=0.1.0 \ No newline at end of file +functions-framework==3.* +firebase_functions~=0.1.0 +pandas==2.0.3 +openpyxl==3.1.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b6430d3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,42 @@ +CacheControl==0.13.1 +cachetools==5.3.1 +certifi==2023.7.22 +cffi==1.15.1 +charset-normalizer==3.2.0 +cryptography==41.0.3 +et-xmlfile==1.1.0 +firebase-admin==6.2.0 +google-api-core==2.11.1 +google-api-python-client==2.97.0 +google-auth==2.22.0 +google-auth-httplib2==0.1.0 +google-cloud-core==2.3.3 +google-cloud-firestore==2.11.1 +google-cloud-storage==2.10.0 +google-crc32c==1.5.0 +google-resumable-media==2.5.0 +googleapis-common-protos==1.60.0 +grpcio==1.57.0 +grpcio-status==1.57.0 +httplib2==0.22.0 +idna==3.4 +msgpack==1.0.5 +numpy==1.25.2 +openpyxl==3.1.2 +pandas==2.0.3 +proto-plus==1.22.3 +protobuf==4.24.0 +pyasn1==0.5.0 +pyasn1-modules==0.3.0 +pycparser==2.21 +PyJWT==2.8.0 +pyparsing==3.1.1 +python-dateutil==2.8.2 +pytz==2023.3 +requests==2.31.0 +rsa==4.9 +six==1.16.0 +tzdata==2023.3 +uritemplate==4.1.1 +urllib3==1.26.16 +XlsxWriter==3.1.2 diff --git a/scripts/consolidated_data.py b/scripts/consolidated_data.py index 11a9a9a..30b0d12 100644 --- a/scripts/consolidated_data.py +++ b/scripts/consolidated_data.py @@ -77,4 +77,4 @@ if __name__ == "__main__": print(f"Data saved to '{output_path}'") # Close the Firebase Admin SDK - firebase_admin.delete_app(firebase_admin.get_app()) + # firebase_admin.delete_app(firebase_admin.get_app())