diff --git a/cloud-functions/python/functions/consolidation/requirements.txt b/cloud-functions/python/functions/consolidation/requirements.txt new file mode 100644 index 0000000..0a87d59 --- /dev/null +++ b/cloud-functions/python/functions/consolidation/requirements.txt @@ -0,0 +1,6 @@ +functions-framework==3.* +firebase_functions~=0.1.0 +pandas==2.0.3 +openpyxl==3.1.2 +firebase-admin==6.2.0 +google-cloud-firestore==2.14.0 \ No newline at end of file diff --git a/scripts/consolidated_data.py b/scripts/consolidated_data.py index f24b551..d759b83 100644 --- a/scripts/consolidated_data.py +++ b/scripts/consolidated_data.py @@ -9,7 +9,7 @@ import sys import platform from datetime import datetime, timedelta -environment = "af3cc" +environment = "qa" if __name__ == "__main__": @@ -67,11 +67,22 @@ if __name__ == "__main__": data.append(test_data) # Convert the data to a DataFrame - df = pd.DataFrame(data) - print(df) + test_df = pd.DataFrame(data) + print(test_df) + print(test_df.columns) + + query = db.collection("patientData").where(filter=FieldFilter("createdAt", ">=", start_date)).where(filter=FieldFilter("createdAt", "<", end_date)) + data = [] + docs = query.stream() + for doc in docs: + row_data = doc.to_dict() + data.append(row_data) + + user_df = pd.DataFrame(data) + + df = pd.merge(test_df, user_df, on='_id') print(df.columns) - - + # Save the DataFrame to a CSV file output_filename = f'data_{environment}_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx' df.to_csv(output_filename, index=False) @@ -82,7 +93,7 @@ if __name__ == "__main__": df = df[(df['testTime'] > start_date) & (df['testTime'] <= end_date)] df = df.sort_values(by=['testTime'], ascending=False) - column_names = ["_id", "classificationResult", "deviceRatioClass", "prdClassification", "slopeRatioClass", "predictedDenovixRatio", "slopeRatio", "calculatedRatio", "deviceRatio", "kitSerial", "led1Gain1", "led1Gain2", "led1Gain4", "abs1", "led1Average", "led1Buffer", "led1Sample", "led2Gain1", "led2Gain2", "led2Gain4", "abs2", "led2Average", "led2Buffer", "led2Sample", "led3Gain1", "led3Gain2", "led3Gain4", "hb3", "abs3", "led3Average", "led3Buffer", "led3Sample", "led4Gain1", "led4Gain2", "led4Gain4", "hb4", "abs4", "led4Average", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "solution", "concentration", "volume", "errorMessages", "deviceId", "deviceSerialNumber", "appVersion", "name", "testTime"] + column_names = ["_id", "classificationResult", "deviceRatioClass", "prdClassification", "slopeRatioClass", "predictedDenovixRatio", "slopeRatio", "calculatedRatio", "deviceRatio", "kitSerial", "led1Gain1", "led1Gain2", "led1Gain4", "abs1", "led1Average", "led1Buffer", "led1Sample", "led2Gain1", "led2Gain2", "led2Gain4", "abs2", "led2Average", "led2Buffer", "led2Sample", "led3Gain1", "led3Gain2", "led3Gain4", "hb3", "abs3", "led3Average", "led3Buffer", "led3Sample", "led4Gain1", "led4Gain2", "led4Gain4", "hb4", "abs4", "led4Average", "led4Average", "led4Buffer", "led4Sample", "batteryLevel", "batteryVoltage", "solution", "concentration", "volume", "errorMessages", "deviceId", "deviceSerialNumber", "appVersion", "name_y", "testTime"] common_columns = [col for col in column_names if col in df.columns] df = df[common_columns]