diff --git a/scripts/tsne1.py b/scripts/tsne1.py new file mode 100644 index 0000000..6d7011d --- /dev/null +++ b/scripts/tsne1.py @@ -0,0 +1,67 @@ +import os +import firebase_admin +from firebase_admin import credentials +from firebase_admin import firestore +from google.cloud.firestore_v1.base_query import FieldFilter +import pandas as pd +from datetime import datetime +import sys +import matplotlib.pyplot as plt +import numpy as np + +cred = credentials.Certificate(os.getcwd() + '/' + 'keys/hpos-qa-firebase-adminsdk.json') +firebase_admin.initialize_app(cred) + +db = firestore.client() + +patient_collection = db.collection("patientData") +test_collection = db.collection("testData") +start_date = '2024-02-01' #sys.argv[1] +end_date = '2024-02-25' #sys.argv[2] + +query = test_collection.where(filter=FieldFilter("testTime", ">=", start_date)).where(filter=FieldFilter("testTime", "<", end_date)) +docs = query.stream() + +data = [] +for doc in docs: + doc_data = doc.to_dict() + data.append(doc_data) + +df = pd.DataFrame(data) +print(df) +print(df.size) + + +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.manifold import TSNE +from sklearn.preprocessing import StandardScaler + +numeric_columns = ['led1Buffer', 'led3Average', + 'led3Sample', 'led4Average', 'calculatedRatio', + 'led4Sample', 'led2Sample', 'led3Buffer', 'deviceRatio', + 'led1Sample', 'led1Average', 'led4Buffer', 'led2Average', + 'led2Buffer'] #df.select_dtypes(include=[float, int]).columns +print(numeric_columns) + +df_numeric = df[numeric_columns]#.dropna() +df_numeric.replace([np.inf, -np.inf], np.nan, inplace=True) +df_numeric.dropna(inplace=True) + +if df_numeric.shape[0] > 0: + scaler = StandardScaler() + df_standardized = pd.DataFrame(scaler.fit_transform(df_numeric), columns=df_numeric.columns) + + tsne = TSNE(n_components=2, random_state=42) + df_tsne = tsne.fit_transform(df_standardized) + + plt.scatter(df_tsne[:, 0], df_tsne[:, 1]) + plt.title("t-SNE Visualization") + plt.xlabel("t-SNE Component 1") + plt.ylabel("t-SNE Component 2") + plt.show() +else: + print("No samples found in the DataFrame after dropping NaN values.") + +firebase_admin.delete_app(firebase_admin.get_app()) +