fix errors in auto dye analysis
This commit is contained in:
Pritimay Sarkar
@@ -68,164 +68,165 @@ df_reference_device.columns.name = 'Wavelength'
|
||||
|
||||
df = pd.read_excel(curdir + path_delim + "data" + path_delim + "data_06_12_2023_12_47.xlsx", sheet_name="data")
|
||||
|
||||
texts_to_check = ['T', 'K']
|
||||
df['solution'] = df['name'].str.extract(f"({'|'.join(texts_to_check)})", flags=re.IGNORECASE)
|
||||
df['solution'] = df['solution'].replace({'T': 'Tartrazine', 'K': 'KMnO4'}, regex=True)
|
||||
|
||||
def extract_numbers(s):
|
||||
match = re.match(r'\d+', s)
|
||||
if match:
|
||||
return int(match.group())
|
||||
else:
|
||||
return None
|
||||
|
||||
df["concentration"] = df['name'].apply(extract_numbers)
|
||||
df['absorbance'] = df.apply(lambda row: row['led2Average'] if row['solution'] == 'Tartrazine' else row['led1Average'], axis=1)
|
||||
|
||||
df_precision_acc = df[["deviceId", "solution", "concentration", "led1Average", "led2Average", "absorbance"]].groupby(["deviceId", "solution", "concentration"]).describe()["absorbance"][["count", "min", "max", "mean"]]
|
||||
df_precision_acc['precision'] = df_precision_acc['max'] - df_precision_acc['min']
|
||||
|
||||
df_precision_acc['precision_result'] = ['Fail' if diff > precision_threshold else 'Pass' for diff in df_precision_acc['precision']]
|
||||
|
||||
df_precision_acc.reset_index(inplace=True)
|
||||
df_precision_acc.set_index(["deviceId", "solution", "concentration"], inplace=True)
|
||||
|
||||
device_precision_results = {}
|
||||
for index, group_df in df_precision_acc.groupby(level=[0, 1, 2]):
|
||||
soln = index[1]
|
||||
concen = index[2]
|
||||
result_values = group_df['precision_result'].values
|
||||
|
||||
if index[0] not in device_precision_results:
|
||||
device_precision_results[index[0]] = "Pass"
|
||||
else:
|
||||
result = 'Fail' if 'Fail' in result_values else 'Pass'
|
||||
if result == 'Fail':
|
||||
device_precision_results[index[0]] = 'Fail'
|
||||
if not df.empty:
|
||||
conditions = [df['name'].str.contains('Tar', case=False, na=False),
|
||||
df['name'].str.contains('KM', case=False, na=False)]
|
||||
|
||||
df_device_precicion_results = pd.DataFrame(list(device_precision_results.items()), columns=['Device', 'Result'])
|
||||
|
||||
df_precision_acc = df_precision_acc.assign(accuracy='', accuracy_result='')
|
||||
|
||||
device_accuracy_results = {}
|
||||
|
||||
for idx, row in df_precision_acc.iterrows():
|
||||
device_id = row.name[0]
|
||||
solution = row.name[1]
|
||||
concentration = str(int(row.name[2]))
|
||||
# print(concentration)
|
||||
reference_value = denovix_reference_values[solution][concentration]
|
||||
choices = ['Tartrazine', 'KMnO4']
|
||||
|
||||
try:
|
||||
reference_value = denovix_reference_values[solution][concentration]
|
||||
accuracy = math.fabs(row['mean'] - reference_value)
|
||||
df_precision_acc.at[idx, 'accuracy'] = accuracy
|
||||
|
||||
accuracy_result = 'Pass' if accuracy < accuracy_threshold else 'Fail'
|
||||
df_precision_acc.at[idx, 'accuracy_result'] = accuracy_result
|
||||
if device_id not in device_accuracy_results:
|
||||
device_accuracy_results[device_id] = accuracy_result
|
||||
df['solution'] = np.select(conditions, choices, default=None)
|
||||
|
||||
def extract_numbers(s):
|
||||
match = re.search(r'-(\d+)', s) or re.search(r'(\d+)', s)
|
||||
if match:
|
||||
return int(match.group(1))
|
||||
else:
|
||||
if accuracy_result == 'Fail':
|
||||
device_accuracy_results[device_id] = 'Fail'
|
||||
except KeyError:
|
||||
df_precision_acc.at[idx, 'accuracy'] = np.nan
|
||||
df_precision_acc.at[idx, 'accuracy_result'] = 'Fail'
|
||||
return None
|
||||
|
||||
df_device_accuracy_results = pd.DataFrame(list(device_accuracy_results.items()), columns=['Device', 'Result'])
|
||||
df["concentration"] = df['name'].apply(extract_numbers)
|
||||
df['absorbance'] = df.apply(lambda row: row['led2Average'] if row['solution'] == 'Tartrazine' else row['led1Average'], axis=1)
|
||||
|
||||
device_results = {}
|
||||
df_precision_acc = df[["deviceId", "solution", "concentration", "led1Average", "led2Average", "absorbance"]].groupby(["deviceId", "solution", "concentration"]).describe()["absorbance"][["count", "min", "max", "mean"]]
|
||||
df_precision_acc['precision'] = df_precision_acc['max'] - df_precision_acc['min']
|
||||
|
||||
df_device_results = pd.merge(df_device_precicion_results, df_device_accuracy_results, on='Device', how='outer', suffixes=('_precision', '_accuracy'))
|
||||
df_device_results['Result'] = np.where((df_device_results['Result_precision'] == 'Fail') | (df_device_results['Result_accuracy'] == 'Fail'), 'Fail', 'Pass')
|
||||
df_precision_acc['precision_result'] = ['Fail' if diff > precision_threshold else 'Pass' for diff in df_precision_acc['precision']]
|
||||
|
||||
df_precision_acc.reset_index(inplace=True)
|
||||
df_precision_acc.set_index(["deviceId", "solution", "concentration"], inplace=True)
|
||||
|
||||
output_filename = f'data/accuracy_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx'
|
||||
writer = pd.ExcelWriter(output_filename, engine = 'xlsxwriter')
|
||||
df_device_results.to_excel(writer, sheet_name="device_results")
|
||||
df_precision_acc.to_excel(writer, sheet_name="concentration")
|
||||
df.to_excel(writer, sheet_name="in")
|
||||
df_reference_device.to_excel(writer, sheet_name="reference_device")
|
||||
device_precision_results = {}
|
||||
for index, group_df in df_precision_acc.groupby(level=[0, 1, 2]):
|
||||
soln = index[1]
|
||||
concen = index[2]
|
||||
result_values = group_df['precision_result'].values
|
||||
|
||||
workbook = writer.book
|
||||
worksheet_device_results = writer.sheets['device_results']
|
||||
|
||||
red_format = workbook.add_format({'bg_color': '#FFC7CE', 'font_color': '#9C0006'})
|
||||
green_format = workbook.add_format({'bg_color': '#C6EFCE', 'font_color': '#006100'})
|
||||
|
||||
worksheet_device_results.conditional_format('E2:E{}'.format(df_device_results.shape[0] + 1), {'type': 'text',
|
||||
'criteria': 'containing',
|
||||
'value': 'Fail',
|
||||
'format': red_format})
|
||||
|
||||
worksheet_device_results.conditional_format('E2:E{}'.format(df_device_results.shape[0] + 1), {'type': 'text',
|
||||
'criteria': 'containing',
|
||||
'value': 'Pass',
|
||||
'format': green_format})
|
||||
|
||||
|
||||
wks1 = workbook.add_worksheet('abs_plot')
|
||||
wks1.write(0,0,'Abs Plot')
|
||||
|
||||
# Write the header
|
||||
wks1.write(0, 0, 'Solution')
|
||||
wks1.write(0, 1, 'Slope')
|
||||
wks1.write(0, 2, 'Intercept')
|
||||
row = 1
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
|
||||
unique_solutions = df['solution'].unique()
|
||||
|
||||
for solution in unique_solutions:
|
||||
if pd.notna(solution):
|
||||
solution_data = df[df['solution'] == solution]
|
||||
if index[0] not in device_precision_results:
|
||||
device_precision_results[index[0]] = "Pass"
|
||||
else:
|
||||
result = 'Fail' if 'Fail' in result_values else 'Pass'
|
||||
if result == 'Fail':
|
||||
device_precision_results[index[0]] = 'Fail'
|
||||
|
||||
concentration_means = solution_data.groupby('concentration')['absorbance'].mean()
|
||||
df_device_precicion_results = pd.DataFrame(list(device_precision_results.items()), columns=['Device', 'Result'])
|
||||
|
||||
df_precision_acc = df_precision_acc.assign(accuracy='', accuracy_result='')
|
||||
|
||||
device_accuracy_results = {}
|
||||
|
||||
for idx, row in df_precision_acc.iterrows():
|
||||
device_id = row.name[0]
|
||||
solution = row.name[1]
|
||||
concentration = str(int(row.name[2]))
|
||||
# print(concentration)
|
||||
|
||||
# Fit a linear regression model
|
||||
coefficients = np.polyfit(concentration_means.index, concentration_means.values, 1)
|
||||
|
||||
# Print the coefficients
|
||||
print(f'Coefficients for {solution}: Slope={coefficients[0]}, Intercept={coefficients[1]}')
|
||||
try:
|
||||
reference_value = denovix_reference_values[solution][concentration]
|
||||
accuracy = math.fabs(row['mean'] - reference_value)
|
||||
df_precision_acc.at[idx, 'accuracy'] = accuracy
|
||||
|
||||
accuracy_result = 'Pass' if accuracy < accuracy_threshold else 'Fail'
|
||||
df_precision_acc.at[idx, 'accuracy_result'] = accuracy_result
|
||||
if device_id not in device_accuracy_results:
|
||||
device_accuracy_results[device_id] = accuracy_result
|
||||
else:
|
||||
if accuracy_result == 'Fail':
|
||||
device_accuracy_results[device_id] = 'Fail'
|
||||
except KeyError:
|
||||
df_precision_acc.at[idx, 'accuracy'] = np.nan
|
||||
df_precision_acc.at[idx, 'accuracy_result'] = 'Fail'
|
||||
|
||||
# Plot the mean absorbance
|
||||
ax.plot(concentration_means.index, concentration_means.values, marker='o', linestyle='-', label=f'Mean Absorbance for {solution}')
|
||||
df_device_accuracy_results = pd.DataFrame(list(device_accuracy_results.items()), columns=['Device', 'Result'])
|
||||
|
||||
# Plot the linear fit
|
||||
ax.plot(concentration_means.index, np.polyval(coefficients, concentration_means.index), linestyle='--', label=f'Linear Fit for {solution}')
|
||||
|
||||
# Add coefficients next to the linear fit trendline
|
||||
annotation_text = f'Slope: {coefficients[0]:.4f}\nIntercept: {coefficients[1]:.4f}'
|
||||
ax.text(concentration_means.index[-1] + 5, np.polyval(coefficients, concentration_means.index[-1]), annotation_text, fontsize=10, verticalalignment='center')
|
||||
device_results = {}
|
||||
|
||||
# Write the coefficients to the worksheet
|
||||
wks1.write(row, 0, solution)
|
||||
wks1.write(row, 1, coefficients[0])
|
||||
wks1.write(row, 2, coefficients[1])
|
||||
|
||||
row += 1
|
||||
df_device_results = pd.merge(df_device_precicion_results, df_device_accuracy_results, on='Device', how='outer', suffixes=('_precision', '_accuracy'))
|
||||
df_device_results['Result'] = np.where((df_device_results['Result_precision'] == 'Fail') | (df_device_results['Result_accuracy'] == 'Fail'), 'Fail', 'Pass')
|
||||
|
||||
# writer.save()
|
||||
|
||||
ax.set_title('Mean Absorbance for Each Solution')
|
||||
ax.set_xlabel('Concentration')
|
||||
ax.set_ylabel('Mean Absorbance')
|
||||
ax.legend()
|
||||
# plt.show()
|
||||
output_filename = f'data/accuracy_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx'
|
||||
writer = pd.ExcelWriter(output_filename, engine = 'xlsxwriter')
|
||||
df_device_results.to_excel(writer, sheet_name="device_results")
|
||||
df_precision_acc.to_excel(writer, sheet_name="concentration")
|
||||
df.to_excel(writer, sheet_name="in")
|
||||
df_reference_device.to_excel(writer, sheet_name="reference_device")
|
||||
|
||||
imgdata=io.BytesIO()
|
||||
fig.savefig(imgdata, format='png')
|
||||
wks1.insert_image(2,2, '', {'image_data': imgdata})
|
||||
workbook = writer.book
|
||||
worksheet_device_results = writer.sheets['device_results']
|
||||
|
||||
### debug each row
|
||||
# # Merge df and df_device_precicion_results on deviceId
|
||||
# df_merged = pd.merge(df, df_device_precicion_results, left_on='deviceId', right_on='Device', how='left')
|
||||
red_format = workbook.add_format({'bg_color': '#FFC7CE', 'font_color': '#9C0006'})
|
||||
green_format = workbook.add_format({'bg_color': '#C6EFCE', 'font_color': '#006100'})
|
||||
|
||||
# # Drop the duplicate "Device" column
|
||||
# df_merged.drop(columns=['Device'], inplace=True)
|
||||
worksheet_device_results.conditional_format('E2:E{}'.format(df_device_results.shape[0] + 1), {'type': 'text',
|
||||
'criteria': 'containing',
|
||||
'value': 'Fail',
|
||||
'format': red_format})
|
||||
|
||||
# # Save the merged DataFrame to the Excel file
|
||||
# df_merged.to_excel(writer, sheet_name="merged_results")
|
||||
worksheet_device_results.conditional_format('E2:E{}'.format(df_device_results.shape[0] + 1), {'type': 'text',
|
||||
'criteria': 'containing',
|
||||
'value': 'Pass',
|
||||
'format': green_format})
|
||||
|
||||
writer.close()
|
||||
|
||||
wks1 = workbook.add_worksheet('abs_plot')
|
||||
wks1.write(0,0,'Abs Plot')
|
||||
|
||||
# Write the header
|
||||
wks1.write(0, 0, 'Solution')
|
||||
wks1.write(0, 1, 'Slope')
|
||||
wks1.write(0, 2, 'Intercept')
|
||||
row = 1
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
|
||||
filtered_df = df[df['solution'].isin(['KMnO4', 'Tartrazine'])]
|
||||
|
||||
for solution in filtered_df['solution'].unique():
|
||||
if pd.notna(solution):
|
||||
solution_data = df[df['solution'] == solution]
|
||||
|
||||
concentration_means = solution_data.groupby('concentration')['absorbance'].mean()
|
||||
|
||||
# Fit a linear regression model
|
||||
coefficients = np.polyfit(concentration_means.index, concentration_means.values, 1)
|
||||
|
||||
print(f'Coefficients for {solution}: Slope={coefficients[0]}, Intercept={coefficients[1]}')
|
||||
|
||||
ax.plot(concentration_means.index, concentration_means.values, marker='o', linestyle='-', label=f'Mean Absorbance for {solution}')
|
||||
|
||||
ax.plot(concentration_means.index, np.polyval(coefficients, concentration_means.index), linestyle='--', label=f'Linear Fit for {solution}')
|
||||
|
||||
annotation_text = f'Slope: {coefficients[0]:.4f}\nIntercept: {coefficients[1]:.4f}'
|
||||
ax.text(concentration_means.index[-1] + 5, np.polyval(coefficients, concentration_means.index[-1]), annotation_text, fontsize=10, verticalalignment='center')
|
||||
|
||||
wks1.write(row, 0, solution)
|
||||
wks1.write(row, 1, coefficients[0])
|
||||
wks1.write(row, 2, coefficients[1])
|
||||
|
||||
row += 1
|
||||
|
||||
ax.set_title('Mean Absorbance for Each Solution')
|
||||
ax.set_xlabel('Concentration')
|
||||
ax.set_ylabel('Mean Absorbance')
|
||||
ax.legend()
|
||||
# plt.show()
|
||||
|
||||
imgdata=io.BytesIO()
|
||||
fig.savefig(imgdata, format='png')
|
||||
wks1.insert_image(2,2, '', {'image_data': imgdata})
|
||||
|
||||
### debug each row
|
||||
# # Merge df and df_device_precicion_results on deviceId
|
||||
# df_merged = pd.merge(df, df_device_precicion_results, left_on='deviceId', right_on='Device', how='left')
|
||||
|
||||
# # Drop the duplicate "Device" column
|
||||
# df_merged.drop(columns=['Device'], inplace=True)
|
||||
|
||||
# # Save the merged DataFrame to the Excel file
|
||||
# df_merged.to_excel(writer, sheet_name="merged_results")
|
||||
|
||||
writer.close()
|
||||
|
||||
else:
|
||||
# If df is empty, create an empty Excel file
|
||||
empty_output_filename = f'data/empty_excel_{datetime.today().strftime("%d_%m_%Y_%H_%M")}.xlsx'
|
||||
pd.DataFrame().to_excel(empty_output_filename, engine='xlsxwriter', index=False)
|
||||
Reference in New Issue
Block a user