From 7c95135c33e30f91a3dadaeafef4792504951543 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 31 Mar 2022 11:04:20 +0200 Subject: [PATCH 1/6] Initial commit to XANES-module --- beamtime/xanes/io.py | 150 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index a818d86..2ed6d29 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -1,2 +1,148 @@ -#hello -#yeah \ No newline at end of file +import pandas as pd +import matplotlib.pyplot as plt +import os + + +def split_xanes_scan(root, destination=None, replace=False): + #root is the path to the beamtime-folder + #destination should be the path to the processed data + + #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw + + with open(filename, 'r') as f: + lines = f.readlines() + + datas = [] + data = [] + headers = [] + header = '' + start = False + + for line in lines: + if line[0:2] == "#L": + start = True + header = line[2:].split() + continue + + elif line[0:2] == "#C": + start = False + + if data: + datas.append(data) + data = [] + + if header: + headers.append(header) + header = '' + + + + if start == False: + continue + + else: + data.append(line.split()) + + + + + edges = {'Mn': [6.0, 6.1, 6.2, 6.3, 6.4, 6.5], 'Fe': [6.8, 6.9, 7.0, 7.1, 7.2], 'Co': [7.6, 7.7, 7.8, 7.9], 'Ni': [8.1, 8.2, 8.3, 8.4, 8.5]} + edge_count = {'Mn': 0, 'Fe': 0, 'Co': 0, 'Ni': 0} + + + for ind, data in enumerate(datas): + df = pd.DataFrame(data) + df.columns = headers[ind] + + edge_start = np.round((float(df["ZapEnergy"].min())), 1) + + for edge, energies in edges.items(): + if edge_start in energies: + edge_actual = edge + edge_count[edge] += 1 + + + + filename = filename.split('/')[-1] + count = str(edge_count[edge_actual]).zfill(4) + + + # Save + if destination: + cwd = os.getcwd() + + if not os.path.isdir(destination): + os.mkdir(destination) + + os.chdir(destination) + + df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + + os.chdir(cwd) + + else: + df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + + +#Function that "collects" all the files in a folder, only accepting .dat-files from xanes-measurements +def get_filenames(path): + + + cwd = os.getcwd() + + # Change into path provided + os.chdir(path) + + filenames = [os.path.join(path, filename) for filename in os.listdir() if os.path.isfile(filename) and filename[-4:] == '.dat'] #changed + + + + # Change directory back to where you ran the script from + os.chdir(cwd) + + return filenames + +def put_in_dataframe(path): + filenames = get_filenames(path) + + #making the column names to be used in the dataframe, making sure the first column is the ZapEnergy + column_names = ["ZapEnergy"] + + for i in range(len(filenames)): + column_names.append(filenames[i]) + + #Taking the first file in the folder and extracting ZapEnergies and intensity from that (only need the intensity from the rest) + first = pd.read_csv(filenames[0], skiprows=0) + + #Making a data frame with the correct columns, and will fill inn data afterwards + df = pd.DataFrame(columns = column_names) + #First putting in the 2theta-values + df["ZapEnergy"]=first["ZapEnergy"] + + #filling in the intensities from all files into the corresponding column in the dataframe + for i in range(len(filenames)): + df2 = pd.read_csv(filenames[i]) + df2 = df2.drop(['Mon','Det1','Det2','Det3','Det4','Det5', 'Det6','Ion1'], axis=1) #, axis=1) + df2 = df2.drop(['MonEx','Ion2','Htime','MusstEnc1','MusstEnc3','MusstEnc4', 'TwoTheta', 'ZCryo'], axis=1) + df2 = df2.drop(['ZBlower1', 'ZBlower2', 'ZSrcur'], axis=1)#, axis=19) #removing the sigma at this point + + ############## THIS PART PICKS OUT WHICH ROI IS OF INTEREST, BUT MUST BE FIXED IF LOOKING AT THREE EDGES (roi00,roi01,roi02) ##################### + if 'xmap_roi01' in df2.columns: + #Trying to pick the roi with the highest difference between maximum and minimum intensity --> biggest edge shift + if max(df2["xmap_roi00"])-min(df2["xmap_roi00"])>max(df2["xmap_roi01"])-min(df2["xmap_roi01"]): + df[filenames[i]]=df2["xmap_roi00"] #forMn + else: + df[filenames[i]]=df2["xmap_roi01"] #forNi + else: + df[filenames[i]]=df2["xmap_roi00"] + ############################################################################################### + + i=i+1 + + + #print(df) + #If I want to make a csv-file of the raw data. Decided that was not necessary: + #df.to_csv('static-Mn-edge.csv') #writing it to a csv, first row is datapoint (index), second column is 2theta, and from there the scans starts + + + return df \ No newline at end of file From b0130d49b88a4faa725fff54269e1239c089d2e1 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 31 Mar 2022 17:05:32 +0200 Subject: [PATCH 2/6] Working on the calibration of the XANES-data, subtracting background and defining post-edge --- beamtime/xanes/calib.py | 192 ++++++++++++++++++++++++---------------- 1 file changed, 118 insertions(+), 74 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 2ef6c4d..3c54dd1 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -1,6 +1,8 @@ import pandas as pd import numpy as np import os +import matplotlib.pyplot as plt +import beamtime.auxillary as aux def rbkerbest(): print("ROSENBORG!<3") @@ -13,82 +15,124 @@ def rbkerbest(): ##Better to make a new function that loops through the files, and performing the split_xanes_scan on -def split_xanes_scan(root, destination=None, replace=False): - #root is the path to the beamtime-folder - #destination should be the path to the processed data +def pre_edge_subtraction(df,filenames, options={}): + + required_options = ['edge', 'print'] + default_options = { + 'edge' : 'Mn', + 'print': False + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge + if str(options['edge']) == 'Mn': + edge_start = 6.45 + if str(options['edge']) == 'Ni': + edge_start = 8.3 + + + #making a function to check the difference between values in the list and the defined start of the edge (where background regression will stop): + absolute_difference_function = lambda list_value : abs(list_value - edge_start) + + #finding the energy data point value that is closest to what I defined as the end of the background + edge_start_value = min(df["ZapEnergy"], key=absolute_difference_function) + + #Finding what the index of the edge shift end point is + start_index=df[df["ZapEnergy"]==edge_start_value].index.values[0] + + #Defining x-range for linear background fit, ending at the edge start index + df_start=df[0:start_index] - #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw + #Making a new dataframe, with only the ZapEnergies as the first column + df_background = pd.DataFrame(df["ZapEnergy"]) + + for files in filenames: + + #Fitting linear function to the pre-edge + d = np.polyfit(df_start["ZapEnergy"],df_start[files],1) + function_pre = np.poly1d(d) + + #making a list, y_pre,so the background will be applied to all ZapEnergy-values + y_pre=function_pre(df["ZapEnergy"]) + + #adding a new column in df_background with the y-values of the background + df_background.insert(1,files,y_pre) - with open(filename, 'r') as f: - lines = f.readlines() - - datas = [] - data = [] - headers = [] - header = '' - start = False + #Plotting the calculated pre-edge background with the region used for the regression - for line in lines: - if line[0:2] == "#L": - start = True - header = line[2:].split() - continue - - elif line[0:2] == "#C": - start = False - - if data: - datas.append(data) - data = [] - - if header: - headers.append(header) - header = '' - - + ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILES IS CORRUPT + #ax = df.plot(x = "ZapEnergy",y=files) + + if options['print'] == True: + #Plotting an example of the edge_start region and the fitted background that will later be subtracted + ax = df.plot(x = "ZapEnergy",y=filenames[0]) #defining x and y + plt.axvline(x = edge_start_value) + fig = plt.figure(figsize=(15,15)) + df_background.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) + + ###################### Subtracting the pre edge from xmap_roi00 ################ + #making a new dataframe to insert the background subtracted intensities + df_new = pd.DataFrame(df["ZapEnergy"]) + #inserting the pre_edge-background subtracted original xmap_roi00 data - if start == False: - continue - - else: - data.append(line.split()) - - - - - edges = {'Mn': [6.0, 6.1, 6.2, 6.3, 6.4, 6.5], 'Fe': [6.8, 6.9, 7.0, 7.1, 7.2], 'Co': [7.6, 7.7, 7.8, 7.9], 'Ni': [8.1, 8.2, 8.3, 8.4, 8.5]} - edge_count = {'Mn': 0, 'Fe': 0, 'Co': 0, 'Ni': 0} + for files in filenames: + newintensity_calc=df[files]-df_background[files] + df_new.insert(1,files,newintensity_calc) + + if options['print'] == True: + #Plotting original data (black) and background subtracted data (red) + ax = df.plot(x = "ZapEnergy",y=filenames[0], color="Black") + plt.axvline(x = edge_start_value) + fig = plt.figure(figsize=(15,15)) + df_new.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) + return df_new + +def post_edge_normalization(df,df_new,filenames, options={}): + + required_options = ['edge', 'print'] + default_options = { + 'edge' : 'Mn', + 'print': False + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge + if str(options['edge']) == 'Mn': + edge_stop = 6.565 + if str(options['edge']) == 'Ni': + edge_stop = 8.361 + + absolute_difference_function = lambda list_value : abs(list_value - edge_stop) + edge_stop_value = min(df_new["ZapEnergy"], key=absolute_difference_function) + end_index=df_new[df_new["ZapEnergy"]==edge_stop_value].index.values[0] + #Defining x-range for linear fit + df_fix=df_new + df_fix.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + df_end=df_fix[end_index:] #The region of interest for the post edge + #print(df_end) + #Fitting linear function to the pre-edge using the background corrected intensities to make the post edge fit + df_postedge = pd.DataFrame(df["ZapEnergy"]) + + function_post_list=[] + for files in filenames: + d = np.polyfit(df_end["ZapEnergy"],df_end[files],1) + function_post = np.poly1d(d) + y_post=function_post(df["ZapEnergy"]) + function_post_list.append(function_post) + df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge + + #print(filenames[0]) + #print(df_postedge) + #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line + if options['print'] == True: + ax = df_new.plot(x = "ZapEnergy",y=filenames) #defining x and y + plt.axvline(x = edge_stop_value) + fig = plt.figure(figsize=(15,15)) + df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) + #print(function_post_list) + #print(function_post) + ax = df_new.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y + df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) + plt.axvline(x = edge_stop_value) - - for ind, data in enumerate(datas): - df = pd.DataFrame(data) - df.columns = headers[ind] - - edge_start = np.round((float(df["ZapEnergy"].min())), 1) - - for edge, energies in edges.items(): - if edge_start in energies: - edge_actual = edge - edge_count[edge] += 1 - - - - filename = filename.split('/')[-1] - count = str(edge_count[edge_actual]).zfill(4) - - - # Save - if destination: - cwd = os.getcwd() - - if not os.path.isdir(destination): - os.mkdir(destination) - - os.chdir(destination) - - df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) - - os.chdir(cwd) - - else: - df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) \ No newline at end of file + \ No newline at end of file From 872ae759b29a45ece7cf070a6deac869a3c6fe3a Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 6 Apr 2022 21:10:40 +0200 Subject: [PATCH 3/6] Optimizing code and splitting into smaller functions --- beamtime/xanes/calib.py | 125 +++++++++++++++++++++------------------- beamtime/xanes/io.py | 2 +- 2 files changed, 68 insertions(+), 59 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 3c54dd1..8cdbdf7 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -3,7 +3,8 @@ import numpy as np import os import matplotlib.pyplot as plt import beamtime.auxillary as aux - +import beamtime.xanes as xas +import beamtime.xanes.io as io def rbkerbest(): print("ROSENBORG!<3") @@ -14,99 +15,108 @@ def rbkerbest(): ##Better to make a new function that loops through the files, and performing the split_xanes_scan on +#Tryiung to make a function that can decide which edge it is based on the first ZapEnergy-value +def finding_edge(df): + if 5.9 < df["ZapEnergy"][0] < 6.5: + edge='Mn' + return(edge) + if 8.0 < df["ZapEnergy"][0] < 8.6: + edge='Ni' + return(edge) -def pre_edge_subtraction(df,filenames, options={}): +#def pre_edge_subtraction(df,filenames, options={}): +def test(innmat): + df_test= xas.io.put_in_dataframe(innmat) + print(df_test) - required_options = ['edge', 'print'] +def pre_edge_subtraction(path, options={}): + required_options = ['print','troubleshoot'] default_options = { - 'edge' : 'Mn', - 'print': False + 'print': False, + 'troubleshoot': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge - if str(options['edge']) == 'Mn': + filenames = xas.io.get_filenames(path) + df= xas.io.put_in_dataframe(path) + edge=finding_edge(df) + + #Defining the end of the region used to define the background, thus start of the edge + #implement widget + if edge == 'Mn': edge_start = 6.45 - if str(options['edge']) == 'Ni': + if edge == 'Ni': edge_start = 8.3 - - #making a function to check the difference between values in the list and the defined start of the edge (where background regression will stop): - absolute_difference_function = lambda list_value : abs(list_value - edge_start) - - #finding the energy data point value that is closest to what I defined as the end of the background - edge_start_value = min(df["ZapEnergy"], key=absolute_difference_function) - - #Finding what the index of the edge shift end point is - start_index=df[df["ZapEnergy"]==edge_start_value].index.values[0] - - #Defining x-range for linear background fit, ending at the edge start index - df_start=df[0:start_index] - - #Making a new dataframe, with only the ZapEnergies as the first column - df_background = pd.DataFrame(df["ZapEnergy"]) + #making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) + df_start=df.loc[df["ZapEnergy"] < edge_start] + + #Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data + df_bkgd = pd.DataFrame(df["ZapEnergy"]) for files in filenames: - #Fitting linear function to the pre-edge + #Fitting linear function to the background d = np.polyfit(df_start["ZapEnergy"],df_start[files],1) - function_pre = np.poly1d(d) + function_bkgd = np.poly1d(d) #making a list, y_pre,so the background will be applied to all ZapEnergy-values - y_pre=function_pre(df["ZapEnergy"]) + y_bkgd=function_bkgd(df["ZapEnergy"]) #adding a new column in df_background with the y-values of the background - df_background.insert(1,files,y_pre) + df_bkgd.insert(1,files,y_bkgd) - #Plotting the calculated pre-edge background with the region used for the regression - - ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILES IS CORRUPT - #ax = df.plot(x = "ZapEnergy",y=files) + if options['troubleshoot'] == True: + ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILE IS CORRUPT + ax = df.plot(x = "ZapEnergy",y=files) + #Plotting the calculated pre-edge background with the region used for the regression if options['print'] == True: #Plotting an example of the edge_start region and the fitted background that will later be subtracted - ax = df.plot(x = "ZapEnergy",y=filenames[0]) #defining x and y - plt.axvline(x = edge_start_value) - fig = plt.figure(figsize=(15,15)) - df_background.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) - + fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) + df.plot(x = "ZapEnergy",y=filenames[0],ax=ax1) #defining x and y + plt.axvline(x = max(df_start["ZapEnergy"])) + #fig = plt.figure(figsize=(15,15)) + df_bkgd.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax1) + ax1.set_title('Data and fitted background') ###################### Subtracting the pre edge from xmap_roi00 ################ #making a new dataframe to insert the background subtracted intensities - df_new = pd.DataFrame(df["ZapEnergy"]) + df_bkgd_sub = pd.DataFrame(df["ZapEnergy"]) #inserting the pre_edge-background subtracted original xmap_roi00 data for files in filenames: - newintensity_calc=df[files]-df_background[files] - df_new.insert(1,files,newintensity_calc) + newintensity_calc=df[files]-df_bkgd[files] + df_bkgd_sub.insert(1,files,newintensity_calc) if options['print'] == True: - #Plotting original data (black) and background subtracted data (red) - ax = df.plot(x = "ZapEnergy",y=filenames[0], color="Black") - plt.axvline(x = edge_start_value) - fig = plt.figure(figsize=(15,15)) - df_new.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) - return df_new + df.plot(x = "ZapEnergy",y=filenames[0], color="Black", ax=ax2, legend=False) + plt.axvline(x = max(df_start["ZapEnergy"])) + df_bkgd_sub.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax2, legend=False) + ax2.set_title('Data and background-subtracted data') -def post_edge_normalization(df,df_new,filenames, options={}): + return df_bkgd_sub - required_options = ['edge', 'print'] +def post_edge_normalization(df,df_backg_sub,filenames, options={}): + + required_options = ['print'] default_options = { - 'edge' : 'Mn', 'print': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - + + edge=finding_edge(df) #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge - if str(options['edge']) == 'Mn': + #Implement widget + if edge == 'Mn': edge_stop = 6.565 - if str(options['edge']) == 'Ni': + if edge == 'Ni': edge_stop = 8.361 absolute_difference_function = lambda list_value : abs(list_value - edge_stop) - edge_stop_value = min(df_new["ZapEnergy"], key=absolute_difference_function) - end_index=df_new[df_new["ZapEnergy"]==edge_stop_value].index.values[0] + edge_stop_value = min(df_backg_sub["ZapEnergy"], key=absolute_difference_function) + end_index=df_backg_sub[df_backg_sub["ZapEnergy"]==edge_stop_value].index.values[0] #Defining x-range for linear fit - df_fix=df_new + df_fix=df_backg_sub df_fix.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit df_end=df_fix[end_index:] #The region of interest for the post edge #print(df_end) @@ -125,14 +135,13 @@ def post_edge_normalization(df,df_new,filenames, options={}): #print(df_postedge) #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line if options['print'] == True: - ax = df_new.plot(x = "ZapEnergy",y=filenames) #defining x and y + ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y plt.axvline(x = edge_stop_value) fig = plt.figure(figsize=(15,15)) df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) #print(function_post_list) #print(function_post) - ax = df_new.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y + ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) plt.axvline(x = edge_stop_value) - - \ No newline at end of file + \ No newline at end of file diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index 2ed6d29..527f300 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -1,7 +1,7 @@ import pandas as pd import matplotlib.pyplot as plt import os - +import numpy as np def split_xanes_scan(root, destination=None, replace=False): #root is the path to the beamtime-folder From 135d577b4b78c3e6985bc99c765cf1e89a272456 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Apr 2022 11:34:44 +0200 Subject: [PATCH 4/6] Adjusting post edge processing --- beamtime/xanes/calib.py | 68 ++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 8cdbdf7..9e3cbdb 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -44,7 +44,7 @@ def pre_edge_subtraction(path, options={}): #Defining the end of the region used to define the background, thus start of the edge #implement widget if edge == 'Mn': - edge_start = 6.45 + edge_start = 6.42 if edge == 'Ni': edge_start = 8.3 @@ -73,30 +73,44 @@ def pre_edge_subtraction(path, options={}): #Plotting the calculated pre-edge background with the region used for the regression if options['print'] == True: #Plotting an example of the edge_start region and the fitted background that will later be subtracted - fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) - df.plot(x = "ZapEnergy",y=filenames[0],ax=ax1) #defining x and y + fig, (ax1,ax2,ax3) = plt.subplots(1,3,figsize=(15,5)) + df.plot(x="ZapEnergy", y=filenames,color="Black",ax=ax1) + df_bkgd.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax1) plt.axvline(x = max(df_start["ZapEnergy"])) #fig = plt.figure(figsize=(15,15)) - df_bkgd.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax1) + df_bkgd.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax2) ax1.set_title('Data and fitted background') + #Zooming into bacground region to confirm fit and limits looks reasonable + df.plot(x = "ZapEnergy",y=filenames,ax=ax2) #defining x and y) + ax2.set_xlim([min(df_start["ZapEnergy"]),max(df_start["ZapEnergy"])+0.01]) + #finding maximum and minimum values in the backgrounds + min_values=[] + max_values=[] + for file in filenames: + min_values.append(min(df_start[file])) + max_values.append(max(df_start[file])) + ax2.set_ylim([min(min_values),max(max_values)]) + plt.axvline(x = max(df_start["ZapEnergy"])) + #ax2.set_xlim([25, 50]) ###################### Subtracting the pre edge from xmap_roi00 ################ + #making a new dataframe to insert the background subtracted intensities df_bkgd_sub = pd.DataFrame(df["ZapEnergy"]) - #inserting the pre_edge-background subtracted original xmap_roi00 data + #inserting the background subtracted original xmap_roi00 data for files in filenames: newintensity_calc=df[files]-df_bkgd[files] df_bkgd_sub.insert(1,files,newintensity_calc) if options['print'] == True: - df.plot(x = "ZapEnergy",y=filenames[0], color="Black", ax=ax2, legend=False) - plt.axvline(x = max(df_start["ZapEnergy"])) - df_bkgd_sub.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax2, legend=False) - ax2.set_title('Data and background-subtracted data') + df.plot(x = "ZapEnergy",y=filenames, color="Black", ax=ax3, legend=False) + #plt.axvline(x = max(df_start["ZapEnergy"])) + df_bkgd_sub.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax3, legend=False) + ax3.set_title('Data and background-subtracted data') - return df_bkgd_sub + return df_bkgd_sub,filenames,edge -def post_edge_normalization(df,df_backg_sub,filenames, options={}): +def post_edge_normalization(path, options={}): required_options = ['print'] default_options = { @@ -104,7 +118,7 @@ def post_edge_normalization(df,df_backg_sub,filenames, options={}): } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - edge=finding_edge(df) + df_bkgd_sub,filenames,edge = pre_edge_subtraction(path) #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge #Implement widget if edge == 'Mn': @@ -112,22 +126,26 @@ def post_edge_normalization(df,df_backg_sub,filenames, options={}): if edge == 'Ni': edge_stop = 8.361 - absolute_difference_function = lambda list_value : abs(list_value - edge_stop) - edge_stop_value = min(df_backg_sub["ZapEnergy"], key=absolute_difference_function) - end_index=df_backg_sub[df_backg_sub["ZapEnergy"]==edge_stop_value].index.values[0] +#============================================================= + #absolute_difference_function = lambda list_value : abs(list_value - edge_stop) + #edge_stop_value = min(df_bkgd_sub["ZapEnergy"], key=absolute_difference_function) + #end_index=df_bkgd_sub[df_bkgd_sub["ZapEnergy"]==edge_stop_value].index.values[0] #Defining x-range for linear fit - df_fix=df_backg_sub - df_fix.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit - df_end=df_fix[end_index:] #The region of interest for the post edge - #print(df_end) + #df_fix=df_bkgd_sub + #df_fix.dropna(inplace=True) + #df_end=df_fix[end_index:] #The region of interest for the post edge + #Fitting linear function to the pre-edge using the background corrected intensities to make the post edge fit - df_postedge = pd.DataFrame(df["ZapEnergy"]) + #=============================================================== + df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge -> to be fitted + df_end.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + df_postedge = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) #making a new dataframe function_post_list=[] for files in filenames: d = np.polyfit(df_end["ZapEnergy"],df_end[files],1) function_post = np.poly1d(d) - y_post=function_post(df["ZapEnergy"]) + y_post=function_post(df_bkgd_sub["ZapEnergy"]) function_post_list.append(function_post) df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge @@ -135,13 +153,13 @@ def post_edge_normalization(df,df_backg_sub,filenames, options={}): #print(df_postedge) #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line if options['print'] == True: - ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y - plt.axvline(x = edge_stop_value) + ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y + plt.axvline(x = min(df_end["ZapEnergy"])) fig = plt.figure(figsize=(15,15)) df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) #print(function_post_list) #print(function_post) - ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y + ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) - plt.axvline(x = edge_stop_value) + plt.axvline(x = min(df_end["ZapEnergy"])) \ No newline at end of file From baa253ab3e9b369960576040668b69a0792e2386 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Apr 2022 12:34:49 +0200 Subject: [PATCH 5/6] tester fra vscode --- test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..35d92ec --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +hei på dej \ No newline at end of file From 7962b3fc5cd67c1e47e04bb2063bac60cd4b3522 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 8 Apr 2022 13:28:47 +0200 Subject: [PATCH 6/6] Fixing functions --- beamtime/xanes/calib.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 9e3cbdb..7a0ecb5 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -126,18 +126,7 @@ def post_edge_normalization(path, options={}): if edge == 'Ni': edge_stop = 8.361 -#============================================================= - #absolute_difference_function = lambda list_value : abs(list_value - edge_stop) - #edge_stop_value = min(df_bkgd_sub["ZapEnergy"], key=absolute_difference_function) - #end_index=df_bkgd_sub[df_bkgd_sub["ZapEnergy"]==edge_stop_value].index.values[0] - #Defining x-range for linear fit - #df_fix=df_bkgd_sub - #df_fix.dropna(inplace=True) - #df_end=df_fix[end_index:] #The region of interest for the post edge - - #Fitting linear function to the pre-edge using the background corrected intensities to make the post edge fit - #=============================================================== - df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge -> to be fitted + df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge, where a regression line will be calculated in the for-loop below df_end.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit df_postedge = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) #making a new dataframe @@ -149,17 +138,14 @@ def post_edge_normalization(path, options={}): function_post_list.append(function_post) df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge - #print(filenames[0]) - #print(df_postedge) #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line if options['print'] == True: ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y plt.axvline(x = min(df_end["ZapEnergy"])) fig = plt.figure(figsize=(15,15)) df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) - #print(function_post_list) - #print(function_post) ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) plt.axvline(x = min(df_end["ZapEnergy"])) - \ No newline at end of file + + return df_bkgd_sub, df_postedge \ No newline at end of file