rasmus/nafuma
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tweaks based on workflow testing

Browse source
This commit is contained in:
rasmusvt 2022-06-29 15:26:43 +02:00
parent c522b73ca4
commit 6bbd6776b8
2 changed files with 214 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

14
nafuma/auxillary.py
View file

@ -12,11 +12,21 @@ def update_options(options, required_options, default_options):
return options return options
def save_options(options, path): def save_options(options, path, ignore=None):
''' Saves any options dictionary to a JSON-file in the specified path''' ''' Saves any options dictionary to a JSON-file in the specified path'''
options_copy = options.copy()
if ignore:
if not isinstance(ignore, list):
ignore = [ignore]
for i in ignore:
options_copy[i] = 'Removed'
with open(path, 'w') as f: with open(path, 'w') as f:
json.dump(options,f) json.dump(options_copy,f, skipkeys=True, indent=4)
def load_options(path): def load_options(path):

200
nafuma/xanes/calib.py
View file

@ -46,12 +46,12 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame:
# FIXME Add log-file # FIXME Add log-file
required_options = ['pre_edge_limit', 'masks', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive'] required_options = ['pre_edge_limits', 'pre_edge_masks', 'pre_edge_polyorder', 'pre_edge_store_data', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive']
default_options = { default_options = {
'pre_edge_limits': [None, None], 'pre_edge_limits': [None, None],
'pre_edge_masks': [], 'pre_edge_masks': [],
'pre_edge_polyorder': 1, 'pre_edge_polyorder': 1,
'pre_edge_save_data': False, 'pre_edge_store_data': False,
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log',
'show_plots': False, 'show_plots': False,
@ -159,7 +159,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame:
if options['log']: if options['log']:
aux.write_log(message=f'Pre edge fitting done.', options=options) aux.write_log(message=f'Pre edge fitting done.', options=options)
if options['pre_edge_save_data']: if options['pre_edge_store_data']:
data['pre_edge_fit_data'] = pre_edge_fit_data data['pre_edge_fit_data'] = pre_edge_fit_data
return pre_edge_fit_data return pre_edge_fit_data
@ -171,7 +171,8 @@ def pre_edge_fit_interactive(data: dict, options: dict) -> None:
w = widgets.interactive( w = widgets.interactive(
btp.ipywidgets_update, func=widgets.fixed(pre_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), btp.ipywidgets_update, func=widgets.fixed(pre_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options),
pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001),
pre_edge_store_data=widgets.Checkbox(value=options['pre_edge_store_data'])
) )
options['widget'] = w options['widget'] = w
@ -183,12 +184,14 @@ def pre_edge_fit_interactive(data: dict, options: dict) -> None:
def pre_edge_subtraction(data: dict, options={}): def pre_edge_subtraction(data: dict, options={}):
required_options = ['log', 'logfile', 'save_plots', 'save_folder'] required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'pre_edge_subtraction_store_data']
default_options = { default_options = {
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_subtraction.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_subtraction.log',
'show_plots': False,
'save_plots': False, 'save_plots': False,
'save_folder': './' 'save_folder': './',
'pre_edge_subtraction_store_data': False
} }
options = aux.update_options(options=options, required_options=required_options, default_options=default_options) options = aux.update_options(options=options, required_options=required_options, default_options=default_options)
@ -203,20 +206,29 @@ def pre_edge_subtraction(data: dict, options={}):
xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data_original'][filename] - data['pre_edge_fit_data'][filename]) xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data_original'][filename] - data['pre_edge_fit_data'][filename])
if options['save_plots'] or options['show_plots']:
fig, ax = plt.subplots(figsize=(10,5))
data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax, label='Original data')
xanes_data_bkgd_subtracted.plot(x='ZapEnergy', y=filename, color='red', ax=ax, label='Pre edge subtracted')
ax.set_title(f'{os.path.basename(filename)} - After subtraction', size=20)
if options['save_plots']: if options['save_plots']:
if not os.path.isdir(options['save_folder']): if not os.path.isdir(options['save_folder']):
os.makedirs(options['save_folder']) os.makedirs(options['save_folder'])
dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_subtraction.png' dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_subtraction.png'
fig, ax = plt.subplots(figsize=(10,5))
data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax)
xanes_data_bkgd_subtracted.plot(x='ZapEnergy', y=filename, color='red', ax=ax)
ax.set_title(f'{os.path.basename(filename)} - After subtraction', size=20)
plt.savefig(dst) plt.savefig(dst)
if not options['show_plots']:
plt.close() plt.close()
if options['pre_edge_subtraction_store_data']:
data['xanes_data'] = xanes_data_bkgd_subtracted
return xanes_data_bkgd_subtracted return xanes_data_bkgd_subtracted
@ -230,14 +242,14 @@ def post_edge_fit(data: dict, options={}):
''' '''
required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'interactive', 'show_plots', 'save_plots', 'save_folder'] required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'post_edge_store_data', 'interactive', 'show_plots', 'save_plots', 'save_folder']
default_options = { default_options = {
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log',
'post_edge_limits': [None, None], 'post_edge_limits': [None, None],
'post_edge_masks': [], 'post_edge_masks': [],
'post_edge_polyorder': 2, 'post_edge_polyorder': 2,
'post_edge_save_data': False, 'post_edge_store_data': False,
'interactive': False, 'interactive': False,
'show_plots': False, 'show_plots': False,
'save_plots': False, 'save_plots': False,
@ -283,10 +295,10 @@ def post_edge_fit(data: dict, options={}):
for i, filename in enumerate(data['path']): for i, filename in enumerate(data['path']):
if options['log']: if options['log']:
aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["polyorder"]}', options=options) aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["post_edge_polyorder"]}', options=options)
#Fitting linear function to the background #Fitting linear function to the background
params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], options['polyorder']) params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], options['post_edge_polyorder'])
fit_function = np.poly1d(params) fit_function = np.poly1d(params)
if options['log']: if options['log']:
@ -340,7 +352,7 @@ def post_edge_fit(data: dict, options={}):
if options['log']: if options['log']:
aux.write_log(message='Post edge fitting done!', options=options) aux.write_log(message='Post edge fitting done!', options=options)
if options['post_edge_save_data']: if options['post_edge_store_data']:
data['post_edge_fit_data'] = post_edge_fit_data data['post_edge_fit_data'] = post_edge_fit_data
@ -352,7 +364,8 @@ def post_edge_fit_interactive(data: dict, options: dict) -> None:
w = widgets.interactive( w = widgets.interactive(
btp.ipywidgets_update, func=widgets.fixed(post_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), btp.ipywidgets_update, func=widgets.fixed(post_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options),
post_edge_limit=widgets.FloatRangeSlider(value=[options['post_edge.limits'][0], options['post_edge.limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) post_edge_limits=widgets.FloatRangeSlider(value=[options['post_edge_limits'][0], options['post_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001),
post_edge_store_data=widgets.Checkbox(value=options['post_edge_store_data'])
) )
options['widget'] = w options['widget'] = w
@ -364,16 +377,19 @@ def smoothing(data: dict, options={}):
# FIXME Add logging # FIXME Add logging
# FIXME Add saving of files # FIXME Add saving of files
required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'smooth_window_length', 'smooth_algorithm', 'smooth_polyorder', 'smooth_save_default'] required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'interactive', 'smooth_window_length', 'smooth_algorithm', 'smooth_polyorder', 'smooth_save_default', 'smooth_store_data']
default_options = { default_options = {
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log',
'show_plots': False,
'save_plots': False, 'save_plots': False,
'save_folder': './', 'save_folder': './',
'interactive': False,
'smooth_window_length': 3, 'smooth_window_length': 3,
'smooth_polyorder': 2, 'smooth_polyorder': 2,
'smooth_algorithm': 'savgol', # At the present, only Savitzky-Golay filter is implemented. Add Gaussian and Boxcar later. 'smooth_algorithm': 'savgol', # At the present, only Savitzky-Golay filter is implemented. Add Gaussian and Boxcar later.
'smooth_save_default': False, 'smooth_save_default': False,
'smooth_store_data': False,
} }
options = aux.update_options(options=options, required_options=required_options, default_options=default_options) options = aux.update_options(options=options, required_options=required_options, default_options=default_options)
@ -387,6 +403,7 @@ def smoothing(data: dict, options={}):
if options['interactive']: if options['interactive']:
data['xanes_data_backup'] = data['xanes_data']
options['interactive'] = False options['interactive'] = False
options['interactive_session_active'] = True options['interactive_session_active'] = True
options['show_plots'] = True options['show_plots'] = True
@ -405,7 +422,7 @@ def smoothing(data: dict, options={}):
df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['smooth_window_length'], options['smooth_polyorder'])) df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['smooth_window_length'], options['smooth_polyorder']))
if options['smooth_save_default']: if options['smooth_save_default']:
if options['smooth.algorithm'] == 'savgol': if options['smooth_algorithm'] == 'savgol':
if options['log']: if options['log']:
aux.write_log(message=f'Smoothing {filename} using default parameters with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options) aux.write_log(message=f'Smoothing {filename} using default parameters with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options)
df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['smooth_window_length'], default_options['smooth_polyorder'])) df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['smooth_window_length'], default_options['smooth_polyorder']))
@ -413,27 +430,36 @@ def smoothing(data: dict, options={}):
if options['save_plots'] or options['show_plots']: if options['save_plots'] or options['show_plots']:
edge_pos = estimate_edge_position(data=data, options=options) edge_pos = estimate_edge_position(data=data, options=options)
intensity_midpoint = df_smooth[filename].iloc[np.where(df_smooth['ZapEnergy'] == find_nearest(df_smooth['ZapEnergy'], edge_pos))].values[0] intensity_midpoint = df_smooth[filename].iloc[np.where(df_smooth['ZapEnergy'] == find_nearest(df_smooth['ZapEnergy'], edge_pos))].values[0]
step_length = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]
if options['smooth_save_default']: if options['smooth_save_default']:
fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5)) fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5))
data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter') data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter')
df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax1) df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax1)
ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20)
data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter') data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter')
df_smooth_default.loc[(df_smooth_default['ZapEnergy'] > edge_pos-0.0015) & (df_smooth_default['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) df_smooth_default.loc[(df_smooth_default['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth_default['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2)
ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20) ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20)
elif not options['save_default']: elif not options['smooth_save_default']:
fig, ax = plt.subplots(figsize=(20,10)) fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))
data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax, kind='scatter') data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter', c='black')
df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax) df_smooth.plot(x='ZapEnergy', y=filename, ax=ax1, c='red')
ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015])
ax.set_ylim([intensity_midpoint*0.9, intensity_midpoint*1.1])
ax.set_title(f'{os.path.basename(filename)} - Smooth', size=20) data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter')
df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2)
#ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015])
#ax.set_ylim([intensity_midpoint*0.9, intensity_midpoint*1.1])
ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20)
ax2.set_title(f'{os.path.basename(filename)} - Smooth Edge Region', size=20)
if options['save_plots']: if options['save_plots']:
if not os.path.isdir(options['save_folder']): if not os.path.isdir(options['save_folder']):
@ -445,9 +471,13 @@ def smoothing(data: dict, options={}):
if not options['show_plots']: if not options['show_plots']:
plt.close() plt.close()
if not options['save_default']: if not options['smooth_save_default']:
df_smooth_default = None df_smooth_default = None
if options['smooth_store_data']:
data['xanes_data'] = df_smooth
options['smooth_store_data'] = False
return df_smooth, df_smooth_default return df_smooth, df_smooth_default
@ -457,16 +487,21 @@ def smoothing_interactive(data: dict, options: dict) -> None:
w = widgets.interactive( w = widgets.interactive(
btp.ipywidgets_update, func=widgets.fixed(smoothing), data=widgets.fixed(data), options=widgets.fixed(options), btp.ipywidgets_update, func=widgets.fixed(smoothing), data=widgets.fixed(data), options=widgets.fixed(options),
smooth_window_length=widgets.IntSlider(value=options['smooth_window_length'], min=1, max=20, step=1), smooth_window_length=widgets.IntSlider(value=options['smooth_window_length'], min=3, max=21, step=2),
smooth_polyorder=widgets.IntSlider(value=options['smooth_polyorder'], min=1, max=5, step=1), smooth_polyorder=widgets.IntSlider(value=options['smooth_polyorder'], min=1, max=5, step=1),
smooth_store_data=widgets.Checkbox(value=options['smooth_store_data'])
) )
options['widget'] = w options['widget'] = w
display(w) display(w)
def restore_from_backup(data):
if 'xanes_data_bakcup' in data.keys():
data['xanes_data'] = data['xanes_data_backup']
def find_nearest(array, value): def find_nearest(array, value):
#function to find the value closes to "value" in an "array" #function to find the value closes to "value" in an "array"
array = np.asarray(array) array = np.asarray(array)
@ -480,7 +515,7 @@ def estimate_edge_position(data: dict, options={}, index=0):
default_options = { default_options = {
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log',
'periods': 2, #Periods needs to be an even number for the shifting of values to work properly 'periods': 6, #Periods needs to be an even number for the shifting of values to work properly
} }
options = aux.update_options(options=options, required_options=required_options, default_options=default_options) options = aux.update_options(options=options, required_options=required_options, default_options=default_options)
@ -509,7 +544,7 @@ def determine_edge_position(data: dict, options={}):
Requires that XANES-data is already loaded in data['xanes_data']. This allows the user to choose when to determine the edge position - whether before or after normalisation, flattening etc.''' Requires that XANES-data is already loaded in data['xanes_data']. This allows the user to choose when to determine the edge position - whether before or after normalisation, flattening etc.'''
required_options = ['save_values', 'log', 'logfile', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'fit_region'] required_options = ['save_values', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'points_around_edge']
default_options = { default_options = {
'save_values': True, # Whether the edge positions should be stored in a dictionary within the main data dictionary. 'save_values': True, # Whether the edge positions should be stored in a dictionary within the main data dictionary.
'log': False, # Toggles logging on/off 'log': False, # Toggles logging on/off
@ -521,9 +556,9 @@ def determine_edge_position(data: dict, options={}):
'diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the differential to 'diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the differential to
'diff.periods': 2, # Sets the number of data points between which the first order difference should be calculated. Needs to be even for subsequent shifting of data to function. 'diff.periods': 2, # Sets the number of data points between which the first order difference should be calculated. Needs to be even for subsequent shifting of data to function.
'double_diff': False, # Toggles calculation of the edge position based on double differential data 'double_diff': False, # Toggles calculation of the edge position based on double differential data
'double_diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the double differential to 'double_diff.polyorder': 1, # Sets the order of the polynomial to fit edge region of the double differential to
'double_diff.periods': 2, # Sets the number of data points between which the second order difference should be calculated. Needs to be even for subsequent shifting of data to function. 'double_diff.periods': 2, # Sets the number of data points between which the second order difference should be calculated. Needs to be even for subsequent shifting of data to function.
'fit_region': None # The length of the region to find points to fit to a function 'points_around_edge': 5 # The length of the region to find points to fit to a function
} }
options = aux.update_options(options=options, required_options=required_options, default_options=default_options) options = aux.update_options(options=options, required_options=required_options, default_options=default_options)
@ -539,6 +574,16 @@ def determine_edge_position(data: dict, options={}):
raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly")
if options['interactive']:
data['xanes_data_backup'] = data['xanes_data']
options['interactive'] = False
options['interactive_session_active'] = True
options['show_plots'] = True
determine_edge_position_interactive(data=data, options=options)
return
# Prepare dataframes for differential data # Prepare dataframes for differential data
if options['diff']: if options['diff']:
@ -554,8 +599,8 @@ def determine_edge_position(data: dict, options={}):
for i, filename in enumerate(data['path']): for i, filename in enumerate(data['path']):
estimated_edge_pos = estimate_edge_position(data, options=options, index=i) estimated_edge_pos = estimate_edge_position(data, options=options, index=i)
if not options['fit_region']:
options['fit_region'] = (5)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]) fit_region = (options['points_around_edge']+1)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0])
#========================== Fitting the first order derivative ========== #========================== Fitting the first order derivative ==========
@ -565,7 +610,7 @@ def determine_edge_position(data: dict, options={}):
df_diff[filename]=df_diff[filename].shift(-int(options['diff.periods']/2)) # Shifts the data back so that the difference between the points is located in the middle of the two points the caluclated difference is between df_diff[filename]=df_diff[filename].shift(-int(options['diff.periods']/2)) # Shifts the data back so that the difference between the points is located in the middle of the two points the caluclated difference is between
# Picks out the points to be fitted # Picks out the points to be fitted
df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] <= estimated_edge_pos+options['fit_region']) & ((df_diff["ZapEnergy"] >= estimated_edge_pos-options['fit_region']))] df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] <= estimated_edge_pos+fit_region) & ((df_diff["ZapEnergy"] >= estimated_edge_pos-fit_region))]
# Fitting a function to the chosen interval # Fitting a function to the chosen interval
@ -594,7 +639,7 @@ def determine_edge_position(data: dict, options={}):
df_double_diff[filename]=df_double_diff[filename].shift(-int(options['double_diff.periods'])) df_double_diff[filename]=df_double_diff[filename].shift(-int(options['double_diff.periods']))
# Pick out region of interest # Pick out region of interest
df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+fit_region) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-fit_region))]
# Fitting a function to the chosen interval # Fitting a function to the chosen interval
params = np.polyfit(df_double_diff_edge["ZapEnergy"], df_double_diff_edge[filename], options['double_diff.polyorder']) params = np.polyfit(df_double_diff_edge["ZapEnergy"], df_double_diff_edge[filename], options['double_diff.polyorder'])
@ -634,10 +679,10 @@ def determine_edge_position(data: dict, options={}):
df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter')
df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2)
ax2.set_xlim([edge_pos_diff-0.0015, edge_pos_diff+0.0015]) ax2.set_xlim([edge_pos_diff-fit_region*1.5, edge_pos_diff+fit_region*1.5])
ax2.axvline(x=estimated_edge_pos-options['fit_region'], ls='--', c='black') ax2.axvline(x=estimated_edge_pos-fit_region, ls='--', c='black')
ax2.axvline(x=edge_pos_diff, ls='--', c='green') ax2.axvline(x=edge_pos_diff, ls='--', c='green')
ax2.axvline(x=estimated_edge_pos+options['fit_region'], ls='--', c='black') ax2.axvline(x=estimated_edge_pos+fit_region, ls='--', c='black')
ax2.set_title('Fit region of differentiated data') ax2.set_title('Fit region of differentiated data')
df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter') df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter')
@ -653,9 +698,9 @@ def determine_edge_position(data: dict, options={}):
df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax5, kind='scatter') df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax5, kind='scatter')
df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax5) df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax5)
ax5.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015]) ax5.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015])
ax5.axvline(x=estimated_edge_pos-options['fit_region'], ls='--', c='black') ax5.axvline(x=estimated_edge_pos-fit_region, ls='--', c='black')
ax5.axvline(x=edge_pos_double_diff, ls='--', c='green') ax5.axvline(x=edge_pos_double_diff, ls='--', c='green')
ax5.axvline(x=estimated_edge_pos+options['fit_region'], ls='--', c='black') ax5.axvline(x=estimated_edge_pos+fit_region, ls='--', c='black')
df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax6, kind='scatter') df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax6, kind='scatter')
df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax6) df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax6)
@ -672,10 +717,10 @@ def determine_edge_position(data: dict, options={}):
df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter')
df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2)
ax2.set_xlim([edge_pos_diff-0.5, edge_pos_diff+0.5]) ax2.set_xlim([edge_pos_diff-fit_region*1.5, edge_pos_diff+fit_region*1.5])
ax2.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_diff-fit_region, ls='--', c='black')
ax2.axvline(x=edge_pos_diff, ls='--', c='green') ax2.axvline(x=edge_pos_diff, ls='--', c='green')
ax2.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_diff+fit_region, ls='--', c='black')
df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3)
df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3)
@ -691,10 +736,10 @@ def determine_edge_position(data: dict, options={}):
df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter')
df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2)
ax2.set_xlim([edge_pos_double_diff-0.5, edge_pos_double_diff+0.5]) ax2.set_xlim([edge_pos_double_diff-fit_region*1.5, edge_pos_double_diff+fit_region*1.5])
ax2.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_double_diff-fit_region, ls='--', c='black')
ax2.axvline(x=edge_pos_double_diff, ls='--', c='green') ax2.axvline(x=edge_pos_double_diff, ls='--', c='green')
ax2.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_double_diff+fit_region, ls='--', c='black')
df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3)
df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3)
@ -724,35 +769,59 @@ def determine_edge_position(data: dict, options={}):
return edge_pos_diff, edge_pos_double_diff return edge_pos_diff, edge_pos_double_diff
def determine_edge_position_interactive(data: dict, options: dict) -> None:
''' Defines the widgets to use with the ipywidgets interactive mode and calls the update function found in btp.ipywidgets. '''
step_size = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]
w = widgets.interactive(
btp.ipywidgets_update, func=widgets.fixed(determine_edge_position), data=widgets.fixed(data), options=widgets.fixed(options),
points_around_edge=widgets.IntSlider(value=options['points_around_edge'], min=1, max=20, step=1),
)
options['widget'] = w
display(w)
def normalise(data: dict, options={}): def normalise(data: dict, options={}):
required_options = ['log', 'logfile', 'save_values'] required_options = ['log', 'logfile', 'normalisation_store_data']
default_options = { default_options = {
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log',
'save_values': True 'normalisation_store_data': False,
} }
options = aux.update_options(options=options, required_options=required_options, default_options=default_options) options = aux.update_options(options=options, required_options=required_options, default_options=default_options)
normalised_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) normalised_df = pd.DataFrame(data['xanes_data']['ZapEnergy'])
data['normalisation_constants'] = {} data['normalisation_constants'] = {}
if options['normalisation_store_data']:
pre_edge_fit_data_norm = pd.DataFrame(data['pre_edge_fit_data']['ZapEnergy'])
post_edge_fit_data_norm = pd.DataFrame(data['post_edge_fit_data']['ZapEnergy'])
#Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0
for filename in data['path']: for filename in data['path']:
e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0'][filename])].index.values[0] e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0_diff'][filename])].index.values[0]
#norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])]
normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind]
normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant) normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant)
if options['normalisation_store_data']:
pre_edge_fit_data_norm.insert(1, filename, data['pre_edge_fit_data'][filename] / normalisation_constant)
post_edge_fit_data_norm.insert(1, filename, data['post_edge_fit_data'][filename] / normalisation_constant)
if options['normalisation_store_data']:
data['xanes_data'] = normalised_df
# Normalise the pre-edge and post-edge fit function data # Normalise the pre-edge and post-edge fit function data
data['pre_edge_fit_data'][filename] = data['pre_edge_fit_data'][filename] / normalisation_constant data['pre_edge_fit_data_norm'] = pre_edge_fit_data_norm
data['post_edge_fit_data'][filename] = data['post_edge_fit_data'][filename] / normalisation_constant data['post_edge_fit_data_norm'] = post_edge_fit_data_norm
data['normalisation_constants'][filename] = normalisation_constant data['normalisation_constants'][filename] = normalisation_constant
if options['save_values']:
data['xanes_data'] = normalised_df
return normalised_df return normalised_df
@ -760,11 +829,11 @@ def normalise(data: dict, options={}):
def flatten(data:dict, options={}): def flatten(data:dict, options={}):
#only picking out zapenergy-values higher than edge position (edge pos and below remains untouched) #only picking out zapenergy-values higher than edge position (edge pos and below remains untouched)
required_options = ['log', 'logfile', 'save_values'] required_options = ['log', 'logfile', 'flatten_store_data']
default_options = { default_options = {
'log': False, 'log': False,
'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_flattening.log', 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_flattening.log',
'save_values': True 'flatten_store_data': False,
} }
options = aux.update_options(options=options, required_options=required_options, default_options=default_options) options = aux.update_options(options=options, required_options=required_options, default_options=default_options)
@ -772,13 +841,14 @@ def flatten(data:dict, options={}):
flattened_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) flattened_df = pd.DataFrame(data['xanes_data']['ZapEnergy'])
for filename in data['path']: for filename in data['path']:
fit_function_diff = -data['post_edge_fit_data'][filename] + data['pre_edge_params'][filename][0] fit_function_diff = data['post_edge_fit_data_norm'][filename] - 1
fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0'][filename]] = 0
fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0_diff'][filename]] = 0
flattened_df[filename] = data['xanes_data'][filename] - fit_function_diff flattened_df[filename] = data['xanes_data'][filename] - fit_function_diff
if options['save_values']: if options['flatten_store_data']:
data['xanes_data'] = flattened_df data['xanes_data'] = flattened_df