From a897596ca38e5308c30a6259ac011a55eab8d4b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 14:37:40 +0200 Subject: [PATCH 001/355] Initial commit --- .gitignore | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 131 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..de5a12d --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# beamtime +A package for processing and analysis of data from beamtime at SNBL From b0b5acdb14984d72e264f34cac4dac2080cb2e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 14:39:36 +0200 Subject: [PATCH 002/355] Create __init__.py --- __init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 __init__.py diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/__init__.py @@ -0,0 +1 @@ + From eaabacb34fcf3dc403800502892f1d2a3c1f996e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 14:51:28 +0200 Subject: [PATCH 003/355] Add subpackage pdf --- pdf/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pdf/__init__.py diff --git a/pdf/__init__.py b/pdf/__init__.py new file mode 100644 index 0000000..e69de29 From c1a4db460aebd320d479752db5b3aaaba2ae73b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 14:51:50 +0200 Subject: [PATCH 004/355] Add subpackage xanes --- xanes/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 xanes/__init__.py diff --git a/xanes/__init__.py b/xanes/__init__.py new file mode 100644 index 0000000..e69de29 From 04c8b05df8019e7a06d96077d6c9ed3a9507b2fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 14:52:07 +0200 Subject: [PATCH 005/355] Add subpackage xrd --- xrd/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 xrd/__init__.py diff --git a/xrd/__init__.py b/xrd/__init__.py new file mode 100644 index 0000000..e69de29 From 977ec020f36935b7610023a23b11c1abcd85a792 Mon Sep 17 00:00:00 2001 From: nanowhale Date: Thu, 9 Sep 2021 15:32:56 +0200 Subject: [PATCH 006/355] add io function --- xanes/io.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 xanes/io.py diff --git a/xanes/io.py b/xanes/io.py new file mode 100644 index 0000000..e69de29 From 2c306003bb308abcc64bac066e78359083e06bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 15:36:22 +0200 Subject: [PATCH 007/355] Add new subfolder to match package structure --- README.md => beamtime/README.md | 0 __init__.py => beamtime/__init__.py | 0 {pdf => beamtime/pdf}/__init__.py | 0 beamtime/xanes/__init__.py | 1 + beamtime/xanes/test.py | 2 ++ {xanes => beamtime/xrd}/__init__.py | 0 setup.py | 11 +++++++++++ xrd/__init__.py | 0 8 files changed, 14 insertions(+) rename README.md => beamtime/README.md (100%) rename __init__.py => beamtime/__init__.py (100%) rename {pdf => beamtime/pdf}/__init__.py (100%) create mode 100644 beamtime/xanes/__init__.py create mode 100644 beamtime/xanes/test.py rename {xanes => beamtime/xrd}/__init__.py (100%) create mode 100644 setup.py delete mode 100644 xrd/__init__.py diff --git a/README.md b/beamtime/README.md similarity index 100% rename from README.md rename to beamtime/README.md diff --git a/__init__.py b/beamtime/__init__.py similarity index 100% rename from __init__.py rename to beamtime/__init__.py diff --git a/pdf/__init__.py b/beamtime/pdf/__init__.py similarity index 100% rename from pdf/__init__.py rename to beamtime/pdf/__init__.py diff --git a/beamtime/xanes/__init__.py b/beamtime/xanes/__init__.py new file mode 100644 index 0000000..0e39ba3 --- /dev/null +++ b/beamtime/xanes/__init__.py @@ -0,0 +1 @@ +from . import test diff --git a/beamtime/xanes/test.py b/beamtime/xanes/test.py new file mode 100644 index 0000000..9462ba3 --- /dev/null +++ b/beamtime/xanes/test.py @@ -0,0 +1,2 @@ +def test_func(): + print('Hello world!') diff --git a/xanes/__init__.py b/beamtime/xrd/__init__.py similarity index 100% rename from xanes/__init__.py rename to beamtime/xrd/__init__.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4c93b78 --- /dev/null +++ b/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup, find_packages + +setup(name='beamtime', + version='0.1', + description='Package for process and analysis of beamtime data from SNBL', + url='http://github.uio.no/rasmusvt/beamtime', + author='Rasmus Vester Thøgersen, Halvor Høen Hval', + author_email='rasmusvt@smn.uio.no', + license='MIT', + packages=find_packages(), + zip_safe=False) diff --git a/xrd/__init__.py b/xrd/__init__.py deleted file mode 100644 index e69de29..0000000 From 776d52e1ff36c095fcb4c425f13341fcd13a10f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 15:39:00 +0200 Subject: [PATCH 008/355] Move io.py into right folder --- {xanes => beamtime/xanes}/io.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {xanes => beamtime/xanes}/io.py (100%) diff --git a/xanes/io.py b/beamtime/xanes/io.py similarity index 100% rename from xanes/io.py rename to beamtime/xanes/io.py From add466456abf2d97b134fbeb807b0883d59d1235 Mon Sep 17 00:00:00 2001 From: nanowhale Date: Thu, 9 Sep 2021 15:44:19 +0200 Subject: [PATCH 009/355] calibration script --- beamtime/xanes/calib.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/xanes/calib.py diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py new file mode 100644 index 0000000..e69de29 From b9d810aafde970f9b9c5e31d55174665cb8850f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 16:02:59 +0200 Subject: [PATCH 010/355] Delete test.py --- beamtime/xanes/test.py | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 beamtime/xanes/test.py diff --git a/beamtime/xanes/test.py b/beamtime/xanes/test.py deleted file mode 100644 index 9462ba3..0000000 --- a/beamtime/xanes/test.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_func(): - print('Hello world!') From 7825871631706e2e18374437d515604f2c5e906e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 9 Sep 2021 16:07:07 +0200 Subject: [PATCH 011/355] Test imports of pandas and numpy --- beamtime/__init__.py | 3 ++- beamtime/xanes/__init__.py | 4 +++- beamtime/xanes/io.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/beamtime/__init__.py b/beamtime/__init__.py index 8b13789..5a49abe 100644 --- a/beamtime/__init__.py +++ b/beamtime/__init__.py @@ -1 +1,2 @@ - +import pandas as pd +import numpy as np diff --git a/beamtime/xanes/__init__.py b/beamtime/xanes/__init__.py index 0e39ba3..2c1cc11 100644 --- a/beamtime/xanes/__init__.py +++ b/beamtime/xanes/__init__.py @@ -1 +1,3 @@ -from . import test +import pandas as pd +import numpy as np +from . import io, calib diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index e69de29..bf09558 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -0,0 +1 @@ +df = pd.DataFrame() From dc2a7ccd59f56c5e584645db8223676ab988ad0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Fri, 10 Sep 2021 10:51:00 +0200 Subject: [PATCH 012/355] Add electrochemistry module --- beamtime/electrochemistry/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/electrochemistry/__init__.py diff --git a/beamtime/electrochemistry/__init__.py b/beamtime/electrochemistry/__init__.py new file mode 100644 index 0000000..e69de29 From aac9491d16d9e4323094bc41500d6cd71cec6e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Fri, 10 Sep 2021 13:05:38 +0200 Subject: [PATCH 013/355] Add io.py --- beamtime/electrochemistry/io.py | 34 +++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 beamtime/electrochemistry/io.py diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py new file mode 100644 index 0000000..f688e90 --- /dev/null +++ b/beamtime/electrochemistry/io.py @@ -0,0 +1,34 @@ +import pandas as pd + + +def read_battsmall(path): + ''' Reads BATTSMALL-data into a DataFrame. + + Input: + path (required): string with path to datafile + + Output: + df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' + + df = pd.read_csv(data1, skiprows=2, sep='\t') + df = df.loc[:, ~df.columns.str.contains('^Unnamed')] + + return df + + +def clean_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): + ''' Takes BATTSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. + Also adds a column indicating whether or not it is charge or discharge. + + Input: + df (required): A pandas DataFrame containing BATTSMALL-data, as obtained from read_battsmall(). + t (optional): Unit for time data. Defaults to ms. + C (optional): Unit for specific capacity. Defaults to mAh/g. + I (optional): Unit for current. Defaults mA. + U (optional): Unit for voltage. Defaults to V. + + Output: + df: A cleaned up DataFrame with desired units and additional column about charge/discharge. + ''' + + \ No newline at end of file From 322d73a241fc4aafbefb65962b7d0d8423aa6db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Fri, 10 Sep 2021 13:16:00 +0200 Subject: [PATCH 014/355] Test --- beamtime/electrochemistry/text.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/electrochemistry/text.txt diff --git a/beamtime/electrochemistry/text.txt b/beamtime/electrochemistry/text.txt new file mode 100644 index 0000000..e69de29 From d200b25b7ae0a8a02eb2368a7af0cb80a03d6704 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 13:17:03 +0200 Subject: [PATCH 015/355] Test 2 --- beamtime/electrochemistry/text.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 beamtime/electrochemistry/text.txt diff --git a/beamtime/electrochemistry/text.txt b/beamtime/electrochemistry/text.txt deleted file mode 100644 index e69de29..0000000 From 59c4249b07bf4b07710e187b1a405bcba3948484 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 13:19:55 +0200 Subject: [PATCH 016/355] Test 3 --- beamtime/electrochemistry/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/electrochemistry/test.txt diff --git a/beamtime/electrochemistry/test.txt b/beamtime/electrochemistry/test.txt new file mode 100644 index 0000000..e69de29 From 039718207ec2bc3744d1c83b20a5a024886c7449 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 10 Sep 2021 13:24:57 +0200 Subject: [PATCH 017/355] Test 5 --- beamtime/test/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test/test.txt diff --git a/beamtime/test/test.txt b/beamtime/test/test.txt new file mode 100644 index 0000000..e69de29 From 27f0245183e4de06f128a4d856b64084663a2d47 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 13:39:05 +0200 Subject: [PATCH 018/355] Remove test-folder --- beamtime/test/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 beamtime/test/test.txt diff --git a/beamtime/test/test.txt b/beamtime/test/test.txt deleted file mode 100644 index e69de29..0000000 From 7b81b9f00417a338b5051c07aee49b22cd2af019 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 14:29:35 +0200 Subject: [PATCH 019/355] Clean up --- beamtime/electrochemistry/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 beamtime/electrochemistry/test.txt diff --git a/beamtime/electrochemistry/test.txt b/beamtime/electrochemistry/test.txt deleted file mode 100644 index e69de29..0000000 From 86c49af9087bf42651c58313815d3ed98da41d91 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 14:30:04 +0200 Subject: [PATCH 020/355] Add plot.py to electrochemistry --- beamtime/electrochemistry/plot.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 beamtime/electrochemistry/plot.py diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py new file mode 100644 index 0000000..514b463 --- /dev/null +++ b/beamtime/electrochemistry/plot.py @@ -0,0 +1,5 @@ +import matplotlib.pyplot as plt + +print('test') + +print('test2') \ No newline at end of file From 9bc0eb44b54f623b0ffeb7639177b287a464c107 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 10 Sep 2021 14:38:35 +0200 Subject: [PATCH 021/355] testttttt --- beamtime/halvorsen.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 beamtime/halvorsen.txt diff --git a/beamtime/halvorsen.txt b/beamtime/halvorsen.txt new file mode 100644 index 0000000..c6ae243 --- /dev/null +++ b/beamtime/halvorsen.txt @@ -0,0 +1 @@ +'''test''' \ No newline at end of file From 6a78d2f27096747980f18b7d52ee7889edf3f663 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 10 Sep 2021 15:00:49 +0200 Subject: [PATCH 022/355] hm --- beamtime/halvorsen.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/beamtime/halvorsen.txt b/beamtime/halvorsen.txt index c6ae243..233bbe4 100644 --- a/beamtime/halvorsen.txt +++ b/beamtime/halvorsen.txt @@ -1 +1,3 @@ -'''test''' \ No newline at end of file +'''test''' + +# changez From ea3eb8c2abb2e2fcb95cdd16e50d6ab568661ea5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 15:05:58 +0200 Subject: [PATCH 023/355] Add unit conversion matrices --- beamtime/electrochemistry/io.py | 45 ++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index f688e90..f6d4f2a 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -31,4 +31,47 @@ def clean_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): df: A cleaned up DataFrame with desired units and additional column about charge/discharge. ''' - \ No newline at end of file + + + + + + + +def unit_conversion(df, t, C, I, U): + + + # Get the units used in the data set + t_prev = df.columns[0].split()[-1].strip('[]') + U_prev = df.columns[1].split()[-1].strip('[]') + I_prev = df.columns[2].split()[-1].strip('[]') + C_prev_num, C_prev_den = df.columns[4].split()[-1].strip('[]').split('/') + + # Define matrix for unit conversion for time + t_units_df = {'h': [1, 60, 3600, 3600000], 'min': [1/60, 1, 60, 60000], 's': [1/3600, 1/60, 1, 1000], 'ms': [1/3600000, 1/60000, 1/1000, 1]} + t_units_df = pd.DataFrame(t_units_df) + t_units_df.index = ['h', 'min', 's', 'ms'] + + # Define matrix for unit conversion for current + I_units_df = {'A': [1, 60, 3600, 3600000], 'mA': [1/60, 1, 60, 60000], 'uA': [1/3600, 1/60, 1, 1000]} + I_units_df = pd.DataFrame(I_units_df) + I_units_df.index = ['A', 'mA', 'uA'] + + # Define matrix for unit conversion for voltage + I_units_df = {'A': [1, 60, 3600, 3600000], 'mA': [1/60, 1, 60, 60000], 'uA': [1/3600, 1/60, 1, 1000]} + I_units_df = pd.DataFrame(I_units_df) + I_units_df.index = ['A', 'mA', 'uA'] + + # Define matrix for unit conversion for capacity + C_units_df = {'Ah': [1, 60, 3600, 3600000], 'mAh': [1/60, 1, 60, 60000], 'uAh': [1/3600, 1/60, 1, 1000]} + C_units_df = pd.DataFrame(C_units_df) + C_units_df.index = ['Ah', 'mAh', 'uAh'] + + + + + + + + + From 78df31fe4c777bc810ad606db1bec6381fcde764 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 10 Sep 2021 15:07:43 +0200 Subject: [PATCH 024/355] second --- beamtime/halvorsen.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beamtime/halvorsen.txt b/beamtime/halvorsen.txt index 233bbe4..cecc2a9 100644 --- a/beamtime/halvorsen.txt +++ b/beamtime/halvorsen.txt @@ -1,3 +1,5 @@ '''test''' # changez +jkonkjnmkjnkj +5616521656 \ No newline at end of file From bd3ca65827bb6084c0e9bcffa58ad550c8925b1c Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 10 Sep 2021 15:38:27 +0200 Subject: [PATCH 025/355] rasmus' splitting fuction --- beamtime/xanes/calib.py | 79 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index e69de29..c1aeb5b 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -0,0 +1,79 @@ +import pandas as pd +import numpy as np +import os + +def split_xanes_scan(filename, destination=None): + + with open(filename, 'r') as f: + lines = f.readlines() + + datas = [] + data = [] + headers = [] + header = '' + start = False + + for line in lines: + if line[0:2] == "#L": + start = True + header = line[2:].split() + continue + + elif line[0:2] == "#C": + start = False + + if data: + datas.append(data) + data = [] + + if header: + headers.append(header) + header = '' + + + + if start == False: + continue + + else: + data.append(line.split()) + + + + + edges = {'Mn': [6.0, 6.1, 6.2, 6.3, 6.4, 6.5], 'Fe': [6.8, 6.9, 7.0, 7.1, 7.2], 'Co': [7.6, 7.7, 7.8, 7.9], 'Ni': [8.1, 8.2, 8.3, 8.4, 8.5]} + edge_count = {'Mn': 0, 'Fe': 0, 'Co': 0, 'Ni': 0} + + + for ind, data in enumerate(datas): + df = pd.DataFrame(data) + df.columns = headers[ind] + + edge_start = np.round((float(df["ZapEnergy"].min())), 1) + + for edge, energies in edges.items(): + if edge_start in energies: + edge_actual = edge + edge_count[edge] += 1 + + + + filename = filename.split('/')[-1] + count = str(edge_count[edge_actual]).zfill(4) + + + # Save + if destination: + cwd = os.getcwd() + + if not os.path.isdir(destination): + os.mkdir(destination) + + os.chdir(destination) + + df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + + os.chdir(cwd) + + else: + df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) \ No newline at end of file From 7b029249021630af29fd8cf30dcea7d4e322a8fa Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 10 Sep 2021 16:29:31 +0200 Subject: [PATCH 026/355] fikifiksi --- beamtime/xanes/calib.py | 4 ++++ beamtime/xanes/io.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index c1aeb5b..6817b69 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -2,6 +2,10 @@ import pandas as pd import numpy as np import os +def rbkerbest(): + print("ROSENBORG!<3") + + def split_xanes_scan(filename, destination=None): with open(filename, 'r') as f: diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index bf09558..e69de29 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -1 +0,0 @@ -df = pd.DataFrame() From e7f3bca3769820b6db82e12535cb646f22f160d0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 16:52:27 +0200 Subject: [PATCH 027/355] Clean up __init__.py's --- beamtime/__init__.py | 3 +-- beamtime/electrochemistry/__init__.py | 1 + beamtime/xanes/__init__.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/beamtime/__init__.py b/beamtime/__init__.py index 5a49abe..8b13789 100644 --- a/beamtime/__init__.py +++ b/beamtime/__init__.py @@ -1,2 +1 @@ -import pandas as pd -import numpy as np + diff --git a/beamtime/electrochemistry/__init__.py b/beamtime/electrochemistry/__init__.py index e69de29..e0c4c87 100644 --- a/beamtime/electrochemistry/__init__.py +++ b/beamtime/electrochemistry/__init__.py @@ -0,0 +1 @@ +from . import io, plot diff --git a/beamtime/xanes/__init__.py b/beamtime/xanes/__init__.py index 2c1cc11..b11c1f3 100644 --- a/beamtime/xanes/__init__.py +++ b/beamtime/xanes/__init__.py @@ -1,3 +1 @@ -import pandas as pd -import numpy as np -from . import io, calib +from . import io, calib \ No newline at end of file From d4b28326974b4994769636810e0592a3bf2e2e34 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 16:53:07 +0200 Subject: [PATCH 028/355] Add working unit conversion function for electrochemistry module --- beamtime/electrochemistry/io.py | 47 ++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index f6d4f2a..623126a 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -10,7 +10,7 @@ def read_battsmall(path): Output: df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' - df = pd.read_csv(data1, skiprows=2, sep='\t') + df = pd.read_csv(path, skiprows=2, sep='\t') df = df.loc[:, ~df.columns.str.contains('^Unnamed')] return df @@ -31,21 +31,29 @@ def clean_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): df: A cleaned up DataFrame with desired units and additional column about charge/discharge. ''' + df = unit_conversion(df=df, t=t, C=C, I=I, U=U) + + return df - - +def test_print(): + print('IT WORKS!') def unit_conversion(df, t, C, I, U): + print('YAHOO!') + + C, m = C.split('/') # Get the units used in the data set t_prev = df.columns[0].split()[-1].strip('[]') U_prev = df.columns[1].split()[-1].strip('[]') I_prev = df.columns[2].split()[-1].strip('[]') - C_prev_num, C_prev_den = df.columns[4].split()[-1].strip('[]').split('/') + C_prev, m_prev = df.columns[4].split()[-1].strip('[]').split('/') + + print(t_prev) # Define matrix for unit conversion for time t_units_df = {'h': [1, 60, 3600, 3600000], 'min': [1/60, 1, 60, 60000], 's': [1/3600, 1/60, 1, 1000], 'ms': [1/3600000, 1/60000, 1/1000, 1]} @@ -53,20 +61,40 @@ def unit_conversion(df, t, C, I, U): t_units_df.index = ['h', 'min', 's', 'ms'] # Define matrix for unit conversion for current - I_units_df = {'A': [1, 60, 3600, 3600000], 'mA': [1/60, 1, 60, 60000], 'uA': [1/3600, 1/60, 1, 1000]} + I_units_df = {'A': [1, 1000, 1000000], 'mA': [1/1000, 1, 1000], 'uA': [1/1000000, 1/1000, 1]} I_units_df = pd.DataFrame(I_units_df) I_units_df.index = ['A', 'mA', 'uA'] # Define matrix for unit conversion for voltage - I_units_df = {'A': [1, 60, 3600, 3600000], 'mA': [1/60, 1, 60, 60000], 'uA': [1/3600, 1/60, 1, 1000]} - I_units_df = pd.DataFrame(I_units_df) - I_units_df.index = ['A', 'mA', 'uA'] + U_units_df = {'V': [1, 1000, 1000000], 'mV': [1/1000, 1, 1000], 'uV': [1/1000000, 1/1000, 1]} + U_units_df = pd.DataFrame(U_units_df) + U_units_df.index = ['V', 'mV', 'uV'] # Define matrix for unit conversion for capacity - C_units_df = {'Ah': [1, 60, 3600, 3600000], 'mAh': [1/60, 1, 60, 60000], 'uAh': [1/3600, 1/60, 1, 1000]} + C_units_df = {'Ah': [1, 1000, 1000000], 'mAh': [1/1000, 1, 1000], 'uAh': [1/1000000, 1/1000, 1]} C_units_df = pd.DataFrame(C_units_df) C_units_df.index = ['Ah', 'mAh', 'uAh'] + # Define matrix for unit conversion for capacity + m_units_df = {'kg': [1, 1000, 1000000, 1000000000], 'g': [1/1000, 1, 1000, 1000000], 'mg': [1/1000000, 1/1000, 1, 1000], 'ug': [1/1000000000, 1/1000000, 1/1000, 1]} + m_units_df = pd.DataFrame(m_units_df) + m_units_df.index = ['kg', 'g', 'mg', 'ug'] + + + + + #print(df["TT [{}]".format(t_prev)]) + df["TT [{}]".format(t_prev)] = df["TT [{}]".format(t_prev)] * t_units_df[t_prev].loc[t] + df["U [{}]".format(U_prev)] = df["U [{}]".format(U_prev)] * U_units_df[U_prev].loc[U] + df["I [{}]".format(I_prev)] = df["I [{}]".format(I_prev)] * I_units_df[I_prev].loc[I] + df["C [{}/{}]".format(C_prev, m_prev)] = df["C [{}/{}]".format(C_prev, m_prev)] * (C_units_df[C_prev].loc[C] / m_units_df[m_prev].loc[m]) + + df.columns = ['TT [{}]'.format(t), 'U [{}]'.format(U), 'I [{}]'.format(I), 'Z1', 'C [{}/{}]'.format(C, m), 'Comment'] + + + + return df + @@ -74,4 +102,3 @@ def unit_conversion(df, t, C, I, U): - From bdf5264bcffe82ad0f3da5e02dbd6c7a43ef0c2d Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 10 Sep 2021 16:53:28 +0200 Subject: [PATCH 029/355] Remove uncessary test prints from io.py --- beamtime/electrochemistry/plot.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py index 514b463..d820b59 100644 --- a/beamtime/electrochemistry/plot.py +++ b/beamtime/electrochemistry/plot.py @@ -1,5 +1 @@ import matplotlib.pyplot as plt - -print('test') - -print('test2') \ No newline at end of file From 994a7d8c54ee0c24ab5ad93571d6df4980609ca1 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 14 Sep 2021 16:22:24 +0200 Subject: [PATCH 030/355] Add plot-function to ec-module --- beamtime/electrochemistry/io.py | 139 ++++++++++++++++++++++++-------- 1 file changed, 105 insertions(+), 34 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 623126a..23679ea 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -1,4 +1,6 @@ import pandas as pd +import numpy as np +import matplotlib.pyplot as plt def read_battsmall(path): @@ -16,36 +18,10 @@ def read_battsmall(path): return df -def clean_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): - ''' Takes BATTSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. - Also adds a column indicating whether or not it is charge or discharge. - Input: - df (required): A pandas DataFrame containing BATTSMALL-data, as obtained from read_battsmall(). - t (optional): Unit for time data. Defaults to ms. - C (optional): Unit for specific capacity. Defaults to mAh/g. - I (optional): Unit for current. Defaults mA. - U (optional): Unit for voltage. Defaults to V. +def unit_conversion(df, units): - Output: - df: A cleaned up DataFrame with desired units and additional column about charge/discharge. - ''' - - df = unit_conversion(df=df, t=t, C=C, I=I, U=U) - - return df - - - -def test_print(): - print('IT WORKS!') - - -def unit_conversion(df, t, C, I, U): - - print('YAHOO!') - - C, m = C.split('/') + C, m = units['C'].split('/') # Get the units used in the data set t_prev = df.columns[0].split()[-1].strip('[]') @@ -53,7 +29,6 @@ def unit_conversion(df, t, C, I, U): I_prev = df.columns[2].split()[-1].strip('[]') C_prev, m_prev = df.columns[4].split()[-1].strip('[]').split('/') - print(t_prev) # Define matrix for unit conversion for time t_units_df = {'h': [1, 60, 3600, 3600000], 'min': [1/60, 1, 60, 60000], 's': [1/3600, 1/60, 1, 1000], 'ms': [1/3600000, 1/60000, 1/1000, 1]} @@ -84,17 +59,101 @@ def unit_conversion(df, t, C, I, U): #print(df["TT [{}]".format(t_prev)]) - df["TT [{}]".format(t_prev)] = df["TT [{}]".format(t_prev)] * t_units_df[t_prev].loc[t] - df["U [{}]".format(U_prev)] = df["U [{}]".format(U_prev)] * U_units_df[U_prev].loc[U] - df["I [{}]".format(I_prev)] = df["I [{}]".format(I_prev)] * I_units_df[I_prev].loc[I] - df["C [{}/{}]".format(C_prev, m_prev)] = df["C [{}/{}]".format(C_prev, m_prev)] * (C_units_df[C_prev].loc[C] / m_units_df[m_prev].loc[m]) + df["TT [{}]".format(t_prev)] = df["TT [{}]".format(t_prev)] * t_units_df[t_prev].loc[units['t']] + df["U [{}]".format(U_prev)] = df["U [{}]".format(U_prev)] * U_units_df[U_prev].loc[units['U']] + df["I [{}]".format(I_prev)] = df["I [{}]".format(I_prev)] * I_units_df[I_prev].loc[units['I']] + df["C [{}/{}]".format(C_prev, m_prev)] = df["C [{}/{}]".format(C_prev, m_prev)] * (C_units_df[C_prev].loc[units['C']] / m_units_df[m_prev].loc[units['m']]) - df.columns = ['TT [{}]'.format(t), 'U [{}]'.format(U), 'I [{}]'.format(I), 'Z1', 'C [{}/{}]'.format(C, m), 'Comment'] + df.columns = ['TT', 'U', 'I', 'Z1', 'C', 'Comment'] return df +#def process_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): + +def process_battsmall_data(df, units=None): + ''' Takes BATTSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. + Splits up into individual charge and discharge DataFrames per cycle, and outputs a list where each element is a tuple with the Chg and DChg-data. E.g. cycles[10][0] gives the charge data for the 11th cycle. + + For this to work, the cycling program must be set to use the counter. + + Input: + df (required): A pandas DataFrame containing BATTSMALL-data, as obtained from read_battsmall(). + t (optional): Unit for time data. Defaults to ms. + C (optional): Unit for specific capacity. Defaults to mAh/g. + I (optional): Unit for current. Defaults mA. + U (optional): Unit for voltage. Defaults to V. + + Output: + cycles: A list with + ''' + + required_units = ['t', 'I', 'U', 'C'] + default_units = {'t': 'h', 'I': 'mA', 'U': 'V', 'C': 'mAh/g'} + + if not units: + units = default_units + + if units: + for unit in required_units: + if unit not in units.values(): + units[unit] = default_units[unit] + + + # Convert all units to the desired units. + df = unit_conversion(df=df, units=units) + + # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current and + df[["Comment"]] = df[["Comment"]].fillna(value={'Comment': ''}) + df = df[df["Comment"].str.contains("program")==False] + + # Creates masks for + chg_mask = df['I'] >= 0 + dchg_mask = df['I'] < 0 + + # Initiate cycles list + cycles = [] + + # Loop through all the cycling steps, change the current and capacities in the + for i in range(df["Z1"].max()): + sub_df = df.loc[df['Z1'] == i].copy() + sub_df.loc[dchg_mask, 'I'] *= -1 + sub_df.loc[dchg_mask, 'C'] *= -1 + + chg_df = sub_df.loc[chg_mask] + dchg_df = sub_df.loc[dchg_mask] + + + cycles.append((chg_df, dchg_df)) + + + return cycles + + + +def plot_gc(cycles, which_cycles='all', chg=True, dchg=True, colours=None, x='C', y='U'): + + fig, ax = prepare_gc_plot() + + + if which_cycles == 'all': + which_cycles = [i for i, c in enumerate(cycles)] + + if not colours: + chg_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B + dchg_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B + + + + for i, cycle in cycles: + if i in which_cycles: + if chg: + cycle[0].plot(ax=ax) + + + + @@ -102,3 +161,15 @@ def unit_conversion(df, t, C, I, U): +def prepare_gc_plot(figsize=(14,7), dpi=None): + + fig, ax = plt.subplots(figsize=figsize, dpi=dpi) + + + return fig, ax + + + + + + From 1ab2efad0d45985f0c2fa052543525a5bfe33b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Wed, 15 Sep 2021 09:50:16 +0200 Subject: [PATCH 031/355] Test push from laptop --- beamtime/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..e69de29 From 69fcb5ff872a47814bbd455a15125244fe2e6e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Wed, 15 Sep 2021 09:52:45 +0200 Subject: [PATCH 032/355] Remove test.txt --- beamtime/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt deleted file mode 100644 index e69de29..0000000 From c5caddf8c98ba0287ec42d9a9bd62a0d31c3ac85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Wed, 15 Sep 2021 09:53:56 +0200 Subject: [PATCH 033/355] Test again --- beamtime/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..e69de29 From a189c2060479d88b517872e2201636878ca30efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Wed, 15 Sep 2021 12:19:41 +0200 Subject: [PATCH 034/355] test --- beamtime/test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/beamtime/test.txt b/beamtime/test.txt index e69de29..989f51a 100644 --- a/beamtime/test.txt +++ b/beamtime/test.txt @@ -0,0 +1 @@ +test! From 553681fa412dccbb958bc1fd64916740c5923884 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Sep 2021 12:32:28 +0200 Subject: [PATCH 035/355] test --- beamtime/test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt deleted file mode 100644 index 989f51a..0000000 --- a/beamtime/test.txt +++ /dev/null @@ -1 +0,0 @@ -test! From bd722d572b3b7c83af170b53342b3b4196f64784 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Sep 2021 12:53:28 +0200 Subject: [PATCH 036/355] test --- beamtime/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..e69de29 From 3179ac46b002e4ddb8931dd643d559d80e7fb8a6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Sep 2021 13:20:03 +0200 Subject: [PATCH 037/355] test --- beamtime/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt deleted file mode 100644 index e69de29..0000000 From 1bf609e318698162e03dadd982e8ef659fec57b3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Sep 2021 13:42:14 +0200 Subject: [PATCH 038/355] test --- beamtime/test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..ff62eb7 --- /dev/null +++ b/beamtime/test.txt @@ -0,0 +1 @@ +dfgdfg From 2e910a2afb91d08bddb5c61efb0c7a9c88a4b11c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Oct 2021 13:07:22 +0200 Subject: [PATCH 039/355] Move plot functions to plot.py --- beamtime/electrochemistry/plot.py | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py index d820b59..70f2b22 100644 --- a/beamtime/electrochemistry/plot.py +++ b/beamtime/electrochemistry/plot.py @@ -1 +1,40 @@ import matplotlib.pyplot as plt +import pandas as pd +import numpy as np + + +def plot_gc(cycles, which_cycles='all', chg=True, dchg=True, colours=None, x='C', y='U'): + + fig, ax = prepare_gc_plot() + + + if which_cycles == 'all': + which_cycles = [i for i, c in enumerate(cycles)] + + if not colours: + chg_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B + dchg_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B + + + + for i, cycle in cycles: + if i in which_cycles: + if chg: + cycle[0].plot(ax=ax) + + + + + + + + + + + +def prepare_gc_plot(figsize=(14,7), dpi=None): + + fig, ax = plt.subplots(figsize=figsize, dpi=dpi) + + + return fig, ax \ No newline at end of file From ed5504449fadb406479233638eee7c84ae57cf1d Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Oct 2021 14:31:32 +0200 Subject: [PATCH 040/355] Change unit conversion function and add time string conversions --- beamtime/electrochemistry/io.py | 277 ++++++++++++++++++++++---------- 1 file changed, 192 insertions(+), 85 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 23679ea..86964ea 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -19,15 +19,166 @@ def read_battsmall(path): -def unit_conversion(df, units): - C, m = units['C'].split('/') +def read_neware(path, summary=False, active_material_weight=None, molecular_weight=None): + ''' Reads electrochemistry data, currently only from the Neware battery cycler. Will convert to .csv if the filetype is .xlsx, + which is the file format the Neware provides for the backup data. In this case it matters if summary is False or not. If file + type is .csv, it will just open the datafile and it does not matter if summary is False or not.''' + + # Convert from .xlsx to .csv to make readtime faster + if filename.split('.')[-1] == 'xlsx': + csv_details = ''.join(filename.split('.')[:-1]) + '_details.csv' + csv_summary = ''.join(filename.split('.')[:-1]) + '_summary.csv' + + Xlsx2csv(filename, outputencoding="utf-8").convert(csv_summary, sheetid=3) + Xlsx2csv(filename, outputencoding="utf-8").convert(csv_details, sheetid=4) + + if summary: + df = pd.read_csv(csv_summary) + else: + df = pd.read_csv(csv_details) + + elif filename.split('.')[-1] == 'csv': + + df = pd.read_csv(filename) + + + return df + + + + + + + +#def process_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): + +def process_battsmall_data(df, units=None, splice_cycles=None, molecular_weight=None): + ''' Takes BATTSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. + Splits up into individual charge and discharge DataFrames per cycle, and outputs a list where each element is a tuple with the Chg and DChg-data. E.g. cycles[10][0] gives the charge data for the 11th cycle. + + For this to work, the cycling program must be set to use the counter. + + Input: + df (required): A pandas DataFrame containing BATTSMALL-data, as obtained from read_battsmall(). + t (optional): Unit for time data. Defaults to ms. + C (optional): Unit for specific capacity. Defaults to mAh/g. + I (optional): Unit for current. Defaults mA. + U (optional): Unit for voltage. Defaults to V. + + Output: + cycles: A list with + ''' + + ######################### + #### UNIT CONVERSION #### + ######################### + + # Complete the list of units - if not all are passed, then default value will be used + required_units = ['t', 'I', 'U', 'C'] + default_units = {'t': 'h', 'I': 'mA', 'U': 'V', 'C': 'mAh/g'} + + if not units: + units = default_units + + if units: + for unit in required_units: + if unit not in units.values(): + units[unit] = default_units[unit] + + # Get the units used in the data set t_prev = df.columns[0].split()[-1].strip('[]') U_prev = df.columns[1].split()[-1].strip('[]') I_prev = df.columns[2].split()[-1].strip('[]') C_prev, m_prev = df.columns[4].split()[-1].strip('[]').split('/') + prev_units = {'t': t_prev, 'I': I_prev, 'U': U_prev, 'C': C_prev} + + # Convert all units to the desired units. + df = unit_conversion(df=df, units=units) + + # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current + df[["Comment"]] = df[["Comment"]].fillna(value={'Comment': ''}) + df = df[df["Comment"].str.contains("program")==False] + + # Creates masks for charge and discharge curves + chg_mask = df['I'] >= 0 + dchg_mask = df['I'] < 0 + + # Initiate cycles list + cycles = [] + + # Loop through all the cycling steps, change the current and capacities in the + for i in range(df["Z1"].max()): + + sub_df = df.loc[df['Z1'] == i].copy() + + sub_df.loc[dchg_mask, 'I'] *= -1 + sub_df.loc[dchg_mask, 'C'] *= -1 + + chg_df = sub_df.loc[chg_mask] + dchg_df = sub_df.loc[dchg_mask] + + # Continue to next iteration if the charge and discharge DataFrames are empty (i.e. no current) + if chg_df.empty and dchg_df.empty: + continue + + cycles.append((chg_df, dchg_df)) + + + + + return cycles + + +def process_neware_data(df, units=None, splice_cycles=None, active_material_weight=None, molecular_weight=None): + + ######################### + #### UNIT CONVERSION #### + ######################### + + # Complete the list of units - if not all are passed, then default value will be used + required_units = ['t', 'I', 'U', 'C'] + default_units = {'t': 'h', 'I': 'mA', 'U': 'V', 'C': 'mAh/g'} + + if not units: + units = default_units + + if units: + for unit in required_units: + if unit not in units.values(): + units[unit] = default_units[unit] + + + + # Get the units used in the data set + t_prev = 's' # default in + U_prev = df.columns[1].split()[-1].strip('[]') + I_prev = df.columns[2].split()[-1].strip('[]') + C_prev, m_prev = df.columns[4].split()[-1].strip('[]').split('/') + prev_units = {'t': t_prev, 'I': I_prev, 'U': U_prev, 'C': C_prev} + + # Convert all units to the desired units. + df = unit_conversion(df=df, units=units) + + + + if active_material_weight: + df["SpecificCapacity(mAh/g)"] = df["Capacity(mAh)"] / (active_material_weight / 1000) + + if molecular_weight: + faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 + seconds_per_hour = 3600 # s h^-1 + f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + + df["IonsExtracted"] = (df["SpecificCapacity(mAh/g)"]*molecular_weight)/f + + +def unit_conversion(df, units, prev_units, kind): + + C, m = units['C'].split('/') + C_prev, m_prev = prev_units['C'].split('/') # Define matrix for unit conversion for time @@ -55,14 +206,11 @@ def unit_conversion(df, units): m_units_df = pd.DataFrame(m_units_df) m_units_df.index = ['kg', 'g', 'mg', 'ug'] - - - #print(df["TT [{}]".format(t_prev)]) df["TT [{}]".format(t_prev)] = df["TT [{}]".format(t_prev)] * t_units_df[t_prev].loc[units['t']] df["U [{}]".format(U_prev)] = df["U [{}]".format(U_prev)] * U_units_df[U_prev].loc[units['U']] df["I [{}]".format(I_prev)] = df["I [{}]".format(I_prev)] * I_units_df[I_prev].loc[units['I']] - df["C [{}/{}]".format(C_prev, m_prev)] = df["C [{}/{}]".format(C_prev, m_prev)] * (C_units_df[C_prev].loc[units['C']] / m_units_df[m_prev].loc[units['m']]) + df["C [{}/{}]".format(C_prev, m_prev)] = df["C [{}/{}]".format(C_prev, m_prev)] * (C_units_df[C_prev].loc[C] / m_units_df[m_prev].loc[m]) df.columns = ['TT', 'U', 'I', 'Z1', 'C', 'Comment'] @@ -70,106 +218,65 @@ def unit_conversion(df, units): return df -#def process_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): - -def process_battsmall_data(df, units=None): - ''' Takes BATTSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. - Splits up into individual charge and discharge DataFrames per cycle, and outputs a list where each element is a tuple with the Chg and DChg-data. E.g. cycles[10][0] gives the charge data for the 11th cycle. - - For this to work, the cycling program must be set to use the counter. - - Input: - df (required): A pandas DataFrame containing BATTSMALL-data, as obtained from read_battsmall(). - t (optional): Unit for time data. Defaults to ms. - C (optional): Unit for specific capacity. Defaults to mAh/g. - I (optional): Unit for current. Defaults mA. - U (optional): Unit for voltage. Defaults to V. - - Output: - cycles: A list with - ''' - - required_units = ['t', 'I', 'U', 'C'] - default_units = {'t': 'h', 'I': 'mA', 'U': 'V', 'C': 'mAh/g'} - - if not units: - units = default_units - - if units: - for unit in required_units: - if unit not in units.values(): - units[unit] = default_units[unit] - - - # Convert all units to the desired units. - df = unit_conversion(df=df, units=units) - - # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current and - df[["Comment"]] = df[["Comment"]].fillna(value={'Comment': ''}) - df = df[df["Comment"].str.contains("program")==False] - - # Creates masks for - chg_mask = df['I'] >= 0 - dchg_mask = df['I'] < 0 - - # Initiate cycles list - cycles = [] - - # Loop through all the cycling steps, change the current and capacities in the - for i in range(df["Z1"].max()): - sub_df = df.loc[df['Z1'] == i].copy() - sub_df.loc[dchg_mask, 'I'] *= -1 - sub_df.loc[dchg_mask, 'C'] *= -1 - - chg_df = sub_df.loc[chg_mask] - dchg_df = sub_df.loc[dchg_mask] - cycles.append((chg_df, dchg_df)) +def convert_time_string(time_string, unit='ms'): + ''' Convert time string from Neware-data with the format hh:mm:ss.xx to any given unit''' + h, m, s = time_string.split(':') + ms = int(s)*1000 + int(m)*1000*60 + int(h)*1000*60*60 - return cycles + factors = {'ms': 1, 's': 1/1000, 'min': 1/(1000*60), 'h': 1/(1000*60*60)} + + t = ms*factors[unit] + + return t -def plot_gc(cycles, which_cycles='all', chg=True, dchg=True, colours=None, x='C', y='U'): +def convert_datetime_string(datetime_string, reference, unit='s'): + ''' Convert time string from Neware-data with the format yyy-mm-dd hh:mm:ss to any given unit''' - fig, ax = prepare_gc_plot() + from datetime import datetime + # Parse the + cur_date, cur_time = datetime_string.split() + cur_y, cur_mo, cur_d = cur_date.split('-') + cur_h, cur_m, cur_s = cur_time.split(':') + cur_date = datetime(int(cur_y), int(cur_mo), int(cur_d), int(cur_h), int(cur_m), int(cur_s)) - if which_cycles == 'all': - which_cycles = [i for i, c in enumerate(cycles)] + ref_date, ref_time = reference.split() + ref_y, ref_mo, ref_d = ref_date.split('-') + ref_h, ref_m, ref_s = ref_time.split(':') + ref_date = datetime(int(ref_y), int(ref_mo), int(ref_d), int(ref_h), int(ref_m), int(ref_s)) - if not colours: - chg_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B - dchg_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B - - + days = cur_date - ref_date - for i, cycle in cycles: - if i in which_cycles: - if chg: - cycle[0].plot(ax=ax) + s = days.seconds + factors = {'ms': 1000, 's': 1, 'min': 1/(60), 'h': 1/(60*60)} + t = s * factors[unit] + return t - +def splice_cycles(first, second): + + first_chg = first[0] + first_dchg = first[1] + first + + second_chg = second[0] + second_dchg = second[1] + + chg_df = first[0].append(second[0]) + + return True -def prepare_gc_plot(figsize=(14,7), dpi=None): - - fig, ax = plt.subplots(figsize=figsize, dpi=dpi) - - - return fig, ax - - - - From 5241776df786196913e9032dc7ab053e30f532ba Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 6 Oct 2021 15:28:29 +0200 Subject: [PATCH 041/355] dette er koedd --- beamtime/xanes/io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index e69de29..d08b5c1 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -0,0 +1 @@ +#hello \ No newline at end of file From 4a987808fc720dd82e15ac385bf529d2b9703b93 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Oct 2021 16:25:08 +0200 Subject: [PATCH 042/355] Update unit conversion functions --- beamtime/electrochemistry/__init__.py | 2 +- beamtime/electrochemistry/io.py | 226 +++++++++++------------ beamtime/electrochemistry/unit_tables.py | 53 ++++++ 3 files changed, 160 insertions(+), 121 deletions(-) create mode 100644 beamtime/electrochemistry/unit_tables.py diff --git a/beamtime/electrochemistry/__init__.py b/beamtime/electrochemistry/__init__.py index e0c4c87..0270f1d 100644 --- a/beamtime/electrochemistry/__init__.py +++ b/beamtime/electrochemistry/__init__.py @@ -1 +1 @@ -from . import io, plot +from . import io, plot, unit_tables diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 86964ea..3e5bf02 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -1,10 +1,11 @@ import pandas as pd import numpy as np import matplotlib.pyplot as plt +import os -def read_battsmall(path): - ''' Reads BATTSMALL-data into a DataFrame. +def read_batsmall(path): + ''' Reads BATSMALL-data into a DataFrame. Input: path (required): string with path to datafile @@ -21,47 +22,46 @@ def read_battsmall(path): def read_neware(path, summary=False, active_material_weight=None, molecular_weight=None): - ''' Reads electrochemistry data, currently only from the Neware battery cycler. Will convert to .csv if the filetype is .xlsx, - which is the file format the Neware provides for the backup data. In this case it matters if summary is False or not. If file - type is .csv, it will just open the datafile and it does not matter if summary is False or not.''' - + ''' Reads electrochemistry data, currently only from the Neware battery cycler. Will convert to .csv if the filetype is .xlsx, + which is the file format the Neware provides for the backup data. In this case it matters if summary is False or not. If file + type is .csv, it will just open the datafile and it does not matter if summary is False or not.''' + from xlsx2csv import Xlsx2csv - # Convert from .xlsx to .csv to make readtime faster - if filename.split('.')[-1] == 'xlsx': - csv_details = ''.join(filename.split('.')[:-1]) + '_details.csv' - csv_summary = ''.join(filename.split('.')[:-1]) + '_summary.csv' + # Convert from .xlsx to .csv to make readtime faster + if path.split('.')[-1] == 'xlsx': + csv_details = ''.join(path.split('.')[:-1]) + '_details.csv' + csv_summary = ''.join(path.split('.')[:-1]) + '_summary.csv' - Xlsx2csv(filename, outputencoding="utf-8").convert(csv_summary, sheetid=3) - Xlsx2csv(filename, outputencoding="utf-8").convert(csv_details, sheetid=4) - - if summary: - df = pd.read_csv(csv_summary) - else: - df = pd.read_csv(csv_details) + if not os.path.isfile(csv_summary): + Xlsx2csv(path, outputencoding="utf-8").convert(csv_summary, sheetid=3) - elif filename.split('.')[-1] == 'csv': + if not os.path.isfile(csv_details): + Xlsx2csv(path, outputencoding="utf-8").convert(csv_details, sheetid=4) - df = pd.read_csv(filename) - - - return df + if summary: + df = pd.read_csv(csv_summary) + else: + df = pd.read_csv(csv_details) + + elif path.split('.')[-1] == 'csv': + df = pd.read_csv(path) + + + return df - -#def process_battsmall_data(df, t='ms', C='mAh/g', I='mA', U='V'): - -def process_battsmall_data(df, units=None, splice_cycles=None, molecular_weight=None): - ''' Takes BATTSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. +def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=None): + ''' Takes BATSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. Splits up into individual charge and discharge DataFrames per cycle, and outputs a list where each element is a tuple with the Chg and DChg-data. E.g. cycles[10][0] gives the charge data for the 11th cycle. For this to work, the cycling program must be set to use the counter. Input: - df (required): A pandas DataFrame containing BATTSMALL-data, as obtained from read_battsmall(). + df (required): A pandas DataFrame containing BATSMALL-data, as obtained from read_batsmall(). t (optional): Unit for time data. Defaults to ms. C (optional): Unit for specific capacity. Defaults to mAh/g. I (optional): Unit for current. Defaults mA. @@ -71,32 +71,13 @@ def process_battsmall_data(df, units=None, splice_cycles=None, molecular_weight= cycles: A list with ''' - ######################### - #### UNIT CONVERSION #### - ######################### - # Complete the list of units - if not all are passed, then default value will be used - required_units = ['t', 'I', 'U', 'C'] - default_units = {'t': 'h', 'I': 'mA', 'U': 'V', 'C': 'mAh/g'} + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. + new_units = set_units(units=units) + old_units = get_old_units(df, kind='batsmall') + df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='batsmall') - if not units: - units = default_units - - if units: - for unit in required_units: - if unit not in units.values(): - units[unit] = default_units[unit] - - - # Get the units used in the data set - t_prev = df.columns[0].split()[-1].strip('[]') - U_prev = df.columns[1].split()[-1].strip('[]') - I_prev = df.columns[2].split()[-1].strip('[]') - C_prev, m_prev = df.columns[4].split()[-1].strip('[]').split('/') - prev_units = {'t': t_prev, 'I': I_prev, 'U': U_prev, 'C': C_prev} - - # Convert all units to the desired units. - df = unit_conversion(df=df, units=units) + df.columns = ['TT', 'U', 'I', 'Z1', 'C', 'Comment'] # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current df[["Comment"]] = df[["Comment"]].fillna(value={'Comment': ''}) @@ -138,93 +119,98 @@ def process_neware_data(df, units=None, splice_cycles=None, active_material_weig #### UNIT CONVERSION #### ######################### + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. + new_units = set_units(units=units) + old_units = get_old_units(df, kind='neware') + df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='neware') + + + # if active_material_weight: + # df["SpecificCapacity(mAh/g)"] = df["Capacity(mAh)"] / (active_material_weight / 1000) + + # if molecular_weight: + # faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 + # seconds_per_hour = 3600 # s h^-1 + # f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + + # df["IonsExtracted"] = (df["SpecificCapacity(mAh/g)"]*molecular_weight)/f + + + return df + + +def unit_conversion(df, new_units, old_units, kind): + from . import unit_tables + + if kind == 'batsmall': + + df["TT [{}]".format(old_units["time"])] = df["TT [{}]".format(old_units["time"])] * unit_tables.time()[old_units["time"]].loc[new_units['time']] + df["U [{}]".format(old_units["voltage"])] = df["U [{}]".format(old_units["voltage"])] * unit_tables.voltage()[old_units["voltage"]].loc[new_units['voltage']] + df["I [{}]".format(old_units["current"])] = df["I [{}]".format(old_units["current"])] * unit_tables.current()[old_units["current"]].loc[new_units['current']] + df["C [{}/{}]".format(old_units["capacity"], old_units["mass"])] = df["C [{}/{}]".format(old_units["capacity"], old_units["mass"])] * (unit_tables.capacity()[old_units["capacity"]].loc[new_units["capacity"]] / unit_tables.mass()[old_units["mass"]].loc[new_units["mass"]]) + + + if kind == 'neware': + df['Current({})'.format(old_units['current'])] = df['Current({})'.format(old_units['current'])] * unit_tables.current()[old_units['current']].loc[new_units['current']] + df['Voltage({})'.format(old_units['voltage'])] = df['Voltage({})'.format(old_units['voltage'])] * unit_tables.voltage()[old_units['voltage']].loc[new_units['voltage']] + df['Capacity({})'.format(old_units['capacity'])] = df['Capacity({})'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] + df['Energy({})'.format(old_units['energy'])] = df['Energy({})'.format(old_units['energy'])] * unit_tables.energy()[old_units['energy']].loc[new_units['energy']] + + df['RelativeTime({})'.format(new_units['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=new_units['time']), axis=1) + + return df + + +def set_units(units=None): + # Complete the list of units - if not all are passed, then default value will be used - required_units = ['t', 'I', 'U', 'C'] - default_units = {'t': 'h', 'I': 'mA', 'U': 'V', 'C': 'mAh/g'} + required_units = ['time', 'current', 'voltage', 'capacity', 'mass', 'energy'] + default_units = {'time': 'h', 'current': 'mA', 'voltage': 'V', 'capacity': 'mAh', 'mass': 'g', 'energy': 'mWh'} if not units: units = default_units if units: for unit in required_units: - if unit not in units.values(): + if unit not in units.keys(): units[unit] = default_units[unit] + + return units + + + +def get_old_units(df, kind): + if kind=='batsmall': + time = df.columns[0].split()[-1].strip('[]') + voltage = df.columns[1].split()[-1].strip('[]') + current = df.columns[2].split()[-1].strip('[]') + capacity, mass = df.columns[4].split()[-1].strip('[]').split('/') + old_units = {'time': time, 'current': current, 'voltage': voltage, 'capacity': capacity, 'mass': mass} - # Get the units used in the data set - t_prev = 's' # default in - U_prev = df.columns[1].split()[-1].strip('[]') - I_prev = df.columns[2].split()[-1].strip('[]') - C_prev, m_prev = df.columns[4].split()[-1].strip('[]').split('/') - prev_units = {'t': t_prev, 'I': I_prev, 'U': U_prev, 'C': C_prev} + if kind=='neware': + + for column in df.columns: + if 'Voltage' in column: + voltage = column.split('(')[-1].strip(')') + elif 'Current' in column: + current = column.split('(')[-1].strip(')') + elif 'Capacity' in column: + capacity = column.split('(')[-1].strip(')') + elif 'Energy' in column: + energy = column.split('(')[-1].strip(')') - # Convert all units to the desired units. - df = unit_conversion(df=df, units=units) - - - - if active_material_weight: - df["SpecificCapacity(mAh/g)"] = df["Capacity(mAh)"] / (active_material_weight / 1000) - - if molecular_weight: - faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 - seconds_per_hour = 3600 # s h^-1 - f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 - - df["IonsExtracted"] = (df["SpecificCapacity(mAh/g)"]*molecular_weight)/f - - -def unit_conversion(df, units, prev_units, kind): - - C, m = units['C'].split('/') - C_prev, m_prev = prev_units['C'].split('/') - - - # Define matrix for unit conversion for time - t_units_df = {'h': [1, 60, 3600, 3600000], 'min': [1/60, 1, 60, 60000], 's': [1/3600, 1/60, 1, 1000], 'ms': [1/3600000, 1/60000, 1/1000, 1]} - t_units_df = pd.DataFrame(t_units_df) - t_units_df.index = ['h', 'min', 's', 'ms'] - - # Define matrix for unit conversion for current - I_units_df = {'A': [1, 1000, 1000000], 'mA': [1/1000, 1, 1000], 'uA': [1/1000000, 1/1000, 1]} - I_units_df = pd.DataFrame(I_units_df) - I_units_df.index = ['A', 'mA', 'uA'] - - # Define matrix for unit conversion for voltage - U_units_df = {'V': [1, 1000, 1000000], 'mV': [1/1000, 1, 1000], 'uV': [1/1000000, 1/1000, 1]} - U_units_df = pd.DataFrame(U_units_df) - U_units_df.index = ['V', 'mV', 'uV'] - - # Define matrix for unit conversion for capacity - C_units_df = {'Ah': [1, 1000, 1000000], 'mAh': [1/1000, 1, 1000], 'uAh': [1/1000000, 1/1000, 1]} - C_units_df = pd.DataFrame(C_units_df) - C_units_df.index = ['Ah', 'mAh', 'uAh'] - - # Define matrix for unit conversion for capacity - m_units_df = {'kg': [1, 1000, 1000000, 1000000000], 'g': [1/1000, 1, 1000, 1000000], 'mg': [1/1000000, 1/1000, 1, 1000], 'ug': [1/1000000000, 1/1000000, 1/1000, 1]} - m_units_df = pd.DataFrame(m_units_df) - m_units_df.index = ['kg', 'g', 'mg', 'ug'] - - #print(df["TT [{}]".format(t_prev)]) - df["TT [{}]".format(t_prev)] = df["TT [{}]".format(t_prev)] * t_units_df[t_prev].loc[units['t']] - df["U [{}]".format(U_prev)] = df["U [{}]".format(U_prev)] * U_units_df[U_prev].loc[units['U']] - df["I [{}]".format(I_prev)] = df["I [{}]".format(I_prev)] * I_units_df[I_prev].loc[units['I']] - df["C [{}/{}]".format(C_prev, m_prev)] = df["C [{}/{}]".format(C_prev, m_prev)] * (C_units_df[C_prev].loc[C] / m_units_df[m_prev].loc[m]) - - df.columns = ['TT', 'U', 'I', 'Z1', 'C', 'Comment'] - - - - return df + old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy} + return old_units def convert_time_string(time_string, unit='ms'): ''' Convert time string from Neware-data with the format hh:mm:ss.xx to any given unit''' h, m, s = time_string.split(':') - ms = int(s)*1000 + int(m)*1000*60 + int(h)*1000*60*60 + ms = float(s)*1000 + int(m)*1000*60 + int(h)*1000*60*60 factors = {'ms': 1, 's': 1/1000, 'min': 1/(1000*60), 'h': 1/(1000*60*60)} diff --git a/beamtime/electrochemistry/unit_tables.py b/beamtime/electrochemistry/unit_tables.py new file mode 100644 index 0000000..c839b9b --- /dev/null +++ b/beamtime/electrochemistry/unit_tables.py @@ -0,0 +1,53 @@ +import pandas as pd + +def time(): + # Define matrix for unit conversion for time + time = {'h': [1, 60, 3600, 3600000], 'min': [1/60, 1, 60, 60000], 's': [1/3600, 1/60, 1, 1000], 'ms': [1/3600000, 1/60000, 1/1000, 1]} + time = pd.DataFrame(time) + time.index = ['h', 'min', 's', 'ms'] + + return time + +def current(): + # Define matrix for unit conversion for current + current = {'A': [1, 1000, 1000000], 'mA': [1/1000, 1, 1000], 'uA': [1/1000000, 1/1000, 1]} + current = pd.DataFrame(current) + current.index = ['A', 'mA', 'uA'] + + return current + +def voltage(): + # Define matrix for unit conversion for voltage + voltage = {'V': [1, 1000, 1000000], 'mV': [1/1000, 1, 1000], 'uV': [1/1000000, 1/1000, 1]} + voltage = pd.DataFrame(voltage) + voltage.index = ['V', 'mV', 'uV'] + + return voltage + +def capacity(): + # Define matrix for unit conversion for capacity + capacity = {'Ah': [1, 1000, 1000000], 'mAh': [1/1000, 1, 1000], 'uAh': [1/1000000, 1/1000, 1]} + capacity = pd.DataFrame(capacity) + capacity.index = ['Ah', 'mAh', 'uAh'] + + return capacity + +def mass(): + # Define matrix for unit conversion for capacity + mass = {'kg': [1, 1000, 1000000, 1000000000], 'g': [1/1000, 1, 1000, 1000000], 'mg': [1/1000000, 1/1000, 1, 1000], 'ug': [1/1000000000, 1/1000000, 1/1000, 1]} + mass = pd.DataFrame(mass) + mass.index = ['kg', 'g', 'mg', 'ug'] + + return mass + + +def energy(): + + energy = {'kWh': [1, 1000, 1000000], 'Wh': [1/1000, 1, 1000], 'mWh': [1/100000, 1/1000, 1]} + energy = pd.DataFrame(energy) + energy.index = ['kWh', 'Wh', 'mWh'] + + return energy + + + From 26bd7d8a15343b7aa46848465afc6fbdba19cdbe Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Oct 2021 14:02:13 +0200 Subject: [PATCH 043/355] Update io.py --- beamtime/electrochemistry/io.py | 141 ++++++++++++++++++++++++-------- 1 file changed, 106 insertions(+), 35 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 3e5bf02..0984581 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -77,26 +77,24 @@ def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=N old_units = get_old_units(df, kind='batsmall') df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='batsmall') - df.columns = ['TT', 'U', 'I', 'Z1', 'C', 'Comment'] - # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current - df[["Comment"]] = df[["Comment"]].fillna(value={'Comment': ''}) - df = df[df["Comment"].str.contains("program")==False] + df[["comment"]] = df[["comment"]].fillna(value={'comment': ''}) + df = df[df["comment"].str.contains("program")==False] # Creates masks for charge and discharge curves - chg_mask = df['I'] >= 0 - dchg_mask = df['I'] < 0 + chg_mask = df['current'] >= 0 + dchg_mask = df['current'] < 0 # Initiate cycles list cycles = [] # Loop through all the cycling steps, change the current and capacities in the - for i in range(df["Z1"].max()): + for i in range(df["count"].max()): - sub_df = df.loc[df['Z1'] == i].copy() + sub_df = df.loc[df['count'] == i].copy() - sub_df.loc[dchg_mask, 'I'] *= -1 - sub_df.loc[dchg_mask, 'C'] *= -1 + sub_df.loc[dchg_mask, 'current'] *= -1 + sub_df.loc[dchg_mask, 'capacity'] *= -1 chg_df = sub_df.loc[chg_mask] dchg_df = sub_df.loc[dchg_mask] @@ -113,27 +111,79 @@ def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=N return cycles -def process_neware_data(df, units=None, splice_cycles=None, active_material_weight=None, molecular_weight=None): +def process_neware_data(df, units=None, splice_cycles=None, active_material_weight=None, molecular_weight=None, reverse_discharge=False): - ######################### - #### UNIT CONVERSION #### - ######################### + """ Takes data from NEWARE in a DataFrame as read by read_neware() and converts units, adds columns and splits into cycles. + + Input: + df: pandas DataFrame containing NEWARE data as read by read_neware() + units: dictionary containing the desired units. keywords: capacity, current, voltage, mass, energy, time + splice_cycles: tuple containing index of cycles that should be spliced. Specifically designed to add two charge steps during the formation cycle with two different max voltages + active_materiale_weight: weight of the active material (in mg) used in the cell. + molecular_weight: the molar mass (in g mol^-1) of the active material, to calculate the number of ions extracted. Assumes one electron per Li+/Na+-ion """ # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. new_units = set_units(units=units) - old_units = get_old_units(df, kind='neware') + old_units = get_old_units(df=df, kind='neware') + + df = add_columns_neware(df=df, active_material_weight=active_material_weight, molecular_weight=molecular_weight, old_units=old_units) + df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='neware') - # if active_material_weight: - # df["SpecificCapacity(mAh/g)"] = df["Capacity(mAh)"] / (active_material_weight / 1000) + # Creates masks for charge and discharge curves + chg_mask = df['status'] == 'CC Chg' + dchg_mask = df['status'] == 'CC DChg' - # if molecular_weight: - # faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 - # seconds_per_hour = 3600 # s h^-1 - # f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + # Initiate cycles list + cycles = [] - # df["IonsExtracted"] = (df["SpecificCapacity(mAh/g)"]*molecular_weight)/f + # Loop through all the cycling steps, change the current and capacities in the + for i in range(df["cycle"].max()): + + sub_df = df.loc[df['cycle'] == i].copy() + + #sub_df.loc[dchg_mask, 'current'] *= -1 + #sub_df.loc[dchg_mask, 'capacity'] *= -1 + + chg_df = sub_df.loc[chg_mask] + dchg_df = sub_df.loc[dchg_mask] + + # Continue to next iteration if the charge and discharge DataFrames are empty (i.e. no current) + if chg_df.empty and dchg_df.empty: + continue + + if reverse_discharge: + max_capacity = dchg_df['capacity'].max() + dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) + + if 'specific_capacity' in df.columns: + max_capacity = dchg_df['specific_capacity'].max() + dchg_df['specific_capacity'] = np.abs(dchg_df['specific_capacity'] - max_capacity) + + if 'ions' in df.columns: + max_capacity = dchg_df['ions'].max() + dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) + + cycles.append((chg_df, dchg_df)) + + + + return cycles + + +def add_columns_neware(df, active_material_weight, molecular_weight, old_units): + + + if active_material_weight: + df["SpecificCapacity({}/mg)".format(old_units["capacity"])] = df["Capacity({})".format(old_units['capacity'])] / (active_material_weight) + + if molecular_weight: + faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 + seconds_per_hour = 3600 # s h^-1 + f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + + df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(old_units['capacity'])]*molecular_weight)*1000/f return df @@ -149,14 +199,34 @@ def unit_conversion(df, new_units, old_units, kind): df["I [{}]".format(old_units["current"])] = df["I [{}]".format(old_units["current"])] * unit_tables.current()[old_units["current"]].loc[new_units['current']] df["C [{}/{}]".format(old_units["capacity"], old_units["mass"])] = df["C [{}/{}]".format(old_units["capacity"], old_units["mass"])] * (unit_tables.capacity()[old_units["capacity"]].loc[new_units["capacity"]] / unit_tables.mass()[old_units["mass"]].loc[new_units["mass"]]) + df.columns = ['time', 'voltage', 'current', 'count', 'specific_capacity', 'comment'] + if kind == 'neware': df['Current({})'.format(old_units['current'])] = df['Current({})'.format(old_units['current'])] * unit_tables.current()[old_units['current']].loc[new_units['current']] df['Voltage({})'.format(old_units['voltage'])] = df['Voltage({})'.format(old_units['voltage'])] * unit_tables.voltage()[old_units['voltage']].loc[new_units['voltage']] df['Capacity({})'.format(old_units['capacity'])] = df['Capacity({})'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] df['Energy({})'.format(old_units['energy'])] = df['Energy({})'.format(old_units['energy'])] * unit_tables.energy()[old_units['energy']].loc[new_units['energy']] + df['CycleTime({})'.format(new_units['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=new_units['time']), axis=1) + df['RunTime({})'.format(new_units['time'])] = df.apply(lambda row : convert_datetime_string(row['Real Time(h:min:s.ms)'], reference=df['Real Time(h:min:s.ms)'].iloc[0], unit=new_units['time']), axis=1) + columns = ['status', 'jump', 'cycle', 'steps', 'current', 'voltage', 'capacity', 'energy'] - df['RelativeTime({})'.format(new_units['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=new_units['time']), axis=1) + if 'SpecificCapacity({}/mg)'.format(old_units['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] = df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] / unit_tables.mass()['mg'].loc[new_units["mass"]] + columns.append('specific_capacity') + + if 'IonsExtracted' in df.columns: + columns.append('ions') + + + + columns.append('cycle_time') + columns.append('run_time') + + + df.drop(['Record number', 'Relative Time(h:min:s.ms)', 'Real Time(h:min:s.ms)'], axis=1, inplace=True) + + df.columns = columns return df @@ -226,25 +296,26 @@ def convert_datetime_string(datetime_string, reference, unit='s'): from datetime import datetime # Parse the - cur_date, cur_time = datetime_string.split() - cur_y, cur_mo, cur_d = cur_date.split('-') - cur_h, cur_m, cur_s = cur_time.split(':') - cur_date = datetime(int(cur_y), int(cur_mo), int(cur_d), int(cur_h), int(cur_m), int(cur_s)) + current_date, current_time = datetime_string.split() + current_year, current_month, current_day = current_date.split('-') + current_hour, current_minute, current_second = current_time.split(':') + current_date = datetime(int(current_year), int(current_month), int(current_day), int(current_hour), int(current_minute), int(current_second)) - ref_date, ref_time = reference.split() - ref_y, ref_mo, ref_d = ref_date.split('-') - ref_h, ref_m, ref_s = ref_time.split(':') - ref_date = datetime(int(ref_y), int(ref_mo), int(ref_d), int(ref_h), int(ref_m), int(ref_s)) + reference_date, reference_time = reference.split() + reference_year, reference_month, reference_day = reference_date.split('-') + reference_hour, reference_minute, reference_second = reference_time.split(':') + reference_date = datetime(int(reference_year), int(reference_month), int(reference_day), int(reference_hour), int(reference_minute), int(reference_second)) - days = cur_date - ref_date + days = current_date - reference_date - s = days.seconds + + s = days.days*24*60*60 + days.seconds factors = {'ms': 1000, 's': 1, 'min': 1/(60), 'h': 1/(60*60)} - t = s * factors[unit] + time = s * factors[unit] - return t + return time def splice_cycles(first, second): From 4f255fd9d5c47d319d9508aa40f70560c6e7b071 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 12 Oct 2021 15:53:48 +0200 Subject: [PATCH 044/355] Add BioLogic-functions to io.py --- beamtime/electrochemistry/io.py | 154 +++++++++++++++++++++++++++++--- 1 file changed, 143 insertions(+), 11 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 0984581..6c6e5fc 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -21,7 +21,7 @@ def read_batsmall(path): -def read_neware(path, summary=False, active_material_weight=None, molecular_weight=None): +def read_neware(path, summary=False): ''' Reads electrochemistry data, currently only from the Neware battery cycler. Will convert to .csv if the filetype is .xlsx, which is the file format the Neware provides for the backup data. In this case it matters if summary is False or not. If file type is .csv, it will just open the datafile and it does not matter if summary is False or not.''' @@ -51,6 +51,29 @@ def read_neware(path, summary=False, active_material_weight=None, molecular_weig +def read_biologic(path): + ''' Reads Bio-Logic-data into a DataFrame. + + Input: + path (required): string with path to datafile + + Output: + df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' + + with open(path, 'r') as f: + lines = f.readlines() + + header_lines = int(lines[1].split()[-1]) - 1 + + + df = pd.read_csv(path, sep='\t', skiprows=header_lines) + df.dropna(inplace=True, axis=1) + + return df + + + + @@ -94,7 +117,7 @@ def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=N sub_df = df.loc[df['count'] == i].copy() sub_df.loc[dchg_mask, 'current'] *= -1 - sub_df.loc[dchg_mask, 'capacity'] *= -1 + sub_df.loc[dchg_mask, 'specific_capacity'] *= -1 chg_df = sub_df.loc[chg_mask] dchg_df = sub_df.loc[dchg_mask] @@ -126,7 +149,7 @@ def process_neware_data(df, units=None, splice_cycles=None, active_material_weig new_units = set_units(units=units) old_units = get_old_units(df=df, kind='neware') - df = add_columns_neware(df=df, active_material_weight=active_material_weight, molecular_weight=molecular_weight, old_units=old_units) + df = add_columns(df=df, active_material_weight=active_material_weight, molecular_weight=molecular_weight, old_units=old_units, kind='neware') df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='neware') @@ -172,19 +195,91 @@ def process_neware_data(df, units=None, splice_cycles=None, active_material_weig return cycles -def add_columns_neware(df, active_material_weight, molecular_weight, old_units): +def process_biologic_data(df, units=None, splice_cycles=None, active_material_weight=None, molecular_weight=None, reverse_discharge=False): + + # Pick out necessary columns + df = df[['Ns changes', 'Ns', 'time/s', 'Ewe/V', 'Energy charge/W.h', 'Energy discharge/W.h', '/mA', 'Capacity/mA.h', 'cycle number']].copy() + + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. + new_units = set_units(units=units) + old_units = get_old_units(df=df, kind='biologic') + + df = add_columns(df=df, active_material_weight=active_material_weight, molecular_weight=molecular_weight, old_units=old_units, kind='biologic') + + df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='biologic') - if active_material_weight: - df["SpecificCapacity({}/mg)".format(old_units["capacity"])] = df["Capacity({})".format(old_units['capacity'])] / (active_material_weight) + # Creates masks for charge and discharge curves + chg_mask = (df['status'] == 1) & (df['status_change'] != 1) + dchg_mask = (df['status'] == 2) & (df['status_change'] != 1) - if molecular_weight: - faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 - seconds_per_hour = 3600 # s h^-1 - f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 - df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(old_units['capacity'])]*molecular_weight)*1000/f + # Initiate cycles list + cycles = [] + + # Loop through all the cycling steps, change the current and capacities in the + for i in range(int(df["cycle"].max())): + + sub_df = df.loc[df['cycle'] == i].copy() + + #sub_df.loc[dchg_mask, 'current'] *= -1 + #sub_df.loc[dchg_mask, 'capacity'] *= -1 + + chg_df = sub_df.loc[chg_mask] + dchg_df = sub_df.loc[dchg_mask] + + # Continue to next iteration if the charge and discharge DataFrames are empty (i.e. no current) + if chg_df.empty and dchg_df.empty: + continue + + if reverse_discharge: + max_capacity = dchg_df['capacity'].max() + dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) + + if 'specific_capacity' in df.columns: + max_capacity = dchg_df['specific_capacity'].max() + dchg_df['specific_capacity'] = np.abs(dchg_df['specific_capacity'] - max_capacity) + + if 'ions' in df.columns: + max_capacity = dchg_df['ions'].max() + dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) + + cycles.append((chg_df, dchg_df)) + + + + return cycles + + +def add_columns(df, active_material_weight, molecular_weight, old_units, kind): + + if kind == 'neware': + if active_material_weight: + df["SpecificCapacity({}/mg)".format(old_units["capacity"])] = df["Capacity({})".format(old_units['capacity'])] / (active_material_weight) + + if molecular_weight: + faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 + seconds_per_hour = 3600 # s h^-1 + f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + + df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(old_units['capacity'])]*molecular_weight)*1000/f + + + if kind == 'biologic': + if active_material_weight: + + capacity = old_units['capacity'].split('h')[0] + '.h' + + + df["SpecificCapacity({}/mg)".format(old_units["capacity"])] = df["Capacity/{}".format(capacity)] / (active_material_weight) + + if molecular_weight: + faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 + seconds_per_hour = 3600 # s h^-1 + f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + + df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(old_units['capacity'])]*molecular_weight)*1000/f return df @@ -228,6 +323,25 @@ def unit_conversion(df, new_units, old_units, kind): df.columns = columns + if kind == 'biologic': + df['time/{}'.format(old_units['time'])] = df["time/{}".format(old_units["time"])] * unit_tables.time()[old_units["time"]].loc[new_units['time']] + df["Ewe/{}".format(old_units["voltage"])] = df["Ewe/{}".format(old_units["voltage"])] * unit_tables.voltage()[old_units["voltage"]].loc[new_units['voltage']] + df["/{}".format(old_units["current"])] = df["/{}".format(old_units["current"])] * unit_tables.current()[old_units["current"]].loc[new_units['current']] + + capacity = old_units['capacity'].split('h')[0] + '.h' + df["Capacity/{}".format(capacity)] = df["Capacity/{}".format(capacity)] * (unit_tables.capacity()[old_units["capacity"]].loc[new_units["capacity"]]) + + columns = ['status_change', 'status', 'time', 'voltage', 'energy_charge', 'energy_discharge', 'current', 'capacity', 'cycle'] + + if 'SpecificCapacity({}/mg)'.format(old_units['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] = df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] / unit_tables.mass()['mg'].loc[new_units["mass"]] + columns.append('specific_capacity') + + if 'IonsExtracted' in df.columns: + columns.append('ions') + + df.columns = columns + return df @@ -274,6 +388,24 @@ def get_old_units(df, kind): old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy} + if kind=='biologic': + + for column in df.columns: + if 'time' in column: + time = column.split('/')[-1] + elif 'Ewe' in column: + voltage = column.split('/')[-1] + elif 'Capacity' in column: + capacity = column.split('/')[-1].replace('.', '') + elif 'Energy' in column: + energy = column.split('/')[-1].replace('.', '') + elif '' in column: + current = column.split('/')[-1] + + old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy, 'time': time} + + + return old_units def convert_time_string(time_string, unit='ms'): From 43e6ef27c842a9e93152ad2a57f695c60afccb13 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 13 Oct 2021 18:06:56 +0200 Subject: [PATCH 045/355] Add plot functionality to electrochemistry --- beamtime/electrochemistry/io.py | 91 +++++++- beamtime/electrochemistry/plot.py | 375 ++++++++++++++++++++++++++++-- 2 files changed, 433 insertions(+), 33 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 6c6e5fc..35d271b 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -4,6 +4,22 @@ import matplotlib.pyplot as plt import os +def read_data(path, kind, options=None): + + if kind == 'neware': + df = read_neware(path) + cycles = process_neware_data(df, options=options) + + elif kind == 'batsmall': + df = read_batsmall(path) + cycles = process_batsmall_data(df=df, options=options) + + elif kind == 'biologic': + df = read_biologic(path) + cycles = process_biologic_data(df=df, options=options) + + return cycles + def read_batsmall(path): ''' Reads BATSMALL-data into a DataFrame. @@ -77,7 +93,7 @@ def read_biologic(path): -def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=None): +def process_batsmall_data(df, options=None): ''' Takes BATSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. Splits up into individual charge and discharge DataFrames per cycle, and outputs a list where each element is a tuple with the Chg and DChg-data. E.g. cycles[10][0] gives the charge data for the 11th cycle. @@ -94,12 +110,24 @@ def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=N cycles: A list with ''' + required_options = ['splice_cycles', 'molecular_weight', 'reverse_discharge', 'units'] + default_options = {'splice_cycles': None, 'molecular_weight': None, 'reverse_discharge': False, 'units': None} + + if not options: + options = default_options + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - new_units = set_units(units=units) + new_units = set_units(units=options['units']) old_units = get_old_units(df, kind='batsmall') df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='batsmall') + options['units'] = new_units + # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current df[["comment"]] = df[["comment"]].fillna(value={'comment': ''}) df = df[df["comment"].str.contains("program")==False] @@ -126,6 +154,18 @@ def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=N if chg_df.empty and dchg_df.empty: continue + if options['reverse_discharge']: + max_capacity = dchg_df['capacity'].max() + dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) + + if 'specific_capacity' in df.columns: + max_capacity = dchg_df['specific_capacity'].max() + dchg_df['specific_capacity'] = np.abs(dchg_df['specific_capacity'] - max_capacity) + + if 'ions' in df.columns: + max_capacity = dchg_df['ions'].max() + dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) + cycles.append((chg_df, dchg_df)) @@ -134,7 +174,7 @@ def process_batsmall_data(df, units=None, splice_cycles=None, molecular_weight=N return cycles -def process_neware_data(df, units=None, splice_cycles=None, active_material_weight=None, molecular_weight=None, reverse_discharge=False): +def process_neware_data(df, options=None): """ Takes data from NEWARE in a DataFrame as read by read_neware() and converts units, adds columns and splits into cycles. @@ -145,14 +185,27 @@ def process_neware_data(df, units=None, splice_cycles=None, active_material_weig active_materiale_weight: weight of the active material (in mg) used in the cell. molecular_weight: the molar mass (in g mol^-1) of the active material, to calculate the number of ions extracted. Assumes one electron per Li+/Na+-ion """ + required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles'] + default_options = {'units': None, 'active_material_weight': None, 'molecular_weight': None, 'reverse_discharge': False, 'splice_cycles': None} + + if not options: + options = default_options + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - new_units = set_units(units=units) + new_units = set_units(units=options['units']) old_units = get_old_units(df=df, kind='neware') - df = add_columns(df=df, active_material_weight=active_material_weight, molecular_weight=molecular_weight, old_units=old_units, kind='neware') + df = add_columns(df=df, active_material_weight=options['active_material_weight'], molecular_weight=options['molecular_weight'], old_units=old_units, kind='neware') df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='neware') + options['units'] = new_units + # Creates masks for charge and discharge curves chg_mask = df['status'] == 'CC Chg' @@ -176,7 +229,7 @@ def process_neware_data(df, units=None, splice_cycles=None, active_material_weig if chg_df.empty and dchg_df.empty: continue - if reverse_discharge: + if options['reverse_discharge']: max_capacity = dchg_df['capacity'].max() dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) @@ -195,19 +248,31 @@ def process_neware_data(df, units=None, splice_cycles=None, active_material_weig return cycles -def process_biologic_data(df, units=None, splice_cycles=None, active_material_weight=None, molecular_weight=None, reverse_discharge=False): +def process_biologic_data(df, options=None): + + required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles'] + default_options = {'units': None, 'active_material_weight': None, 'molecular_weight': None, 'reverse_discharge': False, 'splice_cycles': None} + + if not options: + options = default_options + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] # Pick out necessary columns df = df[['Ns changes', 'Ns', 'time/s', 'Ewe/V', 'Energy charge/W.h', 'Energy discharge/W.h', '/mA', 'Capacity/mA.h', 'cycle number']].copy() # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - new_units = set_units(units=units) + new_units = set_units(units=options['units']) old_units = get_old_units(df=df, kind='biologic') - df = add_columns(df=df, active_material_weight=active_material_weight, molecular_weight=molecular_weight, old_units=old_units, kind='biologic') + df = add_columns(df=df, active_material_weight=options['active_material_weight'], molecular_weight=options['molecular_weight'], old_units=old_units, kind='biologic') df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='biologic') + options['units'] = new_units + # Creates masks for charge and discharge curves chg_mask = (df['status'] == 1) & (df['status_change'] != 1) @@ -233,7 +298,7 @@ def process_biologic_data(df, units=None, splice_cycles=None, active_material_we if chg_df.empty and dchg_df.empty: continue - if reverse_discharge: + if options['reverse_discharge']: max_capacity = dchg_df['capacity'].max() dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) @@ -348,8 +413,8 @@ def unit_conversion(df, new_units, old_units, kind): def set_units(units=None): # Complete the list of units - if not all are passed, then default value will be used - required_units = ['time', 'current', 'voltage', 'capacity', 'mass', 'energy'] - default_units = {'time': 'h', 'current': 'mA', 'voltage': 'V', 'capacity': 'mAh', 'mass': 'g', 'energy': 'mWh'} + required_units = ['time', 'current', 'voltage', 'capacity', 'mass', 'energy', 'specific_capacity'] + default_units = {'time': 'h', 'current': 'mA', 'voltage': 'V', 'capacity': 'mAh', 'mass': 'g', 'energy': 'mWh', 'specific_capacity': None} if not units: units = default_units @@ -359,6 +424,8 @@ def set_units(units=None): if unit not in units.keys(): units[unit] = default_units[unit] + units['specific_capacity'] = r'{} {}'.format(units['capacity'], units['mass']) + '$^{-1}$' + return units diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py index 70f2b22..eabd105 100644 --- a/beamtime/electrochemistry/plot.py +++ b/beamtime/electrochemistry/plot.py @@ -1,40 +1,373 @@ import matplotlib.pyplot as plt +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) + import pandas as pd import numpy as np +import math + +import beamtime.electrochemistry as ec -def plot_gc(cycles, which_cycles='all', chg=True, dchg=True, colours=None, x='C', y='U'): +def plot_gc(path, kind, options=None): - fig, ax = prepare_gc_plot() + # Prepare plot, and read and process data + fig, ax = prepare_gc_plot(options=options) + cycles = ec.io.read_data(path=path, kind=kind, options=options) - if which_cycles == 'all': - which_cycles = [i for i, c in enumerate(cycles)] + # Update options + required_options = ['x_vals', 'y_vals', 'which_cycles', 'chg', 'dchg', 'colours', 'gradient'] + default_options = {'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', 'chg': True, 'dchg': True, 'colours': None, 'gradient': False} - if not colours: - chg_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B - dchg_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B + options = update_options(options=options, required_options=required_options, default_options=default_options) + + # Update list of cycles to correct indices + update_cycles_list(cycles=cycles, options=options) + + colours = generate_colours(cycles=cycles, options=options) + + print(len(options['which_cycles'])) + print(len(colours)) + + + + for i, cycle in enumerate(cycles): + if i in options['which_cycles']: + if options['chg']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + + if options['dchg']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + + + + fig, ax = prettify_gc_plot(fig=fig, ax=ax, options=options) + + return cycles, fig, ax + + +def update_options(options, required_options, default_options): + + if not options: + options = default_options + + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + return options + +def update_cycles_list(cycles, options): + + if not options: + options['which_cycles'] + + if options['which_cycles'] == 'all': + options['which_cycles'] = [i for i in range(len(cycles))] + + + elif type(options['which_cycles']) == list: + options['which_cycles'] = [i-1 for i in options['which_cycles']] + # Tuple is used to define an interval - as elements tuples can't be assigned, I convert it to a list here. + elif type(options['which_cycles']) == tuple: + which_cycles = list(options['which_cycles']) - for i, cycle in cycles: - if i in which_cycles: - if chg: - cycle[0].plot(ax=ax) + if which_cycles[0] <= 0: + which_cycles[0] = 1 + + elif which_cycles[1] < 0: + which_cycles[1] = len(cycles) + + + options['which_cycles'] = [i-1 for i in range(which_cycles[0], which_cycles[1]+1)] + + + return options + + +def prepare_gc_plot(options=None): + + + # First take care of the options for plotting - set any values not specified to the default values + required_options = ['columns', 'width', 'height', 'format', 'dpi', 'facecolor'] + default_options = {'columns': 1, 'width': 14, 'format': 'golden_ratio', 'dpi': None, 'facecolor': 'w'} + + # If none are set at all, just pass the default_options + if not options: + options = default_options + options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 + options['figsize'] = (options['width'], options['height']) + + + # If options is passed, go through to fill out the rest. + else: + # Start by setting the width: + if 'width' not in options.keys(): + options['width'] = default_options['width'] + + # Then set height - check options for format. If not given, set the height to the width scaled by the golden ratio - if the format is square, set the same. This should possibly allow for the tweaking of custom ratios later. + if 'height' not in options.keys(): + if 'format' not in options.keys(): + options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 + elif options['format'] == 'square': + options['height'] = options['width'] + + options['figsize'] = (options['width'], options['height']) + + # After height and width are set, go through the rest of the options to make sure that all the required options are filled + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + fig, ax = plt.subplots(figsize=(options['figsize']), dpi=options['dpi'], facecolor=options['facecolor']) + + linewidth = 1*options['columns'] + axeswidth = 3*options['columns'] + + plt.rc('lines', linewidth=linewidth) + plt.rc('axes', linewidth=axeswidth) + + return fig, ax + + +def prettify_gc_plot(fig, ax, options=None): + + + ################################################################## + ######################### UPDATE OPTIONS ######################### + ################################################################## + + # Define the required options + required_options = [ + 'columns', + 'xticks', 'yticks', + 'show_major_ticks', + 'show_minor_ticks', + 'xlim', 'ylim', + 'hide_x_axis', 'hide_y_axis', + 'x_vals', 'y_vals', + 'xlabel', 'ylabel', + 'units', 'sizes', + 'title' + ] + + + # Define the default options + default_options = { + 'columns': 1, + 'xticks': None, + 'yticks': None, + 'show_major_ticks': [True, True, True, True], + 'show_minor_ticks': [True, True, True, True], + 'xlim': None, + 'ylim': None, + 'hide_x_axis': False, + 'hide_y_axis': False, + 'x_vals': 'specific_capacity', + 'y_vals': 'voltage', + 'xlabel': None, + 'ylabel': None, + 'units': None, + 'sizes': None, + 'title': None + } + + update_options(options, required_options, default_options) + + + ################################################################## + ########################## DEFINE SIZES ########################## + ################################################################## + + # Define the required sizes + required_sizes = [ + 'labels', + 'legend', + 'title', + 'line', 'axes', + 'tick_labels', + 'major_ticks', 'minor_ticks'] + + + + # Define default sizes + default_sizes = { + 'labels': 30*options['columns'], + 'legend': 30*options['columns'], + 'title': 30*options['columns'], + 'line': 3*options['columns'], + 'axes': 3*options['columns'], + 'tick_labels': 30*options['columns'], + 'major_ticks': 20*options['columns'], + 'minor_ticks': 10*options['columns'] + } + + # Initialise dictionary if it doesn't exist + if not options['sizes']: + options['sizes'] = {} + + + # Update dictionary with default values where none is supplied + for size in required_sizes: + if size not in options['sizes']: + options['sizes'][size] = default_sizes[size] + + + ################################################################## + ########################## AXIS LABELS ########################### + ################################################################## + + + if not options['xlabel']: + print(options['x_vals']) + print(options['units']) + options['xlabel'] = prettify_labels(options['x_vals']) + ' [{}]'.format(options['units'][options['x_vals']]) + + else: + options['xlabel'] = options['xlabel'] + ' [{}]'.format(options['units'][options['x_vals']]) + + + if not options['ylabel']: + options['ylabel'] = prettify_labels(options['y_vals']) + ' [{}]'.format(options['units'][options['y_vals']]) + + else: + options['ylabel'] = options['ylabel'] + ' [{}]'.format(options['units'][options['y_vals']]) + + ax.set_xlabel(options['xlabel'], size=options['sizes']['labels']) + ax.set_ylabel(options['ylabel'], size=options['sizes']['labels']) + + ################################################################## + ###################### TICK MARKS & LABELS ####################### + ################################################################## + + ax.tick_params(direction='in', which='major', bottom=options['show_major_ticks'][0], left=options['show_major_ticks'][1], top=options['show_major_ticks'][2], right=options['show_major_ticks'][0], length=options['sizes']['major_ticks'], width=options['sizes']['axes']) + ax.tick_params(direction='in', which='minor', bottom=options['show_minor_ticks'][0], left=options['show_minor_ticks'][1], top=options['show_minor_ticks'][2], right=options['show_minor_ticks'][0], length=options['sizes']['minor_ticks'], width=options['sizes']['axes']) + # DEFINE AND SET TICK DISTANCES + + default_ticks = { + 'specific_capacity': [100, 50], + 'capacity': [0.1, 0.05], + 'voltage': [0.5, 0.25] + } + + + + # Set default tick distances for x-axis if not specified + if not options['xticks']: + + major_xtick = default_ticks[options['x_vals']][0] + minor_xtick = default_ticks[options['x_vals']][1] + + # Otherwise apply user input + else: + major_xtick = options['xticks'][0] + minor_xtick = options['xticks'][1] + + + # Set default tick distances for x-axis if not specified + if not options['yticks']: + + major_ytick = default_ticks[options['y_vals']][0] + minor_ytick = default_ticks[options['y_vals']][1] + + # Otherwise apply user input + else: + major_xtick = options['yticks'][0] + minor_xtick = options['yticks'][1] + + + # Apply values + ax.xaxis.set_major_locator(MultipleLocator(major_xtick)) + ax.xaxis.set_minor_locator(MultipleLocator(minor_xtick)) + + ax.yaxis.set_major_locator(MultipleLocator(major_ytick)) + ax.yaxis.set_minor_locator(MultipleLocator(minor_ytick)) + + + # SET FONTSIZE OF TICK LABELS + + plt.xticks(fontsize=options['sizes']['tick_labels']) + plt.yticks(fontsize=options['sizes']['tick_labels']) + + ################################################################## + ############################# TITLE ############################## + ################################################################## + + if options['title']: + ax.set_title(options['title'], size=options['sizes']['title']) + + ################################################################## + ############################# LEGEND ############################# + ################################################################## + + ax.get_legend().remove() + + return fig, ax - - - - - - -def prepare_gc_plot(figsize=(14,7), dpi=None): - - fig, ax = plt.subplots(figsize=figsize, dpi=dpi) +def prettify_labels(label): + labels_dict = { + 'capacity': 'Capacity', + 'specific_capacity': 'Specific capacity', + 'voltage': 'Voltage', + 'current': 'Current', + 'energy': 'Energy', + } - return fig, ax \ No newline at end of file + return labels_dict[label] + + + + + + + + +def generate_colours(cycles, options): + + # Assign colours from the options dictionary if it is defined, otherwise use standard colours. + if options['colours']: + charge_colour = options['colours'][0] + discharge_colour = options['colours'][1] + + else: + charge_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B, coolors.co + discharge_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B, coolors.co + + + + # If gradient is enabled, find start and end points for each colour + if options['gradient']: + + add_charge = min([(1-x)*0.75 for x in charge_colour]) + add_discharge = min([(1-x)*0.75 for x in discharge_colour]) + + charge_colour_start = charge_colour + charge_colour_end = [x+add_charge for x in charge_colour] + + discharge_colour_start = discharge_colour + discharge_colour_end = [x+add_discharge for x in discharge_colour] + + + + # Generate lists of colours + colours = [] + + for cycle_number in range(0, len(cycles)): + if options['gradient']: + weight_start = (len(cycles) - cycle_number)/len(cycles) + weight_end = cycle_number/len(cycles) + + charge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(charge_colour_start, charge_colour_end)] + discharge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(discharge_colour_start, discharge_colour_end)] + + colours.append([charge_colour, discharge_colour]) + + return colours From 3f3486049b415914c59895c32a1359ad8d757f77 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 14 Oct 2021 14:18:39 +0200 Subject: [PATCH 046/355] working on split_xanes_scan --- beamtime/xanes/calib.py | 13 ++++++++++++- beamtime/xanes/io.py | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 6817b69..2ef6c4d 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -5,9 +5,20 @@ import os def rbkerbest(): print("ROSENBORG!<3") +#def split_xanes_scan(filename, destination=None): -def split_xanes_scan(filename, destination=None): + # with open(filename, 'r') as f: + +##Better to make a new function that loops through the files, and performing the split_xanes_scan on + + +def split_xanes_scan(root, destination=None, replace=False): + #root is the path to the beamtime-folder + #destination should be the path to the processed data + + #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw + with open(filename, 'r') as f: lines = f.readlines() diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index d08b5c1..a818d86 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -1 +1,2 @@ -#hello \ No newline at end of file +#hello +#yeah \ No newline at end of file From be7d153a9deb957beef981c3932c70ec1568ecf1 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 14 Oct 2021 15:10:37 +0200 Subject: [PATCH 047/355] Update options --- beamtime/electrochemistry/io.py | 2 +- beamtime/electrochemistry/plot.py | 63 +++++++++++++++++++------------ 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 35d271b..510be04 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -381,7 +381,7 @@ def unit_conversion(df, new_units, old_units, kind): columns.append('cycle_time') - columns.append('run_time') + columns.append('time') df.drop(['Record number', 'Relative Time(h:min:s.ms)', 'Real Time(h:min:s.ms)'], axis=1, inplace=True) diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py index eabd105..13bb1e1 100644 --- a/beamtime/electrochemistry/plot.py +++ b/beamtime/electrochemistry/plot.py @@ -16,8 +16,8 @@ def plot_gc(path, kind, options=None): # Update options - required_options = ['x_vals', 'y_vals', 'which_cycles', 'chg', 'dchg', 'colours', 'gradient'] - default_options = {'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', 'chg': True, 'dchg': True, 'colours': None, 'gradient': False} + required_options = ['x_vals', 'y_vals', 'which_cycles', 'chg', 'dchg', 'colours', 'differentiate_charge_discharge', 'gradient'] + default_options = {'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', 'chg': True, 'dchg': True, 'colours': None, 'differentiate_charge_discharge': True, 'gradient': False} options = update_options(options=options, required_options=required_options, default_options=default_options) @@ -26,10 +26,6 @@ def plot_gc(path, kind, options=None): colours = generate_colours(cycles=cycles, options=options) - print(len(options['which_cycles'])) - print(len(colours)) - - for i, cycle in enumerate(cycles): if i in options['which_cycles']: @@ -144,10 +140,10 @@ def prettify_gc_plot(fig, ax, options=None): required_options = [ 'columns', 'xticks', 'yticks', - 'show_major_ticks', - 'show_minor_ticks', + 'show_major_ticks', 'show_minor_ticks', 'xlim', 'ylim', 'hide_x_axis', 'hide_y_axis', + 'positions', 'x_vals', 'y_vals', 'xlabel', 'ylabel', 'units', 'sizes', @@ -158,18 +154,13 @@ def prettify_gc_plot(fig, ax, options=None): # Define the default options default_options = { 'columns': 1, - 'xticks': None, - 'yticks': None, - 'show_major_ticks': [True, True, True, True], - 'show_minor_ticks': [True, True, True, True], - 'xlim': None, - 'ylim': None, - 'hide_x_axis': False, - 'hide_y_axis': False, - 'x_vals': 'specific_capacity', - 'y_vals': 'voltage', - 'xlabel': None, - 'ylabel': None, + 'xticks': None, 'yticks': None, + 'show_major_ticks': [True, True, True, True], 'show_minor_ticks': [True, True, True, True], + 'xlim': None,'ylim': None, + 'hide_x_axis': False, 'hide_y_axis': False, + 'positions': {'xaxis': 'bottom', 'yaxis': 'left'}, + 'x_vals': 'specific_capacity', 'y_vals': 'voltage', + 'xlabel': None, 'ylabel': None, 'units': None, 'sizes': None, 'title': None @@ -222,8 +213,6 @@ def prettify_gc_plot(fig, ax, options=None): if not options['xlabel']: - print(options['x_vals']) - print(options['units']) options['xlabel'] = prettify_labels(options['x_vals']) + ' [{}]'.format(options['units'][options['x_vals']]) else: @@ -250,13 +239,21 @@ def prettify_gc_plot(fig, ax, options=None): # DEFINE AND SET TICK DISTANCES + from . import unit_tables + + # Define default ticks and scale to desired units default_ticks = { - 'specific_capacity': [100, 50], - 'capacity': [0.1, 0.05], - 'voltage': [0.5, 0.25] + 'specific_capacity': [100 * (unit_tables.capacity()['mAh'].loc[options['units']['capacity']] / unit_tables.mass()['g'].loc[options['units']['mass']]), 50 * (unit_tables.capacity()['mAh'].loc[options['units']['capacity']] / unit_tables.mass()['g'].loc[options['units']['mass']])], + 'capacity': [0.1 * (unit_tables.capacity()['mAh'].loc[options['units']['capacity']]), 0.05 * (unit_tables.capacity()['mAh'].loc[options['units']['capacity']])], + 'voltage': [0.5 * (unit_tables.voltage()['V'].loc[options['units']['voltage']]), 0.25 * (unit_tables.voltage()['V'].loc[options['units']['voltage']])], + 'time': [10 * (unit_tables.time()['h'].loc[options['units']['time']]), 5 * (unit_tables.time()['h'].loc[options['units']['time']])] } + if options['positions']['yaxis'] == 'right': + ax.yaxis.set_label_position("right") + ax.yaxis.tick_right() + # Set default tick distances for x-axis if not specified if not options['xticks']: @@ -290,11 +287,23 @@ def prettify_gc_plot(fig, ax, options=None): ax.yaxis.set_minor_locator(MultipleLocator(minor_ytick)) + + # SET FONTSIZE OF TICK LABELS plt.xticks(fontsize=options['sizes']['tick_labels']) plt.yticks(fontsize=options['sizes']['tick_labels']) + ################################################################## + ########################## AXES LIMITS ########################### + ################################################################## + + if options['xlim']: + plt.xlim(options['xlim']) + + if options['ylim']: + plt.ylim(options['ylim']) + ################################################################## ############################# TITLE ############################## ################################################################## @@ -319,6 +328,7 @@ def prettify_labels(label): 'voltage': 'Voltage', 'current': 'Current', 'energy': 'Energy', + 'time': 'Time' } return labels_dict[label] @@ -341,6 +351,9 @@ def generate_colours(cycles, options): charge_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B, coolors.co discharge_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B, coolors.co + if not options['differentiate_charge_discharge']: + discharge_colour = charge_colour + # If gradient is enabled, find start and end points for each colour From b66b7d8ea0bbd68ff2788cad50cccf59f2ec8dea Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 14 Oct 2021 15:10:55 +0200 Subject: [PATCH 048/355] Add requirements --- beamtime/requirements.txt | 107 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 beamtime/requirements.txt diff --git a/beamtime/requirements.txt b/beamtime/requirements.txt new file mode 100644 index 0000000..3f449e8 --- /dev/null +++ b/beamtime/requirements.txt @@ -0,0 +1,107 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: win-64 +backcall=0.2.0=py_0 +beamtime=0.1=pypi_0 +blas=1.0=mkl +bottleneck=1.3.2=py39h7cc1a96_1 +ca-certificates=2021.10.8=h5b45459_0 +cached-property=1.5.2=hd8ed1ab_1 +cached_property=1.5.2=pyha770c72_1 +certifi=2021.10.8=py39hcbf5309_0 +colorama=0.4.4=py_0 +cycler=0.10.0=py_2 +debugpy=1.4.1=py39hd77b12b_0 +decorator=4.4.2=py_0 +fabio=0.12.0=py39h5d4886f_0 +freetype=2.10.4=h546665d_1 +glymur=0.9.4=pyhd8ed1ab_0 +h5py=3.2.1=nompi_py39hf27771d_100 +hdf5=1.10.6=nompi_h5268f04_1114 +hdf5plugin=3.1.1=py39h71586dd_0 +icc_rt=2019.0.0=h0cc432a_1 +icu=68.1=h0e60522_0 +intel-openmp=2021.3.0=haa95532_3372 +ipykernel=6.4.1=py39haa95532_1 +ipython=7.27.0=py39hd4e2768_0 +ipython_genutils=0.2.0=pyhd3eb1b0_1 +jbig=2.1=h8d14728_2003 +jedi=0.18.0=py39haa95532_1 +jpeg=9d=h8ffe710_0 +jupyter_client=6.1.7=py_0 +jupyter_core=4.8.1=py39haa95532_0 +kiwisolver=1.3.2=py39h2e07f2f_0 +krb5=1.19.2=hbae68bd_2 +lcms2=2.12=h2a16943_0 +lerc=3.0=h0e60522_0 +libclang=11.1.0=default_h5c34c98_1 +libcurl=7.79.1=h789b8ee_1 +libdeflate=1.8=h8ffe710_0 +libiconv=1.16=he774522_0 +libpng=1.6.37=h1d00b33_2 +libssh2=1.10.0=h680486a_2 +libtiff=4.3.0=hd413186_2 +libxml2=2.9.12=hf5bbc77_0 +libxslt=1.1.33=h65864e5_2 +libzlib=1.2.11=h8ffe710_1013 +lxml=4.6.3=py39h4fd7cdf_0 +lz4-c=1.9.3=h8ffe710_1 +mako=1.1.5=pyhd8ed1ab_0 +markupsafe=2.0.1=py39hb82d6ee_0 +matplotlib=3.4.3=py39hcbf5309_1 +matplotlib-base=3.4.3=py39h581301d_1 +matplotlib-inline=0.1.2=pyhd3eb1b0_2 +mkl=2021.3.0=haa95532_524 +mkl-service=2.4.0=py39h2bbff1b_0 +mkl_fft=1.3.0=py39h277e83a_2 +mkl_random=1.2.2=py39hf11a4ad_0 +numexpr=2.7.3=py39hb80d3ca_1 +numpy=1.21.2=py39hfca59bb_0 +numpy-base=1.21.2=py39h0829f74_0 +olefile=0.46=pyh9f0ad1d_1 +openjpeg=2.4.0=hb211442_1 +openssl=1.1.1l=h8ffe710_0 +pandas=1.3.3=py39h6214cd6_0 +parso=0.8.0=py_0 +pickleshare=0.7.5=pyhd3eb1b0_1003 +pillow=8.3.2=py39h916092e_0 +pip=21.2.4=py39haa95532_0 +prompt-toolkit=3.0.8=py_0 +pyfai=0.20.0=hd8ed1ab_0 +pyfai-base=0.20.0=py39h2e25243_0 +pygments=2.7.1=py_0 +pyparsing=2.4.7=pyh9f0ad1d_0 +pyqt=5.12.3=py39hcbf5309_7 +pyqt-impl=5.12.3=py39h415ef7b_7 +pyqt5-sip=4.19.18=py39h415ef7b_7 +pyqtchart=5.12=py39h415ef7b_7 +pyqtwebengine=5.12.1=py39h415ef7b_7 +pyreadline=2.1=py39hcbf5309_1004 +python=3.9.7=h6244533_1 +python-dateutil=2.8.2=pyhd3eb1b0_0 +python_abi=3.9=2_cp39 +pytz=2021.3=pyhd3eb1b0_0 +pywin32=228=py39hbaba5e8_1 +pyzmq=22.2.1=py39hd77b12b_1 +qt=5.12.9=h5909a2a_4 +qtconsole=5.1.1=pyhd8ed1ab_0 +qtpy=1.11.2=pyhd8ed1ab_0 +scipy=1.7.1=py39hbe87c03_2 +setuptools=58.0.4=py39haa95532_0 +silx=0.15.2=hd8ed1ab_0 +silx-base=0.15.2=py39h2e25243_0 +six=1.16.0=pyhd3eb1b0_0 +sqlite=3.36.0=h2bbff1b_0 +tk=8.6.11=h8ffe710_1 +tornado=6.1=py39h2bbff1b_0 +traitlets=5.0.5=py_0 +tzdata=2021a=h5d7bf9c_0 +vc=14.2=h21ff451_1 +vs2015_runtime=14.27.29016=h5e58377_2 +wcwidth=0.2.5=py_0 +wheel=0.37.0=pyhd3eb1b0_1 +wincertstore=0.2=py39haa95532_2 +xlsx2csv=0.7.8=pypi_0 +xz=5.2.5=h62dcd97_1 +zlib=1.2.11=h8ffe710_1013 +zstd=1.5.0=h6255e5f_0 From d2b1c213c4386b6ab84c4641a9bb3ec3b162b1ce Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 21 Oct 2021 14:37:50 +0200 Subject: [PATCH 049/355] Update requirements with --no-builds prefix --- beamtime/requirements.txt | 220 ++++++++++++++++++++------------------ 1 file changed, 113 insertions(+), 107 deletions(-) diff --git a/beamtime/requirements.txt b/beamtime/requirements.txt index 3f449e8..da79e27 100644 --- a/beamtime/requirements.txt +++ b/beamtime/requirements.txt @@ -1,107 +1,113 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: win-64 -backcall=0.2.0=py_0 -beamtime=0.1=pypi_0 -blas=1.0=mkl -bottleneck=1.3.2=py39h7cc1a96_1 -ca-certificates=2021.10.8=h5b45459_0 -cached-property=1.5.2=hd8ed1ab_1 -cached_property=1.5.2=pyha770c72_1 -certifi=2021.10.8=py39hcbf5309_0 -colorama=0.4.4=py_0 -cycler=0.10.0=py_2 -debugpy=1.4.1=py39hd77b12b_0 -decorator=4.4.2=py_0 -fabio=0.12.0=py39h5d4886f_0 -freetype=2.10.4=h546665d_1 -glymur=0.9.4=pyhd8ed1ab_0 -h5py=3.2.1=nompi_py39hf27771d_100 -hdf5=1.10.6=nompi_h5268f04_1114 -hdf5plugin=3.1.1=py39h71586dd_0 -icc_rt=2019.0.0=h0cc432a_1 -icu=68.1=h0e60522_0 -intel-openmp=2021.3.0=haa95532_3372 -ipykernel=6.4.1=py39haa95532_1 -ipython=7.27.0=py39hd4e2768_0 -ipython_genutils=0.2.0=pyhd3eb1b0_1 -jbig=2.1=h8d14728_2003 -jedi=0.18.0=py39haa95532_1 -jpeg=9d=h8ffe710_0 -jupyter_client=6.1.7=py_0 -jupyter_core=4.8.1=py39haa95532_0 -kiwisolver=1.3.2=py39h2e07f2f_0 -krb5=1.19.2=hbae68bd_2 -lcms2=2.12=h2a16943_0 -lerc=3.0=h0e60522_0 -libclang=11.1.0=default_h5c34c98_1 -libcurl=7.79.1=h789b8ee_1 -libdeflate=1.8=h8ffe710_0 -libiconv=1.16=he774522_0 -libpng=1.6.37=h1d00b33_2 -libssh2=1.10.0=h680486a_2 -libtiff=4.3.0=hd413186_2 -libxml2=2.9.12=hf5bbc77_0 -libxslt=1.1.33=h65864e5_2 -libzlib=1.2.11=h8ffe710_1013 -lxml=4.6.3=py39h4fd7cdf_0 -lz4-c=1.9.3=h8ffe710_1 -mako=1.1.5=pyhd8ed1ab_0 -markupsafe=2.0.1=py39hb82d6ee_0 -matplotlib=3.4.3=py39hcbf5309_1 -matplotlib-base=3.4.3=py39h581301d_1 -matplotlib-inline=0.1.2=pyhd3eb1b0_2 -mkl=2021.3.0=haa95532_524 -mkl-service=2.4.0=py39h2bbff1b_0 -mkl_fft=1.3.0=py39h277e83a_2 -mkl_random=1.2.2=py39hf11a4ad_0 -numexpr=2.7.3=py39hb80d3ca_1 -numpy=1.21.2=py39hfca59bb_0 -numpy-base=1.21.2=py39h0829f74_0 -olefile=0.46=pyh9f0ad1d_1 -openjpeg=2.4.0=hb211442_1 -openssl=1.1.1l=h8ffe710_0 -pandas=1.3.3=py39h6214cd6_0 -parso=0.8.0=py_0 -pickleshare=0.7.5=pyhd3eb1b0_1003 -pillow=8.3.2=py39h916092e_0 -pip=21.2.4=py39haa95532_0 -prompt-toolkit=3.0.8=py_0 -pyfai=0.20.0=hd8ed1ab_0 -pyfai-base=0.20.0=py39h2e25243_0 -pygments=2.7.1=py_0 -pyparsing=2.4.7=pyh9f0ad1d_0 -pyqt=5.12.3=py39hcbf5309_7 -pyqt-impl=5.12.3=py39h415ef7b_7 -pyqt5-sip=4.19.18=py39h415ef7b_7 -pyqtchart=5.12=py39h415ef7b_7 -pyqtwebengine=5.12.1=py39h415ef7b_7 -pyreadline=2.1=py39hcbf5309_1004 -python=3.9.7=h6244533_1 -python-dateutil=2.8.2=pyhd3eb1b0_0 -python_abi=3.9=2_cp39 -pytz=2021.3=pyhd3eb1b0_0 -pywin32=228=py39hbaba5e8_1 -pyzmq=22.2.1=py39hd77b12b_1 -qt=5.12.9=h5909a2a_4 -qtconsole=5.1.1=pyhd8ed1ab_0 -qtpy=1.11.2=pyhd8ed1ab_0 -scipy=1.7.1=py39hbe87c03_2 -setuptools=58.0.4=py39haa95532_0 -silx=0.15.2=hd8ed1ab_0 -silx-base=0.15.2=py39h2e25243_0 -six=1.16.0=pyhd3eb1b0_0 -sqlite=3.36.0=h2bbff1b_0 -tk=8.6.11=h8ffe710_1 -tornado=6.1=py39h2bbff1b_0 -traitlets=5.0.5=py_0 -tzdata=2021a=h5d7bf9c_0 -vc=14.2=h21ff451_1 -vs2015_runtime=14.27.29016=h5e58377_2 -wcwidth=0.2.5=py_0 -wheel=0.37.0=pyhd3eb1b0_1 -wincertstore=0.2=py39haa95532_2 -xlsx2csv=0.7.8=pypi_0 -xz=5.2.5=h62dcd97_1 -zlib=1.2.11=h8ffe710_1013 -zstd=1.5.0=h6255e5f_0 +name: beamtime +channels: + - conda-forge + - anaconda + - diffpy + - defaults +dependencies: + - backcall=0.2.0 + - blas=1.0 + - bottleneck=1.3.2 + - ca-certificates=2021.10.8 + - cached-property=1.5.2 + - cached_property=1.5.2 + - certifi=2021.10.8 + - colorama=0.4.4 + - cycler=0.10.0 + - debugpy=1.4.1 + - decorator=4.4.2 + - fabio=0.12.0 + - freetype=2.10.4 + - glymur=0.9.4 + - h5py=3.2.1 + - hdf5=1.10.6 + - hdf5plugin=3.1.1 + - icc_rt=2019.0.0 + - icu=68.1 + - intel-openmp=2021.3.0 + - ipykernel=6.4.1 + - ipython=7.27.0 + - ipython_genutils=0.2.0 + - jbig=2.1 + - jedi=0.18.0 + - jpeg=9d + - jupyter_client=6.1.7 + - jupyter_core=4.8.1 + - kiwisolver=1.3.2 + - krb5=1.19.2 + - lcms2=2.12 + - lerc=3.0 + - libclang=11.1.0 + - libcurl=7.79.1 + - libdeflate=1.8 + - libiconv=1.16 + - libpng=1.6.37 + - libssh2=1.10.0 + - libtiff=4.3.0 + - libxml2=2.9.12 + - libxslt=1.1.33 + - libzlib=1.2.11 + - lxml=4.6.3 + - lz4-c=1.9.3 + - mako=1.1.5 + - markupsafe=2.0.1 + - matplotlib=3.4.3 + - matplotlib-base=3.4.3 + - matplotlib-inline=0.1.2 + - mkl=2021.3.0 + - mkl-service=2.4.0 + - mkl_fft=1.3.0 + - mkl_random=1.2.2 + - numexpr=2.7.3 + - numpy=1.21.2 + - numpy-base=1.21.2 + - olefile=0.46 + - openjpeg=2.4.0 + - openssl=1.1.1l + - pandas=1.3.3 + - parso=0.8.0 + - pickleshare=0.7.5 + - pillow=8.3.2 + - pip=21.2.4 + - prompt-toolkit=3.0.8 + - pyfai=0.20.0 + - pyfai-base=0.20.0 + - pygments=2.7.1 + - pyparsing=2.4.7 + - pyqt=5.12.3 + - pyqt-impl=5.12.3 + - pyqt5-sip=4.19.18 + - pyqtchart=5.12 + - pyqtwebengine=5.12.1 + - pyreadline=2.1 + - python=3.9.7 + - python-dateutil=2.8.2 + - python_abi=3.9 + - pytz=2021.3 + - pywin32=228 + - pyzmq=22.2.1 + - qt=5.12.9 + - qtconsole=5.1.1 + - qtpy=1.11.2 + - scipy=1.7.1 + - setuptools=58.0.4 + - silx=0.15.2 + - silx-base=0.15.2 + - six=1.16.0 + - sqlite=3.36.0 + - tk=8.6.11 + - tornado=6.1 + - traitlets=5.0.5 + - tzdata=2021a + - vc=14.2 + - vs2015_runtime=14.27.29016 + - wcwidth=0.2.5 + - wheel=0.37.0 + - wincertstore=0.2 + - xz=5.2.5 + - zlib=1.2.11 + - zstd=1.5.0 + - pip: + - beamtime==0.1 + - xlsx2csv==0.7.8 +prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime From 11a344c98555065897910ebf46d73720e33372aa Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 21 Oct 2021 14:41:10 +0200 Subject: [PATCH 050/355] Add xrd-functions --- beamtime/xrd/__init__.py | 1 + beamtime/xrd/io.py | 54 ++++++++++++++++++++++++++++++++++++++++ beamtime/xrd/plot.py | 2 ++ 3 files changed, 57 insertions(+) create mode 100644 beamtime/xrd/io.py create mode 100644 beamtime/xrd/plot.py diff --git a/beamtime/xrd/__init__.py b/beamtime/xrd/__init__.py index e69de29..e0e052c 100644 --- a/beamtime/xrd/__init__.py +++ b/beamtime/xrd/__init__.py @@ -0,0 +1 @@ +from . import io, plot \ No newline at end of file diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py new file mode 100644 index 0000000..b66dcde --- /dev/null +++ b/beamtime/xrd/io.py @@ -0,0 +1,54 @@ +import fabio, pyFAI +import numpy as np +import os + + +def get_image_array(path): + + image = fabio.open(path) + image_array = image.data + + return image_array + + +def integrate_1d(path, calibrant, bins, options): + + required_options = ['unit', 'extension'] + + default_options = {'unit': '2th_deg', 'extension': '_integrated.dat'} + + if not options: + options = default_options + + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + + image = get_image_array(path) + ai = pyFAI.load(calibrant) + + filename = os.path.split(path)[-1].split('.')[0] + options['extension'] + + res = ai.integrate1d(image, bins, unit=options['unit'], filename=filename) + + + + + +def view_integrator(calibrant): + ''' Prints out information about the azimuthal integrator + + Input: + calibrant: Path to the azimuthal integrator file (.PONI) + + Output: + None''' + + ai = pyFAI.load(calibrant) + + print("pyFAI version:", pyFAI.version) + print("\nIntegrator: \n", ai) + + diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py new file mode 100644 index 0000000..e37c7b8 --- /dev/null +++ b/beamtime/xrd/plot.py @@ -0,0 +1,2 @@ +import matplotlib.pyplot as plt +import numpy From 2f8f87c8f0b054269d8e37d92a486ad77ad15ff3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 21 Oct 2021 15:03:41 +0200 Subject: [PATCH 051/355] Update requirements file with --from-history tag --- beamtime/requirements.txt | 110 ++------------------------------------ 1 file changed, 3 insertions(+), 107 deletions(-) diff --git a/beamtime/requirements.txt b/beamtime/requirements.txt index da79e27..6b5d7f9 100644 --- a/beamtime/requirements.txt +++ b/beamtime/requirements.txt @@ -1,113 +1,9 @@ name: beamtime channels: - - conda-forge - - anaconda - diffpy - defaults dependencies: - - backcall=0.2.0 - - blas=1.0 - - bottleneck=1.3.2 - - ca-certificates=2021.10.8 - - cached-property=1.5.2 - - cached_property=1.5.2 - - certifi=2021.10.8 - - colorama=0.4.4 - - cycler=0.10.0 - - debugpy=1.4.1 - - decorator=4.4.2 - - fabio=0.12.0 - - freetype=2.10.4 - - glymur=0.9.4 - - h5py=3.2.1 - - hdf5=1.10.6 - - hdf5plugin=3.1.1 - - icc_rt=2019.0.0 - - icu=68.1 - - intel-openmp=2021.3.0 - - ipykernel=6.4.1 - - ipython=7.27.0 - - ipython_genutils=0.2.0 - - jbig=2.1 - - jedi=0.18.0 - - jpeg=9d - - jupyter_client=6.1.7 - - jupyter_core=4.8.1 - - kiwisolver=1.3.2 - - krb5=1.19.2 - - lcms2=2.12 - - lerc=3.0 - - libclang=11.1.0 - - libcurl=7.79.1 - - libdeflate=1.8 - - libiconv=1.16 - - libpng=1.6.37 - - libssh2=1.10.0 - - libtiff=4.3.0 - - libxml2=2.9.12 - - libxslt=1.1.33 - - libzlib=1.2.11 - - lxml=4.6.3 - - lz4-c=1.9.3 - - mako=1.1.5 - - markupsafe=2.0.1 - - matplotlib=3.4.3 - - matplotlib-base=3.4.3 - - matplotlib-inline=0.1.2 - - mkl=2021.3.0 - - mkl-service=2.4.0 - - mkl_fft=1.3.0 - - mkl_random=1.2.2 - - numexpr=2.7.3 - - numpy=1.21.2 - - numpy-base=1.21.2 - - olefile=0.46 - - openjpeg=2.4.0 - - openssl=1.1.1l - - pandas=1.3.3 - - parso=0.8.0 - - pickleshare=0.7.5 - - pillow=8.3.2 - - pip=21.2.4 - - prompt-toolkit=3.0.8 - - pyfai=0.20.0 - - pyfai-base=0.20.0 - - pygments=2.7.1 - - pyparsing=2.4.7 - - pyqt=5.12.3 - - pyqt-impl=5.12.3 - - pyqt5-sip=4.19.18 - - pyqtchart=5.12 - - pyqtwebengine=5.12.1 - - pyreadline=2.1 - - python=3.9.7 - - python-dateutil=2.8.2 - - python_abi=3.9 - - pytz=2021.3 - - pywin32=228 - - pyzmq=22.2.1 - - qt=5.12.9 - - qtconsole=5.1.1 - - qtpy=1.11.2 - - scipy=1.7.1 - - setuptools=58.0.4 - - silx=0.15.2 - - silx-base=0.15.2 - - six=1.16.0 - - sqlite=3.36.0 - - tk=8.6.11 - - tornado=6.1 - - traitlets=5.0.5 - - tzdata=2021a - - vc=14.2 - - vs2015_runtime=14.27.29016 - - wcwidth=0.2.5 - - wheel=0.37.0 - - wincertstore=0.2 - - xz=5.2.5 - - zlib=1.2.11 - - zstd=1.5.0 - - pip: - - beamtime==0.1 - - xlsx2csv==0.7.8 + - pandas + - ipykernel + - pyfai prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime From d19d3f78ac3aaecffe1501d1203afa46000142ff Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 21 Oct 2021 15:12:27 +0200 Subject: [PATCH 052/355] dfgsdfgsdfgasdfgfdeagrfds --- beamtime/requirements.txt | 118 +++++++++++++++++++++++++++++++++++--- 1 file changed, 109 insertions(+), 9 deletions(-) diff --git a/beamtime/requirements.txt b/beamtime/requirements.txt index 6b5d7f9..4df8eb2 100644 --- a/beamtime/requirements.txt +++ b/beamtime/requirements.txt @@ -1,9 +1,109 @@ -name: beamtime -channels: - - diffpy - - defaults -dependencies: - - pandas - - ipykernel - - pyfai -prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: win-64 +backcall=0.2.0=pyhd3eb1b0_0 +beamtime=0.1=pypi_0 +blas=1.0=mkl +bottleneck=1.3.2=py39h7cc1a96_1 +ca-certificates=2021.10.8=h5b45459_0 +cached-property=1.5.2=hd8ed1ab_1 +cached_property=1.5.2=pyha770c72_1 +certifi=2021.10.8=py39haa95532_0 +colorama=0.4.4=pyhd3eb1b0_0 +cycler=0.10.0=py_2 +debugpy=1.4.1=py39hd77b12b_0 +decorator=5.1.0=pyhd3eb1b0_0 +entrypoints=0.3=py39haa95532_0 +fabio=0.12.0=py39h5d4886f_0 +freetype=2.10.4=h546665d_1 +glymur=0.9.4=pyhd8ed1ab_0 +h5py=3.2.1=nompi_py39hf27771d_100 +hdf5=1.10.6=nompi_h5268f04_1114 +hdf5plugin=3.1.1=py39h71586dd_0 +icc_rt=2019.0.0=h0cc432a_1 +icu=68.1=h6c2663c_0 +intel-openmp=2021.3.0=haa95532_3372 +ipykernel=6.4.1=py39haa95532_1 +ipython=7.27.0=py39hd4e2768_0 +ipython_genutils=0.2.0=pyhd3eb1b0_1 +jbig=2.1=h8d14728_2003 +jedi=0.18.0=py39haa95532_1 +jpeg=9d=h2bbff1b_0 +jupyter_client=7.0.1=pyhd3eb1b0_0 +jupyter_core=4.8.1=py39haa95532_0 +kiwisolver=1.3.2=py39h2e07f2f_0 +krb5=1.19.2=hbae68bd_2 +lerc=3.0=h0e60522_0 +libclang=11.1.0=default_h5c34c98_1 +libcurl=7.79.1=h789b8ee_1 +libdeflate=1.8=h2bbff1b_5 +libiconv=1.16=he774522_0 +libpng=1.6.37=h1d00b33_2 +libssh2=1.10.0=h680486a_2 +libtiff=4.3.0=hd413186_2 +libwebp=1.2.0=h2bbff1b_0 +libxml2=2.9.12=h0ad7f3c_0 +libxslt=1.1.34=he774522_0 +libzlib=1.2.11=h8ffe710_1013 +lxml=4.6.3=py39h4fd7cdf_0 +lz4-c=1.9.3=h8ffe710_1 +mako=1.1.5=pyhd8ed1ab_0 +markupsafe=2.0.1=py39h2bbff1b_0 +matplotlib=3.4.3=py39hcbf5309_1 +matplotlib-base=3.4.3=py39h581301d_1 +matplotlib-inline=0.1.2=pyhd3eb1b0_2 +mkl=2021.3.0=haa95532_524 +mkl-service=2.4.0=py39h2bbff1b_0 +mkl_fft=1.3.1=py39h277e83a_0 +mkl_random=1.2.2=py39hf11a4ad_0 +nest-asyncio=1.5.1=pyhd3eb1b0_0 +numexpr=2.7.3=py39hb80d3ca_1 +numpy=1.21.2=py39hfca59bb_0 +numpy-base=1.21.2=py39h0829f74_0 +olefile=0.46=pyh9f0ad1d_1 +openjpeg=2.4.0=hb211442_1 +openssl=1.1.1l=h8ffe710_0 +pandas=1.3.3=py39h6214cd6_0 +parso=0.8.2=pyhd3eb1b0_0 +pickleshare=0.7.5=pyhd3eb1b0_1003 +pillow=8.4.0=py39hd45dc43_0 +pip=21.2.4=py39haa95532_0 +prompt-toolkit=3.0.20=pyhd3eb1b0_0 +pyfai=0.20.0=hd8ed1ab_0 +pyfai-base=0.20.0=py39h2e25243_0 +pygments=2.10.0=pyhd3eb1b0_0 +pyparsing=2.4.7=pyhd3eb1b0_0 +pyqt=5.12.3=py39hcbf5309_7 +pyqt-impl=5.12.3=py39h415ef7b_7 +pyqt5-sip=4.19.18=py39h415ef7b_7 +pyqtchart=5.12=py39h415ef7b_7 +pyqtwebengine=5.12.1=py39h415ef7b_7 +pyreadline=2.1=py39hcbf5309_1004 +python=3.9.7=h6244533_1 +python-dateutil=2.8.2=pyhd3eb1b0_0 +python_abi=3.9=2_cp39 +pytz=2021.3=pyhd3eb1b0_0 +pywin32=228=py39hbaba5e8_1 +pyzmq=22.2.1=py39hd77b12b_1 +qt=5.12.9=h5909a2a_4 +qtconsole=5.1.1=pyhd3eb1b0_0 +qtpy=1.11.2=pyhd8ed1ab_0 +scipy=1.7.1=py39hbe87c03_2 +setuptools=58.0.4=py39haa95532_0 +silx=0.15.2=hd8ed1ab_0 +silx-base=0.15.2=py39h2e25243_0 +six=1.16.0=pyhd3eb1b0_0 +sqlite=3.36.0=h2bbff1b_0 +tk=8.6.11=h8ffe710_1 +tornado=6.1=py39h2bbff1b_0 +traitlets=5.1.0=pyhd3eb1b0_0 +tzdata=2021a=h5d7bf9c_0 +vc=14.2=h21ff451_1 +vs2015_runtime=14.27.29016=h5e58377_2 +wcwidth=0.2.5=pyhd3eb1b0_0 +wheel=0.37.0=pyhd3eb1b0_1 +wincertstore=0.2=py39haa95532_2 +xlsx2csv=0.7.8=pypi_0 +xz=5.2.5=h62dcd97_1 +zlib=1.2.11=h8ffe710_1013 +zstd=1.5.0=h6255e5f_0 From 0e3e0e8a4fe3cb6063eab1a5899194d4f6e57f88 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 21 Oct 2021 15:18:08 +0200 Subject: [PATCH 053/355] tstsdtdftg --- beamtime/requirements.txt | 211 +++++++++++++++++++------------------- 1 file changed, 105 insertions(+), 106 deletions(-) diff --git a/beamtime/requirements.txt b/beamtime/requirements.txt index 4df8eb2..d8eeb75 100644 --- a/beamtime/requirements.txt +++ b/beamtime/requirements.txt @@ -1,109 +1,108 @@ # This file may be used to create an environment using: # $ conda create --name --file # platform: win-64 -backcall=0.2.0=pyhd3eb1b0_0 -beamtime=0.1=pypi_0 -blas=1.0=mkl -bottleneck=1.3.2=py39h7cc1a96_1 -ca-certificates=2021.10.8=h5b45459_0 -cached-property=1.5.2=hd8ed1ab_1 -cached_property=1.5.2=pyha770c72_1 -certifi=2021.10.8=py39haa95532_0 -colorama=0.4.4=pyhd3eb1b0_0 -cycler=0.10.0=py_2 -debugpy=1.4.1=py39hd77b12b_0 -decorator=5.1.0=pyhd3eb1b0_0 -entrypoints=0.3=py39haa95532_0 -fabio=0.12.0=py39h5d4886f_0 -freetype=2.10.4=h546665d_1 -glymur=0.9.4=pyhd8ed1ab_0 -h5py=3.2.1=nompi_py39hf27771d_100 -hdf5=1.10.6=nompi_h5268f04_1114 -hdf5plugin=3.1.1=py39h71586dd_0 -icc_rt=2019.0.0=h0cc432a_1 -icu=68.1=h6c2663c_0 -intel-openmp=2021.3.0=haa95532_3372 -ipykernel=6.4.1=py39haa95532_1 -ipython=7.27.0=py39hd4e2768_0 -ipython_genutils=0.2.0=pyhd3eb1b0_1 -jbig=2.1=h8d14728_2003 -jedi=0.18.0=py39haa95532_1 -jpeg=9d=h2bbff1b_0 -jupyter_client=7.0.1=pyhd3eb1b0_0 -jupyter_core=4.8.1=py39haa95532_0 -kiwisolver=1.3.2=py39h2e07f2f_0 -krb5=1.19.2=hbae68bd_2 -lerc=3.0=h0e60522_0 -libclang=11.1.0=default_h5c34c98_1 -libcurl=7.79.1=h789b8ee_1 -libdeflate=1.8=h2bbff1b_5 -libiconv=1.16=he774522_0 -libpng=1.6.37=h1d00b33_2 -libssh2=1.10.0=h680486a_2 -libtiff=4.3.0=hd413186_2 -libwebp=1.2.0=h2bbff1b_0 -libxml2=2.9.12=h0ad7f3c_0 -libxslt=1.1.34=he774522_0 -libzlib=1.2.11=h8ffe710_1013 -lxml=4.6.3=py39h4fd7cdf_0 -lz4-c=1.9.3=h8ffe710_1 -mako=1.1.5=pyhd8ed1ab_0 -markupsafe=2.0.1=py39h2bbff1b_0 -matplotlib=3.4.3=py39hcbf5309_1 -matplotlib-base=3.4.3=py39h581301d_1 -matplotlib-inline=0.1.2=pyhd3eb1b0_2 -mkl=2021.3.0=haa95532_524 -mkl-service=2.4.0=py39h2bbff1b_0 -mkl_fft=1.3.1=py39h277e83a_0 -mkl_random=1.2.2=py39hf11a4ad_0 -nest-asyncio=1.5.1=pyhd3eb1b0_0 -numexpr=2.7.3=py39hb80d3ca_1 -numpy=1.21.2=py39hfca59bb_0 -numpy-base=1.21.2=py39h0829f74_0 -olefile=0.46=pyh9f0ad1d_1 -openjpeg=2.4.0=hb211442_1 -openssl=1.1.1l=h8ffe710_0 -pandas=1.3.3=py39h6214cd6_0 -parso=0.8.2=pyhd3eb1b0_0 -pickleshare=0.7.5=pyhd3eb1b0_1003 -pillow=8.4.0=py39hd45dc43_0 -pip=21.2.4=py39haa95532_0 -prompt-toolkit=3.0.20=pyhd3eb1b0_0 -pyfai=0.20.0=hd8ed1ab_0 -pyfai-base=0.20.0=py39h2e25243_0 -pygments=2.10.0=pyhd3eb1b0_0 -pyparsing=2.4.7=pyhd3eb1b0_0 -pyqt=5.12.3=py39hcbf5309_7 -pyqt-impl=5.12.3=py39h415ef7b_7 -pyqt5-sip=4.19.18=py39h415ef7b_7 -pyqtchart=5.12=py39h415ef7b_7 -pyqtwebengine=5.12.1=py39h415ef7b_7 -pyreadline=2.1=py39hcbf5309_1004 -python=3.9.7=h6244533_1 -python-dateutil=2.8.2=pyhd3eb1b0_0 -python_abi=3.9=2_cp39 -pytz=2021.3=pyhd3eb1b0_0 -pywin32=228=py39hbaba5e8_1 -pyzmq=22.2.1=py39hd77b12b_1 -qt=5.12.9=h5909a2a_4 -qtconsole=5.1.1=pyhd3eb1b0_0 -qtpy=1.11.2=pyhd8ed1ab_0 -scipy=1.7.1=py39hbe87c03_2 -setuptools=58.0.4=py39haa95532_0 -silx=0.15.2=hd8ed1ab_0 -silx-base=0.15.2=py39h2e25243_0 -six=1.16.0=pyhd3eb1b0_0 -sqlite=3.36.0=h2bbff1b_0 -tk=8.6.11=h8ffe710_1 -tornado=6.1=py39h2bbff1b_0 -traitlets=5.1.0=pyhd3eb1b0_0 -tzdata=2021a=h5d7bf9c_0 -vc=14.2=h21ff451_1 -vs2015_runtime=14.27.29016=h5e58377_2 -wcwidth=0.2.5=pyhd3eb1b0_0 -wheel=0.37.0=pyhd3eb1b0_1 -wincertstore=0.2=py39haa95532_2 -xlsx2csv=0.7.8=pypi_0 -xz=5.2.5=h62dcd97_1 -zlib=1.2.11=h8ffe710_1013 -zstd=1.5.0=h6255e5f_0 +@EXPLICIT +https://repo.anaconda.com/pkgs/main/win-64/blas-1.0-mkl.conda +https://conda.anaconda.org/conda-forge/win-64/ca-certificates-2021.10.8-h5b45459_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/icc_rt-2019.0.0-h0cc432a_1.conda +https://repo.anaconda.com/pkgs/main/win-64/intel-openmp-2021.3.0-haa95532_3372.conda +https://repo.anaconda.com/pkgs/main/noarch/tzdata-2021a-h5d7bf9c_0.conda +https://repo.anaconda.com/pkgs/main/win-64/vs2015_runtime-14.27.29016-h5e58377_2.conda +https://repo.anaconda.com/pkgs/main/win-64/mkl-2021.3.0-haa95532_524.conda +https://repo.anaconda.com/pkgs/main/win-64/vc-14.2-h21ff451_1.conda +https://repo.anaconda.com/pkgs/main/win-64/icu-68.1-h6c2663c_0.conda +https://conda.anaconda.org/conda-forge/win-64/jbig-2.1-h8d14728_2003.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/jpeg-9d-h2bbff1b_0.conda +https://conda.anaconda.org/conda-forge/win-64/lerc-3.0-h0e60522_0.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/libclang-11.1.0-default_h5c34c98_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/libdeflate-1.8-h2bbff1b_5.conda +https://conda.anaconda.org/conda-forge/win-64/libiconv-1.16-he774522_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/libwebp-1.2.0-h2bbff1b_0.conda +https://conda.anaconda.org/conda-forge/win-64/libzlib-1.2.11-h8ffe710_1013.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/lz4-c-1.9.3-h8ffe710_1.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/openssl-1.1.1l-h8ffe710_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/sqlite-3.36.0-h2bbff1b_0.conda +https://conda.anaconda.org/conda-forge/win-64/tk-8.6.11-h8ffe710_1.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/xz-5.2.5-h62dcd97_1.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/krb5-1.19.2-hbae68bd_2.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/python-3.9.7-h6244533_1.conda +https://conda.anaconda.org/conda-forge/win-64/zlib-1.2.11-h8ffe710_1013.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/backcall-0.2.0-pyhd3eb1b0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/certifi-2021.10.8-py39haa95532_0.conda +https://repo.anaconda.com/pkgs/main/noarch/colorama-0.4.4-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/win-64/debugpy-1.4.1-py39hd77b12b_0.conda +https://repo.anaconda.com/pkgs/main/noarch/decorator-5.1.0-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/win-64/entrypoints-0.3-py39haa95532_0.conda +https://repo.anaconda.com/pkgs/main/noarch/ipython_genutils-0.2.0-pyhd3eb1b0_1.conda +https://conda.anaconda.org/conda-forge/win-64/libpng-1.6.37-h1d00b33_2.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/libssh2-1.10.0-h680486a_2.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/libxml2-2.9.12-h0ad7f3c_0.conda +https://repo.anaconda.com/pkgs/main/win-64/markupsafe-2.0.1-py39h2bbff1b_0.conda +https://repo.anaconda.com/pkgs/main/noarch/nest-asyncio-1.5.1-pyhd3eb1b0_0.conda +https://conda.anaconda.org/conda-forge/noarch/olefile-0.46-pyh9f0ad1d_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/parso-0.8.2-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pickleshare-0.7.5-pyhd3eb1b0_1003.conda +https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.7-pyhd3eb1b0_0.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/python_abi-3.9-2_cp39.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/pytz-2021.3-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/win-64/pywin32-228-py39hbaba5e8_1.conda +https://repo.anaconda.com/pkgs/main/win-64/pyzmq-22.2.1-py39hd77b12b_1.conda +https://conda.anaconda.org/conda-forge/noarch/qtpy-1.11.2-pyhd8ed1ab_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/win-64/tornado-6.1-py39h2bbff1b_0.conda +https://repo.anaconda.com/pkgs/main/noarch/traitlets-5.1.0-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.2.5-pyhd3eb1b0_0.conda +https://repo.anaconda.com/pkgs/main/noarch/wheel-0.37.0-pyhd3eb1b0_1.conda +https://repo.anaconda.com/pkgs/main/win-64/wincertstore-0.2-py39haa95532_2.conda +https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.0-h6255e5f_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/cycler-0.10.0-py_2.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/freetype-2.10.4-h546665d_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/jedi-0.18.0-py39haa95532_1.conda +https://repo.anaconda.com/pkgs/main/win-64/jupyter_core-4.8.1-py39haa95532_0.conda +https://conda.anaconda.org/conda-forge/win-64/kiwisolver-1.3.2-py39h2e07f2f_0.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/libcurl-7.79.1-h789b8ee_1.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/libtiff-4.3.0-hd413186_2.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/libxslt-1.1.34-he774522_0.conda +https://conda.anaconda.org/conda-forge/noarch/mako-1.1.5-pyhd8ed1ab_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/matplotlib-inline-0.1.2-pyhd3eb1b0_2.conda +https://repo.anaconda.com/pkgs/main/win-64/mkl-service-2.4.0-py39h2bbff1b_0.conda +https://repo.anaconda.com/pkgs/main/noarch/prompt-toolkit-3.0.20-pyhd3eb1b0_0.conda +https://conda.anaconda.org/conda-forge/win-64/pyqt5-sip-4.19.18-py39h415ef7b_7.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/pyreadline-2.1-py39hcbf5309_1004.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.2-pyhd3eb1b0_0.conda +https://conda.anaconda.org/conda-forge/win-64/qt-5.12.9-h5909a2a_4.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/setuptools-58.0.4-py39haa95532_0.conda +https://conda.anaconda.org/conda-forge/win-64/hdf5-1.10.6-nompi_h5268f04_1114.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/jupyter_client-7.0.1-pyhd3eb1b0_0.conda +https://conda.anaconda.org/conda-forge/win-64/lxml-4.6.3-py39h4fd7cdf_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/numpy-base-1.21.2-py39h0829f74_0.conda +https://conda.anaconda.org/conda-forge/win-64/openjpeg-2.4.0-hb211442_1.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/pillow-8.4.0-py39hd45dc43_0.conda +https://repo.anaconda.com/pkgs/main/win-64/pip-21.2.4-py39haa95532_0.conda +https://repo.anaconda.com/pkgs/main/noarch/pygments-2.10.0-pyhd3eb1b0_0.conda +https://conda.anaconda.org/conda-forge/win-64/pyqt-impl-5.12.3-py39h415ef7b_7.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/ipython-7.27.0-py39hd4e2768_0.conda +https://conda.anaconda.org/conda-forge/win-64/pyqtchart-5.12-py39h415ef7b_7.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/pyqtwebengine-5.12.1-py39h415ef7b_7.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/ipykernel-6.4.1-py39haa95532_1.conda +https://conda.anaconda.org/conda-forge/win-64/pyqt-5.12.3-py39hcbf5309_7.tar.bz2 +https://repo.anaconda.com/pkgs/main/noarch/qtconsole-5.1.1-pyhd3eb1b0_0.conda +https://conda.anaconda.org/conda-forge/noarch/glymur-0.9.4-pyhd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/fabio-0.12.0-py39h5d4886f_0.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/h5py-3.2.1-nompi_py39hf27771d_100.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/hdf5plugin-3.1.1-py39h71586dd_0.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/matplotlib-3.4.3-py39hcbf5309_1.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/matplotlib-base-3.4.3-py39h581301d_1.tar.bz2 +https://conda.anaconda.org/conda-forge/win-64/silx-base-0.15.2-py39h2e25243_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/silx-0.15.2-hd8ed1ab_0.tar.bz2 +https://conda.anaconda.org/conda-forge/noarch/pyfai-0.20.0-hd8ed1ab_0.tar.bz2 +https://repo.anaconda.com/pkgs/main/win-64/bottleneck-1.3.2-py39h7cc1a96_1.conda +https://repo.anaconda.com/pkgs/main/win-64/mkl_fft-1.3.1-py39h277e83a_0.conda +https://repo.anaconda.com/pkgs/main/win-64/mkl_random-1.2.2-py39hf11a4ad_0.conda +https://repo.anaconda.com/pkgs/main/win-64/numpy-1.21.2-py39hfca59bb_0.conda +https://repo.anaconda.com/pkgs/main/win-64/numexpr-2.7.3-py39hb80d3ca_1.conda +https://repo.anaconda.com/pkgs/main/win-64/scipy-1.7.1-py39hbe87c03_2.conda +https://repo.anaconda.com/pkgs/main/win-64/pandas-1.3.3-py39h6214cd6_0.conda +https://conda.anaconda.org/conda-forge/win-64/pyfai-base-0.20.0-py39h2e25243_0.tar.bz2 From 75acc9cfecd263c20ee8b8c40fe743b716ca8959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= Date: Thu, 21 Oct 2021 18:57:03 +0200 Subject: [PATCH 054/355] Update README.md Add installation instructions and instructions on basic use of electrochemistry module --- beamtime/README.md | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/beamtime/README.md b/beamtime/README.md index de5a12d..0caa104 100644 --- a/beamtime/README.md +++ b/beamtime/README.md @@ -1,2 +1,77 @@ # beamtime A package for processing and analysis of data from beamtime at SNBL + + +# 1.0 Installation + +### 1.1 Virtual environment + +It is recommended to use a new dedicated virtual environment to use this package to ensure that all depedency versions are the same. This can be done by first creating a new environtment with the Python version 3.9.7 as follows: + +``` +conda create --name beamtime python=3.9.7 +``` + +Here you can replace `beamtime` with any name you prefer. If you do, make sure you replace it in the subsequent commands as well. + +In order to use the virtual environment, you need to activate it. This is done by typing + +``` +conda activate beamtime +``` + +Note that you might have to initialise your shell first to allow for this. If so, you'll get a message saying so and you need to follow the instructions you get. + +Once you are able to activate the virtual environment, you need to install all the packages required. These are listed in the requirements.txt file in the beamtime package folder, and can be installed by running the command (inside that folder): + +``` +conda install --file requirements.txt +``` + + +Lastly, you might want to install this virtual environment as its own Jupyter kernel if you are planning to use Jupyter Notebook / Labs with this package. This way you won't have to activate the environment everytime you use it, you just create a Jupyter Notebook with this kernel. + +To do so, run the command: + +``` +python -m ipykernel install --user --name beamtime --display-name "Python 3.9.7 (Beamtime)" +``` + +Here you need to change `beamtime` if you named it something else, and the display name can be whatever you want it to be. Note that you need to have the package `ipykernel` installed for this, but it should be installed from running the install command above. + +### 1.2 Installation of the package + +In order to also use `beamtime` package itself, you need to install it. It is not uploaded to any package manager, but it can be installed from the main folder containing the `setup.py` file. Run the following command in this folder: + +``` +pip install . +``` + +# 2.0 The `electrochemistry` module + +The `electrochemistry` module allows to plot galvanostatic cycling data from BioLogic, Neware and Batsmall. + +General use: + +```py +import beamtime.electrochemistry as ec + +path = 'path/to/data/file' +options = { + 'x_vals': 'specific_capacity', + 'y_vals': 'voltage', + 'active_material_weight': 4.3 + } + +cycles, fig, ax = ec.plot.plot_gc(path=path, kind='neware', options=options) + +``` + +Note that no options needs to be specified, all options will have default values that should make somewhat sense. A comprehensive list of options will be updated later. + +The return values from the `plot_gc` function are: +- `cycles`, a list of lists containing the charge and discharge cycles of each cycle in the form of a `pandas` DataFrame +- `fig`, the `matplotlib.pyplot' Figure object to allow for any modifications after initial plotting. +- `ax`, the 'matplotlib'pyplot' Axes object to allow for any modification after initial plotting. + +If these are not required, you can simply assign the return values to an underscore instead. However, omitting assignment will print all the DataFrames. From fa9f641c3fd385d0a20236fdb93196e24f751ba8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sat, 23 Oct 2021 18:37:21 +0200 Subject: [PATCH 055/355] Add XRD functionality --- beamtime/xrd/io.py | 99 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index b66dcde..349f635 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -1,4 +1,5 @@ import fabio, pyFAI +import pandas as pd import numpy as np import os @@ -11,11 +12,29 @@ def get_image_array(path): return image_array -def integrate_1d(path, calibrant, bins, options): +def integrate_1d(calibrant, bins, path=None, image=None, options=None): + ''' Integrates an image file to a 1D diffractogram. - required_options = ['unit', 'extension'] + Input: + calibrant: path to .poni-file + bins: Number of bins to divide image into + path (optional): path to image file - either this or image must be specified. If both is passed, image is prioritsed + image (optional): image array (Numpy) as extracted from get_image_array + options (optional): dictionary of options - default_options = {'unit': '2th_deg', 'extension': '_integrated.dat'} + Output: + df: DataFrame contianing 1D diffractogram if option 'return' is True + ''' + + required_options = ['unit', 'extension', 'filename', 'save_folder', 'overwrite', 'return'] + + default_options = { + 'unit': '2th_deg', + 'extension': '_integrated.dat', + 'filename': None, + 'save_folder': '.', + 'overwrite': False, + 'return': False} if not options: options = default_options @@ -26,15 +45,85 @@ def integrate_1d(path, calibrant, bins, options): options[option] = default_options[option] - image = get_image_array(path) + if not image: + image = get_image_array(path) + ai = pyFAI.load(calibrant) - filename = os.path.split(path)[-1].split('.')[0] + options['extension'] + + if not options['filename']: + if path: + filename = os.path.join(options['save_folder'], os.path.split(path)[-1].split('.')[0] + options['extension']) + else: + filename = os.path.join(options['save_folder'], 'integrated.dat') + + + if not options['overwrite']: + trunk = os.path.join(options['save_folder'], filename.split('\\')[-1].split('.')[0]) + extension = filename.split('.')[-1] + counter = 0 + + while os.path.isfile(filename): + counter_string = str(counter) + filename = trunk + '_' + counter_string.zfill(4) + '.' + extension + counter += 1 + + + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + res = ai.integrate1d(image, bins, unit=options['unit'], filename=filename) + if options['return']: + return open_1d_data(filename) +def open_1d_data(path, options=None): + + with open(path, 'r') as f: + position = 0 + + current_line = f.readline() + + while current_line[0] == '#': + position = f.tell() + current_line = f.readline() + + f.seek(position) + + df = pd.read_csv(f, header=None, delim_whitespace=True) + + df.columns = ['2th', 'I'] + + + return df + + + +def average_images(images): + ''' Takes a list of path to image files, reads them and averages them before returning the average image''' + + + image_arrays = [] + + for image in images: + image_array = xrd.io.get_image_array(os.path.join(root, image)) + image_arrays.append(image_array) + + + image_arrays = np.array(image_arrays) + + image_average = image_arrays.mean(axis=0) + + + return image_average + + +def subtract_dark(image, dark): + + return image - dark + def view_integrator(calibrant): From ca14345088056ba6999bea129032c2741a4c400c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 24 Oct 2021 13:10:23 +0200 Subject: [PATCH 056/355] Test --- beamtime/xrd/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/xrd/test.txt diff --git a/beamtime/xrd/test.txt b/beamtime/xrd/test.txt new file mode 100644 index 0000000..e69de29 From efca9edaab92cdd2c5abe6e4e8874a3d534d3ff8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 24 Oct 2021 13:49:56 +0200 Subject: [PATCH 057/355] test --- beamtime/xrd/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 beamtime/xrd/test.txt diff --git a/beamtime/xrd/test.txt b/beamtime/xrd/test.txt deleted file mode 100644 index e69de29..0000000 From 65a54979704dcebd6423fb121651730bf81d1656 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 24 Oct 2021 14:08:52 +0200 Subject: [PATCH 058/355] test --- beamtime/xrd/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/xrd/test.txt diff --git a/beamtime/xrd/test.txt b/beamtime/xrd/test.txt new file mode 100644 index 0000000..e69de29 From 5c977cc3878e1d738799ab93f5d49461fa2b54ca Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 25 Oct 2021 08:33:58 +0200 Subject: [PATCH 059/355] Add placeholder functions to xrd/io.py --- beamtime/halvorsen.txt | 5 - beamtime/test.txt | 1 - beamtime/xrd/io.py | 315 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 315 insertions(+), 6 deletions(-) delete mode 100644 beamtime/halvorsen.txt delete mode 100644 beamtime/test.txt diff --git a/beamtime/halvorsen.txt b/beamtime/halvorsen.txt deleted file mode 100644 index cecc2a9..0000000 --- a/beamtime/halvorsen.txt +++ /dev/null @@ -1,5 +0,0 @@ -'''test''' - -# changez -jkonkjnmkjnkj -5616521656 \ No newline at end of file diff --git a/beamtime/test.txt b/beamtime/test.txt deleted file mode 100644 index ff62eb7..0000000 --- a/beamtime/test.txt +++ /dev/null @@ -1 +0,0 @@ -dfgdfg diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 349f635..7706ce3 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -3,6 +3,12 @@ import pandas as pd import numpy as np import os +import zipfile +import io +import sys +import glob +import xml.etree.ElementTree as ET + def get_image_array(path): @@ -141,3 +147,312 @@ def view_integrator(calibrant): print("\nIntegrator: \n", ai) +def brml_reader(file_name): + """ + From: https://github.com/aboulle/DxTools/blob/master/data_reader.py + + Extracts brml into a temporary unzip file, and parses all xml files. + For every intensity value, saves all relevant motor coordinates and sensor values. + All values are save in temporary SPEC-style tmp file. + """ +#***************************************************************************************************** +# Unzip xml files +#***************************************************************************************************** + extract_path = os.path.join(os.getcwd(),"unzip") + if sys.platform == "win32": + print("Detected platform: Windows") + os.system("RMDIR "+ extract_path +" /s /q") + elif sys.platform == "darwin": + print("Detected platform: MacOS") + os.system("rm -rf "+ extract_path) + elif sys.platform == "linux" or sys.platform == "linux2": + print("Detected platform: Linux") + os.system("rm -rf "+ extract_path) + + #Extract all RawData*.xml files and InstructionContainer the brml to temporary unzip file + with zipfile.ZipFile(file_name,"r") as brml: + for info in brml.infolist(): + if ("RawData" in info.filename) or ("InstructionContainer" in info.filename): + #if ("RawData" in info.filename): + brml.extract(info.filename, extract_path) +#***************************************************************************************************** +# For time counting, the number of days is initialized to 0. +# Compatibility fixes with D8 advance and older Discover: offsets, chi and tx, ty are initialized to 0 +#***************************************************************************************************** + # Initialize the number of days to 0 + n_day = 0. + # Initialize all offsets to 0 + off_tth = off_om = off_phi = off_chi = off_tx = off_ty = 0 + # Set Chi, tx and ty to 0 for D8 advance (July 2017 Julia Stroh) + chi = tx = ty = "0" +#***************************************************************************************************** +#Modification June 2017 (Duc Dinh) +#In some RawData.xml files, wavelength and static motors are missing. +#Find wl and static motors in MeasurementContainer.xml +#***************************************************************************************************** + data_path = os.path.join(extract_path, "*0","InstructionContainer.xml") + for file in sorted(glob.glob(data_path)): + tree = ET.parse(file) + root = tree.getroot() + for chain in root.findall("./ComparisonMethod/HrxrdAlignmentData"): + wl = chain.find("WaveLength").attrib["Value"] + + for chain in root.findall("./ComparisonMethod/HrxrdAlignmentData/Data"): + if chain.get("LogicName") == "TwoTheta": + tth = chain.find("TheoreticalPosition").attrib["Value"] + off_tth = chain.find("PositionOffset").attrib["Value"] + tth = str(float(tth)-float(off_tth)) + + if chain.get("LogicName") == "Theta": + om = chain.find("TheoreticalPosition").attrib["Value"] + off_om = chain.find("PositionOffset").attrib["Value"] + om = str(float(om)-float(off_om)) + + if chain.get("LogicName") == "Chi": + chi = chain.find("TheoreticalPosition").attrib["Value"] + off_chi = chain.find("PositionOffset").attrib["Value"] + chi = str(float(chi)-float(off_chi)) + + if chain.get("LogicName") == "Phi": + phi = chain.find("TheoreticalPosition").attrib["Value"] + off_phi = chain.find("PositionOffset").attrib["Value"] + phi = str(float(phi)-float(off_phi)) + + if chain.get("LogicName") == "X": + tx = chain.find("TheoreticalPosition").attrib["Value"] + off_tx = chain.find("PositionOffset").attrib["Value"] + tx = str(float(tx)-float(off_tx)) + + if chain.get("LogicName") == "Y": + ty = chain.find("TheoreticalPosition").attrib["Value"] + off_ty = chain.find("PositionOffset").attrib["Value"] + ty = str(float(ty)-float(off_ty)) + os.remove(file) + + #Create ouput file + outfile = open("tmp", "w", encoding='utf8') # Create output data file + outfile.write("#temperature khi phi x y theta offset 2theta scanning motor intensity time\n") +#***************************************************************************************************** +# Finds scan type, wl, scanning motors and fixed motors values in RawData*.xml +#***************************************************************************************************** + data_path = os.path.join(extract_path, "*0","*.xml") #reading files in Experiment0 folder + for file in sorted(glob.glob(data_path), key=file_nb): + new_file = 0 + check_temperature = 0 + check_1Dmode = 0 + #parsing XML file + tree = ET.parse(file) + root = tree.getroot() + #obtain scan type + for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation") or root.findall("./ScanInformation")): + scan_type = chain.get("VisibleName") + if ("PSD" in scan_type) or ("Psd" in scan_type): + scan_type = "PSDFIXED" + if ("Coupled" in scan_type) or ("coupled" in scan_type) or ("2Theta-Omega" in scan_type): + scan_type = "COUPLED" + if ("Rocking" in scan_type) or ("rocking" in scan_type): + scan_type = "THETA" + + # Check if temperature is recorded + for chain in (root.findall("./DataRoutes/DataRoute/DataViews/RawDataView/Recording") or root.findall("./DataViews/RawDataView/Recording")): + if "Temperature" in chain.get("LogicName"): + check_temperature = 1 + + #Find wl in RawData.xml + for chain in root.findall("./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1"): + wl = chain.get("Value") + + # Find the fast-scanning axis + for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation/ScanAxes/ScanAxisInfo") or root.findall("./ScanInformation/ScanAxes/ScanAxisInfo")): + if new_file == 0: + if chain.get("AxisName") == "TwoTheta": #Added offset correction / June 2017. Only relevant if offset in InstructionContainer. 0 otherwise. + off_scan = float(off_tth) + elif chain.get("AxisName") == "Theta": + off_scan = float(off_om) + else: + off_scan = 0 + step = chain.find("Increment").text + start = chain.find("Start").text + stop = chain.find("Stop").text + ref = chain.find("Reference").text + start = str(float(ref)+float(start)-off_scan) #Added offset correction / June 2017. + #start = str(float(ref)+float(start)) + new_file += 1 + + # Find scanning motors + for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation/ScanAxes/ScanAxisInfo") or root.findall("./ScanInformation/ScanAxes/ScanAxisInfo")): + if chain.get("AxisName") == "TwoTheta": + tth = chain.find("Start").text + ref = chain.find("Reference").text + tth = str(float(ref)+float(tth)-float(off_tth)) #Added offset correction / June 2017. + #tth = str(float(ref)+float(tth)) + + if chain.get("AxisName") == "Theta": + om = chain.find("Start").text + ref = chain.find("Reference").text + om = str(float(ref)+float(om)-float(off_om)) #Added offset correction / June 2017. + #om = str(float(ref)+float(om)) + + if chain.get("AxisName") == "Chi": + chi = chain.find("Start").text + ref = chain.find("Reference").text + chi = str(float(ref)+float(chi)-float(off_chi)) #Added offset correction / June 2017. + #chi = str(float(ref)+float(chi)) + + if chain.get("AxisName") == "Phi": + phi = chain.find("Start").text + ref = chain.find("Reference").text + phi = str(float(ref)+float(phi)-float(off_phi)) #Added offset correction / June 2017. + #phi = str(float(ref)+float(phi)) + + if chain.get("AxisName") == "X": + tx = chain.find("Start").text + ref = chain.find("Reference").text + tx = str(float(ref)+float(tx)-float(off_tx)) #Added offset correction / June 2017. + #tx = str(float(ref)+float(tx)) + + if chain.get("AxisName") == "Y": + ty = chain.find("Start").text + ref = chain.find("Reference").text + ty = str(float(ref)+float(ty)-float(off_ty)) #Added offset correction / June 2017. + #ty = str(float(ref)+float(ty)) + + # Find static motors + for chain in root.findall("./FixedInformation/Drives/InfoData"): + if chain.get("LogicName") == "TwoTheta": + tth = chain.find("Position").attrib["Value"] + tth = str(float(tth)-float(off_tth)) #Added offset correction / June 2017. + + if chain.get("LogicName") == "Theta": + om = chain.find("Position").attrib["Value"] + om = str(float(om)-float(off_om)) #Added offset correction / June 2017. + + if chain.get("LogicName") == "Chi": + chi = chain.find("Position").attrib["Value"] + chi = str(float(chi)-float(off_chi)) #Added offset correction / June 2017. + + if chain.get("LogicName") == "Phi": + phi = chain.find("Position").attrib["Value"] + phi = str(float(phi)-float(off_phi)) #Added offset correction / June 2017. + + if chain.get("LogicName") == "X": + tx = chain.find("Position").attrib["Value"] + tx = str(float(tx)-float(off_tx)) #Added offset correction / June 2017. + + if chain.get("LogicName") == "Y": + ty = chain.find("Position").attrib["Value"] + ty = str(float(ty)-float(off_ty)) #Added offset correction / June 2017. + + offset = str(float(om) - float(tth)/2.) + +#***************************************************************************************************** +# This section computes scanning time, scanning angular range and scanning speed +# in order to convert 2th values to time values (July 2017, Julia Stroh) +#***************************************************************************************************** + for chain in (root.findall("./TimeStampStarted")): + d_start = ((chain.text).split("T")[0]).split("-")[2] + t_start = ((chain.text).split("T")[1]).split("+")[0] + h_start, min_start, sec_start = t_start.split(":") + t_start = float(h_start)*3600 + float(min_start)*60 + float(sec_start) + + if file_nb(file)==0: + abs_start = t_start + for chain in (root.findall("./TimeStampFinished")): + d_stop = ((chain.text).split("T")[0]).split("-")[2] + t_stop = ((chain.text).split("T")[1]).split("+")[0] + h_stop, min_stop, sec_stop = t_stop.split(":") + t_stop = float(h_stop)*3600 + float(min_stop)*60 + float(sec_stop) + + # Check if detector is in 1D mode + for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation") or root.findall("./ScanInformation")): + if chain.find("TimePerStep").text != chain.find("TimePerStepEffective").text: + check_1Dmode = 1 + + # Check if day changed between start and stop and correct accordingly + if d_stop != d_start: + t_stop += 24*3600. + total_scan_time = t_stop - t_start + + #scanning range + dth_scan = float(stop)-float(start) + #psd range + dth_psd = 0 + for chain in root.findall("./FixedInformation/Detectors/InfoData/AngularOpening"): + dth_psd = chain.get("Value") + total_dth = float(dth_psd)*check_1Dmode+float(dth_scan) + + scan_speed = total_dth / total_scan_time +#***************************************************************************************************** +# Finds intensity values. If temperature is recorded, also fin temperature values. +# The intensity data is formatted differently in PSDfixed mode and when temperature is recorded +#***************************************************************************************************** + if "PSDFIXED" in scan_type: + if check_temperature == 0: + for chain in (root.findall("./DataRoutes/DataRoute") or root.findall("./")): + intensity = (chain.find("Datum").text).split(',') + + for chain in (root.findall("./DataRoutes/DataRoute/DataViews/RawDataView/Recording") or root.findall("./DataViews/RawDataView/Recording")): + if chain.get("LogicName") == "Counter1D": + n_channels = int(chain.find("Size/X").text) + + line_count = 0 + int_shift = len(intensity) - n_channels + for i in range(n_channels): #the intensity values are shifted to the right by int_shift + if i == 0: + scanning = float(start) + else: + scanning += float(step) + line_count += 1 + t_2th = (t_start+n_day*24*3600 - abs_start)+((float(dth_psd)*check_1Dmode + scanning - float(start)) / scan_speed) + outfile.write("25" + " " + (chi) + " " + (phi) + + " " + (tx) + " " + (ty) + " " + (om) + + " " + (offset) + " " + (tth) + " " + str(scanning) + + " " + intensity[i+int_shift] +" " + str(t_2th) +'\n') + else: + return implementation_warning, 0, 0 + + #if "COUPLED" in scan_type: + # to do check in brml that all scans (except psd fixed) share the same data structure (wrt temperature) + else: + if check_temperature == 0: + line_count = 0 + for chain in (root.findall("./DataRoutes/DataRoute/Datum") or root.findall("./Datum")): + if line_count == 0: + scanning = float(start) + else: + scanning += float(step) + line_count += 1 + intensity = (chain.text).split(',')[-1] + #compute time corresponding to scanning angle (July 2017) + t_2th = (t_start+n_day*24*3600 - abs_start)+((float(dth_psd)*check_1Dmode + scanning - float(start)) / scan_speed) + outfile.write("25" + " " + (chi) + " " + (phi) + + " " + (tx) + " " + (ty) + " " + (om) + + " " + (offset) + " " + (tth) + " " + str(round(scanning, 4)) + + " " + intensity + " " + str(t_2th) +'\n') + else: + line_count = 0 + for chain in (root.findall("./DataRoutes/DataRoute/Datum") or root.findall("./Datum")): + if line_count == 0: + scanning = float(start) + else: + scanning += float(step) + line_count += 1 + t_2th = (t_start+n_day*24*3600 - abs_start)+((float(dth_psd)*check_1Dmode + scanning - float(start)) / scan_speed) + intensity = (chain.text).split(',')[-2] + temperature = (chain.text).split(',')[-1] + outfile.write(temperature + " " + (chi) + " " + (phi) + + " " + (tx) + " " + (ty) + " " + (om) + + " " + (offset) + " " + (tth) + " " + str(round(scanning, 4)) + + " " + intensity + " " + str(t_2th) +'\n') + + if d_stop != d_start: + n_day+=1 + + outfile.close() + if sys.platform == "win32": + os.system("RMDIR "+ extract_path +" /s /q") + elif sys.platform == "darwin": + os.system("rm -rf "+ extract_path) + elif sys.platform == "linux" or sys.platform == "linux2": + os.system("rm -rf "+ extract_path) + return scan_type, line_count, wl \ No newline at end of file From acdc9399ebfc5598a6bf1a89139697b2d977d4ea Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 27 Oct 2021 15:33:56 +0200 Subject: [PATCH 060/355] Add read brml functionality --- beamtime/xrd/io.py | 342 +++++++-------------------------------------- 1 file changed, 54 insertions(+), 288 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 7706ce3..38220cd 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -2,6 +2,7 @@ import fabio, pyFAI import pandas as pd import numpy as np import os +import shutil import zipfile import io @@ -147,312 +148,77 @@ def view_integrator(calibrant): print("\nIntegrator: \n", ai) -def brml_reader(file_name): - """ - From: https://github.com/aboulle/DxTools/blob/master/data_reader.py - - Extracts brml into a temporary unzip file, and parses all xml files. - For every intensity value, saves all relevant motor coordinates and sensor values. - All values are save in temporary SPEC-style tmp file. - """ -#***************************************************************************************************** -# Unzip xml files -#***************************************************************************************************** - extract_path = os.path.join(os.getcwd(),"unzip") - if sys.platform == "win32": - print("Detected platform: Windows") - os.system("RMDIR "+ extract_path +" /s /q") - elif sys.platform == "darwin": - print("Detected platform: MacOS") - os.system("rm -rf "+ extract_path) - elif sys.platform == "linux" or sys.platform == "linux2": - print("Detected platform: Linux") - os.system("rm -rf "+ extract_path) - #Extract all RawData*.xml files and InstructionContainer the brml to temporary unzip file - with zipfile.ZipFile(file_name,"r") as brml: + +def read_brml(path, options=None): + + + required_options = ['extract_folder'] + default_options = { + 'extract_folder': 'temp', + 'save_folder': None + } + + + if not options: + options = default_options + + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + + + if not os.path.isdir(options['extract_folder']): + os.mkdir(options['extract_folder']) + + + # Extract the RawData0.xml file from the brml-file + with zipfile.ZipFile(path, 'r') as brml: for info in brml.infolist(): - if ("RawData" in info.filename) or ("InstructionContainer" in info.filename): - #if ("RawData" in info.filename): - brml.extract(info.filename, extract_path) -#***************************************************************************************************** -# For time counting, the number of days is initialized to 0. -# Compatibility fixes with D8 advance and older Discover: offsets, chi and tx, ty are initialized to 0 -#***************************************************************************************************** - # Initialize the number of days to 0 - n_day = 0. - # Initialize all offsets to 0 - off_tth = off_om = off_phi = off_chi = off_tx = off_ty = 0 - # Set Chi, tx and ty to 0 for D8 advance (July 2017 Julia Stroh) - chi = tx = ty = "0" -#***************************************************************************************************** -#Modification June 2017 (Duc Dinh) -#In some RawData.xml files, wavelength and static motors are missing. -#Find wl and static motors in MeasurementContainer.xml -#***************************************************************************************************** - data_path = os.path.join(extract_path, "*0","InstructionContainer.xml") - for file in sorted(glob.glob(data_path)): - tree = ET.parse(file) - root = tree.getroot() - for chain in root.findall("./ComparisonMethod/HrxrdAlignmentData"): - wl = chain.find("WaveLength").attrib["Value"] + if "RawData" in info.filename: + brml.extract(info.filename, temp) - for chain in root.findall("./ComparisonMethod/HrxrdAlignmentData/Data"): - if chain.get("LogicName") == "TwoTheta": - tth = chain.find("TheoreticalPosition").attrib["Value"] - off_tth = chain.find("PositionOffset").attrib["Value"] - tth = str(float(tth)-float(off_tth)) - if chain.get("LogicName") == "Theta": - om = chain.find("TheoreticalPosition").attrib["Value"] - off_om = chain.find("PositionOffset").attrib["Value"] - om = str(float(om)-float(off_om)) - if chain.get("LogicName") == "Chi": - chi = chain.find("TheoreticalPosition").attrib["Value"] - off_chi = chain.find("PositionOffset").attrib["Value"] - chi = str(float(chi)-float(off_chi)) + # Parse the RawData0.xml file + path = os.path.join(options['extract_folder'], 'RawData0.xml') - if chain.get("LogicName") == "Phi": - phi = chain.find("TheoreticalPosition").attrib["Value"] - off_phi = chain.find("PositionOffset").attrib["Value"] - phi = str(float(phi)-float(off_phi)) + tree = ET.parse(path) + root = tree.getroot() - if chain.get("LogicName") == "X": - tx = chain.find("TheoreticalPosition").attrib["Value"] - off_tx = chain.find("PositionOffset").attrib["Value"] - tx = str(float(tx)-float(off_tx)) + shutil.rmtree(options['extract_folder']) - if chain.get("LogicName") == "Y": - ty = chain.find("TheoreticalPosition").attrib["Value"] - off_ty = chain.find("PositionOffset").attrib["Value"] - ty = str(float(ty)-float(off_ty)) - os.remove(file) + diffractogram = [] - #Create ouput file - outfile = open("tmp", "w", encoding='utf8') # Create output data file - outfile.write("#temperature khi phi x y theta offset 2theta scanning motor intensity time\n") -#***************************************************************************************************** -# Finds scan type, wl, scanning motors and fixed motors values in RawData*.xml -#***************************************************************************************************** - data_path = os.path.join(extract_path, "*0","*.xml") #reading files in Experiment0 folder - for file in sorted(glob.glob(data_path), key=file_nb): - new_file = 0 - check_temperature = 0 - check_1Dmode = 0 - #parsing XML file - tree = ET.parse(file) - root = tree.getroot() - #obtain scan type - for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation") or root.findall("./ScanInformation")): - scan_type = chain.get("VisibleName") - if ("PSD" in scan_type) or ("Psd" in scan_type): - scan_type = "PSDFIXED" - if ("Coupled" in scan_type) or ("coupled" in scan_type) or ("2Theta-Omega" in scan_type): - scan_type = "COUPLED" - if ("Rocking" in scan_type) or ("rocking" in scan_type): - scan_type = "THETA" + for chain in root.findall('./DataRoutes/DataRoute'): + if chain.get('Description') == 'Originally measured data.': + for data in chain.findall('Datum'): + data = data.text.split(',') + twotheta, intensity = float(data[2]), float(data[3]) + + if twotheta > 0: + diffractogram.append({'2th': twotheta, 'I': intensity}) - # Check if temperature is recorded - for chain in (root.findall("./DataRoutes/DataRoute/DataViews/RawDataView/Recording") or root.findall("./DataViews/RawDataView/Recording")): - if "Temperature" in chain.get("LogicName"): - check_temperature = 1 + diffractogram = pd.DataFrame(diffractogram) - #Find wl in RawData.xml - for chain in root.findall("./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1"): - wl = chain.get("Value") - # Find the fast-scanning axis - for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation/ScanAxes/ScanAxisInfo") or root.findall("./ScanInformation/ScanAxes/ScanAxisInfo")): - if new_file == 0: - if chain.get("AxisName") == "TwoTheta": #Added offset correction / June 2017. Only relevant if offset in InstructionContainer. 0 otherwise. - off_scan = float(off_tth) - elif chain.get("AxisName") == "Theta": - off_scan = float(off_om) - else: - off_scan = 0 - step = chain.find("Increment").text - start = chain.find("Start").text - stop = chain.find("Stop").text - ref = chain.find("Reference").text - start = str(float(ref)+float(start)-off_scan) #Added offset correction / June 2017. - #start = str(float(ref)+float(start)) - new_file += 1 - # Find scanning motors - for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation/ScanAxes/ScanAxisInfo") or root.findall("./ScanInformation/ScanAxes/ScanAxisInfo")): - if chain.get("AxisName") == "TwoTheta": - tth = chain.find("Start").text - ref = chain.find("Reference").text - tth = str(float(ref)+float(tth)-float(off_tth)) #Added offset correction / June 2017. - #tth = str(float(ref)+float(tth)) + if options['save_folder']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - if chain.get("AxisName") == "Theta": - om = chain.find("Start").text - ref = chain.find("Reference").text - om = str(float(ref)+float(om)-float(off_om)) #Added offset correction / June 2017. - #om = str(float(ref)+float(om)) + diffractogram.to_csv(options['save_folder']) - if chain.get("AxisName") == "Chi": - chi = chain.find("Start").text - ref = chain.find("Reference").text - chi = str(float(ref)+float(chi)-float(off_chi)) #Added offset correction / June 2017. - #chi = str(float(ref)+float(chi)) - if chain.get("AxisName") == "Phi": - phi = chain.find("Start").text - ref = chain.find("Reference").text - phi = str(float(ref)+float(phi)-float(off_phi)) #Added offset correction / June 2017. - #phi = str(float(ref)+float(phi)) - if chain.get("AxisName") == "X": - tx = chain.find("Start").text - ref = chain.find("Reference").text - tx = str(float(ref)+float(tx)-float(off_tx)) #Added offset correction / June 2017. - #tx = str(float(ref)+float(tx)) + return diffractogram + - if chain.get("AxisName") == "Y": - ty = chain.find("Start").text - ref = chain.find("Reference").text - ty = str(float(ref)+float(ty)-float(off_ty)) #Added offset correction / June 2017. - #ty = str(float(ref)+float(ty)) - # Find static motors - for chain in root.findall("./FixedInformation/Drives/InfoData"): - if chain.get("LogicName") == "TwoTheta": - tth = chain.find("Position").attrib["Value"] - tth = str(float(tth)-float(off_tth)) #Added offset correction / June 2017. - if chain.get("LogicName") == "Theta": - om = chain.find("Position").attrib["Value"] - om = str(float(om)-float(off_om)) #Added offset correction / June 2017. - - if chain.get("LogicName") == "Chi": - chi = chain.find("Position").attrib["Value"] - chi = str(float(chi)-float(off_chi)) #Added offset correction / June 2017. - - if chain.get("LogicName") == "Phi": - phi = chain.find("Position").attrib["Value"] - phi = str(float(phi)-float(off_phi)) #Added offset correction / June 2017. - - if chain.get("LogicName") == "X": - tx = chain.find("Position").attrib["Value"] - tx = str(float(tx)-float(off_tx)) #Added offset correction / June 2017. - - if chain.get("LogicName") == "Y": - ty = chain.find("Position").attrib["Value"] - ty = str(float(ty)-float(off_ty)) #Added offset correction / June 2017. - - offset = str(float(om) - float(tth)/2.) - -#***************************************************************************************************** -# This section computes scanning time, scanning angular range and scanning speed -# in order to convert 2th values to time values (July 2017, Julia Stroh) -#***************************************************************************************************** - for chain in (root.findall("./TimeStampStarted")): - d_start = ((chain.text).split("T")[0]).split("-")[2] - t_start = ((chain.text).split("T")[1]).split("+")[0] - h_start, min_start, sec_start = t_start.split(":") - t_start = float(h_start)*3600 + float(min_start)*60 + float(sec_start) - - if file_nb(file)==0: - abs_start = t_start - for chain in (root.findall("./TimeStampFinished")): - d_stop = ((chain.text).split("T")[0]).split("-")[2] - t_stop = ((chain.text).split("T")[1]).split("+")[0] - h_stop, min_stop, sec_stop = t_stop.split(":") - t_stop = float(h_stop)*3600 + float(min_stop)*60 + float(sec_stop) - - # Check if detector is in 1D mode - for chain in (root.findall("./DataRoutes/DataRoute/ScanInformation") or root.findall("./ScanInformation")): - if chain.find("TimePerStep").text != chain.find("TimePerStepEffective").text: - check_1Dmode = 1 - - # Check if day changed between start and stop and correct accordingly - if d_stop != d_start: - t_stop += 24*3600. - total_scan_time = t_stop - t_start - - #scanning range - dth_scan = float(stop)-float(start) - #psd range - dth_psd = 0 - for chain in root.findall("./FixedInformation/Detectors/InfoData/AngularOpening"): - dth_psd = chain.get("Value") - total_dth = float(dth_psd)*check_1Dmode+float(dth_scan) - - scan_speed = total_dth / total_scan_time -#***************************************************************************************************** -# Finds intensity values. If temperature is recorded, also fin temperature values. -# The intensity data is formatted differently in PSDfixed mode and when temperature is recorded -#***************************************************************************************************** - if "PSDFIXED" in scan_type: - if check_temperature == 0: - for chain in (root.findall("./DataRoutes/DataRoute") or root.findall("./")): - intensity = (chain.find("Datum").text).split(',') - - for chain in (root.findall("./DataRoutes/DataRoute/DataViews/RawDataView/Recording") or root.findall("./DataViews/RawDataView/Recording")): - if chain.get("LogicName") == "Counter1D": - n_channels = int(chain.find("Size/X").text) - - line_count = 0 - int_shift = len(intensity) - n_channels - for i in range(n_channels): #the intensity values are shifted to the right by int_shift - if i == 0: - scanning = float(start) - else: - scanning += float(step) - line_count += 1 - t_2th = (t_start+n_day*24*3600 - abs_start)+((float(dth_psd)*check_1Dmode + scanning - float(start)) / scan_speed) - outfile.write("25" + " " + (chi) + " " + (phi) - + " " + (tx) + " " + (ty) + " " + (om) - + " " + (offset) + " " + (tth) + " " + str(scanning) - + " " + intensity[i+int_shift] +" " + str(t_2th) +'\n') - else: - return implementation_warning, 0, 0 - - #if "COUPLED" in scan_type: - # to do check in brml that all scans (except psd fixed) share the same data structure (wrt temperature) - else: - if check_temperature == 0: - line_count = 0 - for chain in (root.findall("./DataRoutes/DataRoute/Datum") or root.findall("./Datum")): - if line_count == 0: - scanning = float(start) - else: - scanning += float(step) - line_count += 1 - intensity = (chain.text).split(',')[-1] - #compute time corresponding to scanning angle (July 2017) - t_2th = (t_start+n_day*24*3600 - abs_start)+((float(dth_psd)*check_1Dmode + scanning - float(start)) / scan_speed) - outfile.write("25" + " " + (chi) + " " + (phi) - + " " + (tx) + " " + (ty) + " " + (om) - + " " + (offset) + " " + (tth) + " " + str(round(scanning, 4)) - + " " + intensity + " " + str(t_2th) +'\n') - else: - line_count = 0 - for chain in (root.findall("./DataRoutes/DataRoute/Datum") or root.findall("./Datum")): - if line_count == 0: - scanning = float(start) - else: - scanning += float(step) - line_count += 1 - t_2th = (t_start+n_day*24*3600 - abs_start)+((float(dth_psd)*check_1Dmode + scanning - float(start)) / scan_speed) - intensity = (chain.text).split(',')[-2] - temperature = (chain.text).split(',')[-1] - outfile.write(temperature + " " + (chi) + " " + (phi) - + " " + (tx) + " " + (ty) + " " + (om) - + " " + (offset) + " " + (tth) + " " + str(round(scanning, 4)) - + " " + intensity + " " + str(t_2th) +'\n') - - if d_stop != d_start: - n_day+=1 - outfile.close() - if sys.platform == "win32": - os.system("RMDIR "+ extract_path +" /s /q") - elif sys.platform == "darwin": - os.system("rm -rf "+ extract_path) - elif sys.platform == "linux" or sys.platform == "linux2": - os.system("rm -rf "+ extract_path) - return scan_type, line_count, wl \ No newline at end of file + + + From d95b670af70b9bba764b573484b3d98c1db99692 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 28 Oct 2021 16:47:07 +0200 Subject: [PATCH 061/355] Corrected some bugs --- beamtime/xrd/io.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 38220cd..1fffdcb 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -5,9 +5,6 @@ import os import shutil import zipfile -import io -import sys -import glob import xml.etree.ElementTree as ET @@ -108,14 +105,22 @@ def open_1d_data(path, options=None): +def generate_image_list(path, options=None): + ''' Generates a list of paths to pass to the average_images() function''' + + required_options = ['scans_per_image'] + default_options = { + 'scans_per_image': 5 + } + + def average_images(images): ''' Takes a list of path to image files, reads them and averages them before returning the average image''' - image_arrays = [] for image in images: - image_array = xrd.io.get_image_array(os.path.join(root, image)) + image_array = xrd.io.get_image_array(image) image_arrays.append(image_array) @@ -178,12 +183,12 @@ def read_brml(path, options=None): with zipfile.ZipFile(path, 'r') as brml: for info in brml.infolist(): if "RawData" in info.filename: - brml.extract(info.filename, temp) + brml.extract(info.filename, options['extract_folder']) # Parse the RawData0.xml file - path = os.path.join(options['extract_folder'], 'RawData0.xml') + path = os.path.join(options['extract_folder'], 'Experiment0/RawData0.xml') tree = ET.parse(path) root = tree.getroot() From 92fb8988fb0b2b8cccb53b7b918b7242fbfdf295 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 31 Oct 2021 09:58:18 +0100 Subject: [PATCH 062/355] Add differentiation between stillscan and 2th scan --- beamtime/xrd/io.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 1fffdcb..c5e5966 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -16,6 +16,13 @@ def get_image_array(path): return image_array +def get_image_headers(path): + + image = fabio.open(path) + + return image.header + + def integrate_1d(calibrant, bins, path=None, image=None, options=None): ''' Integrates an image file to a 1D diffractogram. @@ -198,13 +205,25 @@ def read_brml(path, options=None): diffractogram = [] for chain in root.findall('./DataRoutes/DataRoute'): - if chain.get('Description') == 'Originally measured data.': - for data in chain.findall('Datum'): - data = data.text.split(',') - twotheta, intensity = float(data[2]), float(data[3]) - - if twotheta > 0: - diffractogram.append({'2th': twotheta, 'I': intensity}) + + for scantype in chain.findall('ScanInformation/ScanMode'): + if scantype.text == 'StillScan': + + if chain.get('Description') == 'Originally measured data.': + for data in chain.findall('Datum'): + data = data.text.split(',') + data = [float(i) for i in data] + twotheta, intensity = float(data[2]), float(data[3]) + + + else: + if chain.get('Description') == 'Originally measured data.': + for data in chain.findall('Datum'): + data = data.text.split(',') + twotheta, intensity = float(data[2]), float(data[3]) + + if twotheta > 0: + diffractogram.append({'2th': twotheta, 'I': intensity}) diffractogram = pd.DataFrame(diffractogram) From b8cd8e81af88d624830a1632ac55a72d3fe5e168 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 5 Nov 2021 20:13:22 +0100 Subject: [PATCH 063/355] Add splice_cycles to electrochemistry module --- beamtime/electrochemistry/io.py | 80 ++++++++++++++++++++++++------- beamtime/electrochemistry/plot.py | 3 +- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/beamtime/electrochemistry/io.py b/beamtime/electrochemistry/io.py index 510be04..1963253 100644 --- a/beamtime/electrochemistry/io.py +++ b/beamtime/electrochemistry/io.py @@ -111,7 +111,7 @@ def process_batsmall_data(df, options=None): ''' required_options = ['splice_cycles', 'molecular_weight', 'reverse_discharge', 'units'] - default_options = {'splice_cycles': None, 'molecular_weight': None, 'reverse_discharge': False, 'units': None} + default_options = {'splice_cycles': False, 'molecular_weight': None, 'reverse_discharge': False, 'units': None} if not options: options = default_options @@ -128,6 +128,10 @@ def process_batsmall_data(df, options=None): options['units'] = new_units + + if options['splice_cycles']: + df = splice_cycles(df=df, kind='batsmall') + # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current df[["comment"]] = df[["comment"]].fillna(value={'comment': ''}) df = df[df["comment"].str.contains("program")==False] @@ -142,7 +146,7 @@ def process_batsmall_data(df, options=None): # Loop through all the cycling steps, change the current and capacities in the for i in range(df["count"].max()): - sub_df = df.loc[df['count'] == i].copy() + sub_df = df.loc[df['count'] == i+1].copy() sub_df.loc[dchg_mask, 'current'] *= -1 sub_df.loc[dchg_mask, 'specific_capacity'] *= -1 @@ -174,6 +178,61 @@ def process_batsmall_data(df, options=None): return cycles +def splice_cycles(df, kind): + + if kind == 'batsmall': + + # Creates masks for charge and discharge curves + chg_mask = df['current'] >= 0 + dchg_mask = df['current'] < 0 + + # Get the number of cycles in the dataset + max_count = df["count"].max() + + # Loop through all the cycling steps, change the current and capacities in the + for i in range(df["count"].max()): + sub_df = df.loc[df['count'] == i+1] + sub_df_chg = sub_df.loc[chg_mask] + #sub_df_dchg = sub_df.loc[dchg_mask] + + # get indices where the program changed + chg_indices = sub_df_chg[sub_df_chg["comment"].str.contains("program")==True].index.to_list() + + # Delete first item if first cycle after rest (this will just be the start of the cycling) + if i+1 == 1: + del chg_indices[0] + + + if chg_indices: + last_chg = chg_indices.pop() + + + #dchg_indices = sub_df_dchg[sub_df_dchg["comment"].str.contains("program")==True].index.to_list() + #if dchg_indices: + # del dchg_indices[0] + + + + if chg_indices: + for i in chg_indices: + add = df['specific_capacity'].iloc[i-1] + df['specific_capacity'].iloc[i:last_chg] = df['specific_capacity'].iloc[i:last_chg] + add + + #if dchg_indices: + # for i in dchg_indices: + # add = df['specific_capacity'].iloc[i-1] + # df['specific_capacity'].iloc[i:last_dchg] = df['specific_capacity'].iloc[i:last_dchg] + add + + + + + return df + + + + + + def process_neware_data(df, options=None): """ Takes data from NEWARE in a DataFrame as read by read_neware() and converts units, adds columns and splits into cycles. @@ -217,7 +276,7 @@ def process_neware_data(df, options=None): # Loop through all the cycling steps, change the current and capacities in the for i in range(df["cycle"].max()): - sub_df = df.loc[df['cycle'] == i].copy() + sub_df = df.loc[df['cycle'] == i+1].copy() #sub_df.loc[dchg_mask, 'current'] *= -1 #sub_df.loc[dchg_mask, 'capacity'] *= -1 @@ -516,21 +575,6 @@ def convert_datetime_string(datetime_string, reference, unit='s'): return time -def splice_cycles(first, second): - - first_chg = first[0] - first_dchg = first[1] - first - - second_chg = second[0] - second_dchg = second[1] - - chg_df = first[0].append(second[0]) - - return True - - - diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py index 13bb1e1..ef1efc8 100644 --- a/beamtime/electrochemistry/plot.py +++ b/beamtime/electrochemistry/plot.py @@ -315,7 +315,8 @@ def prettify_gc_plot(fig, ax, options=None): ############################# LEGEND ############################# ################################################################## - ax.get_legend().remove() + if ax.get_legend(): + ax.get_legend().remove() return fig, ax From 2b1b9b0d9b3bdec4c5e5b70fc6ec9d58dc1958e2 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 8 Nov 2021 17:24:58 +0100 Subject: [PATCH 064/355] Add plot functionality for single diffractograms --- beamtime/electrochemistry/plot.py | 4 +- beamtime/xrd/io.py | 54 +++-- beamtime/xrd/plot.py | 327 +++++++++++++++++++++++++++++- 3 files changed, 363 insertions(+), 22 deletions(-) diff --git a/beamtime/electrochemistry/plot.py b/beamtime/electrochemistry/plot.py index ef1efc8..64b7115 100644 --- a/beamtime/electrochemistry/plot.py +++ b/beamtime/electrochemistry/plot.py @@ -275,8 +275,8 @@ def prettify_gc_plot(fig, ax, options=None): # Otherwise apply user input else: - major_xtick = options['yticks'][0] - minor_xtick = options['yticks'][1] + major_ytick = options['yticks'][0] + minor_ytick = options['yticks'][1] # Apply values diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index c5e5966..2afe41c 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -90,25 +90,7 @@ def integrate_1d(calibrant, bins, path=None, image=None, options=None): return open_1d_data(filename) -def open_1d_data(path, options=None): - with open(path, 'r') as f: - position = 0 - - current_line = f.readline() - - while current_line[0] == '#': - position = f.tell() - current_line = f.readline() - - f.seek(position) - - df = pd.read_csv(f, header=None, delim_whitespace=True) - - df.columns = ['2th', 'I'] - - - return df @@ -165,7 +147,7 @@ def view_integrator(calibrant): def read_brml(path, options=None): - required_options = ['extract_folder'] + required_options = ['extract_folder', 'save_folder'] default_options = { 'extract_folder': 'temp', 'save_folder': None @@ -240,9 +222,43 @@ def read_brml(path, options=None): return diffractogram +def read_diffractogram(path, options=None): + + with open(path, 'r') as f: + position = 0 + + current_line = f.readline() + + while current_line[0] == '#': + position = f.tell() + current_line = f.readline() + f.seek(position) + + diffractogram = pd.read_csv(f, header=None, delim_whitespace=True) + + diffractogram.columns = ['2th', 'I'] + + + return diffractogram + + +def read_data(path, kind, options=None): + + if kind == 'beamline': + diffractogram = read_diffractogram(path, options=options) + + elif kind == 'recx': + diffractogram = read_brml(path, options=options) + + elif kind == 'image': + diffractogram = get_image_array(path) + + + return diffractogram + diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index e37c7b8..79feffa 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -1,2 +1,327 @@ import matplotlib.pyplot as plt -import numpy +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) + +import pandas as pd +import numpy as np +import math + +import beamtime.xrd as xrd + + + + +def plot_diffractogram(path, kind, options=None): + + # Prepare plot, and read and process data + fig, ax = prepare_diffractogram_plot(options=options) + diffractogram = xrd.io.read_data(path=path, kind=kind, options=options) + + + # Update options + required_options = ['x_vals', 'y_vals', 'scatter'] + + + + default_options = { + 'x_vals': '2th', + 'y_vals': 'I', + 'scatter': False + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + if options['scatter']: + diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, kind='scatter') + + else: + diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax) + + + + fig, ax = prettify_diffractogram_plot(fig=fig, ax=ax, options=options) + + + return diffractogram, fig, ax + + + +def prepare_diffractogram_plot(options=None): + # First take care of the options for plotting - set any values not specified to the default values + required_options = ['columns', 'width', 'height', 'format', 'dpi', 'facecolor'] + default_options = {'columns': 1, 'width': 14, 'format': 'golden_ratio', 'dpi': None, 'facecolor': 'w'} + + + # Define the required sizes + required_sizes = ['lines', 'axes'] + + + + + # If none are set at all, just pass the default_options + if not options: + options = default_options + options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 + options['figsize'] = (options['width'], options['height']) + + + # Define default sizes + default_sizes = { + 'lines': 3*options['columns'], + 'axes': 3*options['columns'] + } + + # Initialise dictionary if it doesn't exist + if not 'sizes' in options.keys(): + options['sizes'] = {} + + + # Update dictionary with default values where none is supplied + for size in required_sizes: + if size not in options['sizes']: + options['sizes'][size] = default_sizes[size] + + + # If options is passed, go through to fill out the rest. + else: + # Start by setting the width: + if 'width' not in options.keys(): + options['width'] = default_options['width'] + + # Then set height - check options for format. If not given, set the height to the width scaled by the golden ratio - if the format is square, set the same. This should possibly allow for the tweaking of custom ratios later. + if 'height' not in options.keys(): + if 'format' not in options.keys(): + options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 + elif options['format'] == 'square': + options['height'] = options['width'] + + options['figsize'] = (options['width'], options['height']) + + # After height and width are set, go through the rest of the options to make sure that all the required options are filled + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + fig, ax = plt.subplots(figsize=(options['figsize']), dpi=options['dpi'], facecolor=options['facecolor']) + + + plt.rc('lines', linewidth=options['sizes']['lines']) + plt.rc('axes', linewidth=options['sizes']['axes']) + + return fig, ax + +def prettify_diffractogram_plot(fig, ax, options=None): + + ################################################################## + ######################### UPDATE OPTIONS ######################### + ################################################################## + + # Define the required options + required_options = [ + 'columns', + 'xticks', 'yticks', + 'units', + 'show_major_ticks', 'show_minor_ticks', 'show_tick_labels', + 'xlim', 'ylim', + 'hide_x_axis', 'hide_y_axis', + 'positions', + 'xlabel', 'ylabel', + 'sizes', + 'title' + ] + + + # Define the default options + default_options = { + 'columns': 1, + 'xticks': [10, 5], 'yticks': [10000, 5000], + 'units': {'2th': '$^o$', 'I': 'arb. u.'}, + 'show_major_ticks': [True, False, True, False], 'show_minor_ticks': [True, False, True, False], 'show_tick_labels': [True, False, False, False], + 'xlim': None,'ylim': None, + 'hide_x_axis': False, 'hide_y_axis': False, + 'positions': {'xaxis': 'bottom', 'yaxis': 'left'}, + 'xlabel': None, 'ylabel': None, + 'sizes': None, + 'title': None + } + + options = update_options(options, required_options, default_options) + + + + ################################################################## + ########################## DEFINE SIZES ########################## + ################################################################## + + # Define the required sizes + required_sizes = [ + 'labels', + 'legend', + 'title', + 'line', 'axes', + 'tick_labels', + 'major_ticks', 'minor_ticks'] + + + + # Define default sizes + default_sizes = { + 'labels': 30*options['columns'], + 'legend': 30*options['columns'], + 'title': 30*options['columns'], + 'line': 3*options['columns'], + 'axes': 3*options['columns'], + 'tick_labels': 30*options['columns'], + 'major_ticks': 20*options['columns'], + 'minor_ticks': 10*options['columns'] + } + + # Initialise dictionary if it doesn't exist + if not options['sizes']: + options['sizes'] = {} + + + # Update dictionary with default values where none is supplied + for size in required_sizes: + if size not in options['sizes']: + options['sizes'][size] = default_sizes[size] + + + ################################################################## + ########################## AXIS LABELS ########################### + ################################################################## + + + if not options['xlabel']: + options['xlabel'] = prettify_labels(options['x_vals']) + ' [{}]'.format(options['units'][options['x_vals']]) + + else: + options['xlabel'] = options['xlabel'] + ' [{}]'.format(options['units'][options['x_vals']]) + + + if not options['ylabel']: + options['ylabel'] = prettify_labels(options['y_vals']) + ' [{}]'.format(options['units'][options['y_vals']]) + + else: + options['ylabel'] = options['ylabel'] + ' [{}]'.format(options['units'][options['y_vals']]) + + ax.set_xlabel(options['xlabel'], size=options['sizes']['labels']) + ax.set_ylabel(options['ylabel'], size=options['sizes']['labels']) + + ################################################################## + ###################### TICK MARKS & LABELS ####################### + ################################################################## + + ax.tick_params( + direction='in', + which='major', + bottom=options['show_major_ticks'][0], labelbottom=options['show_tick_labels'][0], + left=options['show_major_ticks'][1], labelleft=options['show_tick_labels'][1], + top=options['show_major_ticks'][2], labeltop=options['show_tick_labels'][2], + right=options['show_major_ticks'][3], labelright=options['show_tick_labels'][3], + length=options['sizes']['major_ticks'], + width=options['sizes']['axes']) + + + ax.tick_params(direction='in', which='minor', bottom=options['show_minor_ticks'][0], left=options['show_minor_ticks'][1], top=options['show_minor_ticks'][2], right=options['show_minor_ticks'][3], length=options['sizes']['minor_ticks'], width=options['sizes']['axes']) + + + + if options['positions']['yaxis'] == 'right': + ax.yaxis.set_label_position("right") + ax.yaxis.tick_right() + + + if options['hide_x_axis']: + ax.axes.xaxis.set_visible(False) + + if options['hide_y_axis']: + ax.axes.yaxis.set_visible(False) + + + + # Otherwise apply user input + if options['xticks']: + major_xtick = options['xticks'][0] + minor_xtick = options['xticks'][1] + + + if options['yticks']: + + major_ytick = options['yticks'][0] + minor_ytick = options['yticks'][1] + + + # Apply values + ax.xaxis.set_major_locator(MultipleLocator(major_xtick)) + ax.xaxis.set_minor_locator(MultipleLocator(minor_xtick)) + + ax.yaxis.set_major_locator(MultipleLocator(major_ytick)) + ax.yaxis.set_minor_locator(MultipleLocator(minor_ytick)) + + + + + # SET FONTSIZE OF TICK LABELS + + plt.xticks(fontsize=options['sizes']['tick_labels']) + plt.yticks(fontsize=options['sizes']['tick_labels']) + + ################################################################## + ########################## AXES LIMITS ########################### + ################################################################## + + if options['xlim']: + plt.xlim(options['xlim']) + + if options['ylim']: + plt.ylim(options['ylim']) + + ################################################################## + ############################# TITLE ############################## + ################################################################## + + if options['title']: + ax.set_title(options['title'], size=options['sizes']['title']) + + ################################################################## + ############################# LEGEND ############################# + ################################################################## + + if ax.get_legend(): + ax.get_legend().remove() + + return fig, ax + + + +def prettify_labels(label): + + labels_dict = { + '2th': '2$\\theta$', + 'I': 'Intensity' + } + + return labels_dict[label] + +#def plot_diffractograms(): + + + + +#def plot_heatmap(): + + + + + +def update_options(options, required_options, default_options): + + if not options: + options = default_options + + else: + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + return options \ No newline at end of file From b4a8eb5eec4d9b00e5319592f3878cdf350253a0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 10 Nov 2021 13:49:10 +0100 Subject: [PATCH 065/355] Add plotting of multiple diffractograms --- beamtime/xrd/plot.py | 56 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 79feffa..23b143b 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -303,10 +303,64 @@ def prettify_labels(label): return labels_dict[label] -#def plot_diffractograms(): +def plot_diffractograms(paths, kind, options=None): + fig, ax = prepare_diffractogram_plot(options=options) + diffractograms = [] + + for path in paths: + diffractogram = xrd.io.read_data(path=path, kind=kind, options=options) + diffractograms.append(diffractogram) + + + required_options = ['type', 'xvals', 'yvals', 'x_offset', 'y_offset', 'normalise', 'normalise_around', 'reverse_order'] + default_options = { + 'type': 'stacked', + 'xvals': '2th', + 'yvals': 'I', + 'x_offset': 0, + 'y_offset': 0.2, + 'normalise': True, + 'normalise_around': None, + 'reverse_order': False + } + + + # If reverse_order is enabled, reverse the order + if options['reverse_order']: + diffractograms = reverse_diffractograms(diffractograms) + + + # If normalise is enbaled, normalise all the diffractograms + if options['normalise']: + if not options['normalise_around']: + for diffractogram in diffractograms: + diffractogram["I"] = diffractogram["I"]/diffractogram["I"].max() + else: + diffractogram["I"] = diffractogram["I"]/diffractogram["I"].loc[(diffractogram['2th'] > options['normalise_around'][0]) & (diffractogram['2th'] < options['normalise_around'][1])].max() + + + if options['type'] == 'stacked': + for diffractogram in diffractograms: + diffractogram.plot(x=options['xvals'], y=options['yvals'], ax=ax) + + + fig, ax = prettify_diffractogram_plot(fig=fig, ax=ax, options=options) + + + return diffractogram, fig, ax + + +def reverse_diffractograms(diffractograms): + + rev_diffractograms = [] + + for i in len(diffractograms): + rev_diffractograms.append(diffractograms.pop()) + + return rev_diffractograms #def plot_heatmap(): From cffc0a8f6a5732bfa97dd6bf7545f27d4319a7d5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 11 Mar 2022 10:00:29 +0100 Subject: [PATCH 066/355] Add aux and plotting functions --- beamtime/auxillary.py | 42 +++++++ beamtime/plotting.py | 278 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 beamtime/auxillary.py create mode 100644 beamtime/plotting.py diff --git a/beamtime/auxillary.py b/beamtime/auxillary.py new file mode 100644 index 0000000..76ec551 --- /dev/null +++ b/beamtime/auxillary.py @@ -0,0 +1,42 @@ +import json + +def update_options(options, required_options, default_options): + ''' Takes a dictionary of options along with a list of required options and dictionary of default options, and sets all keyval-pairs of options that is not already defined to the default values''' + + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + + return options + +def save_options(options, path): + ''' Saves any options dictionary to a JSON-file in the specified path''' + + with open(path, 'w') as f: + json.dump(options,f) + + +def load_options(path): + ''' Loads JSON-file into a dictionary''' + + with open(path, 'r') as f: + options = json.load(f) + + return(options) + + + +def swap_values(dict, key1, key2): + + key1_val = dict[key1] + dict[key1] = dict[key2] + dict[key2] = key1_val + + return dict + + + +def hello_world2(a=1, b=2): + + print(f'Halla, MAFAKKAS! a = {a} og b = {b}') \ No newline at end of file diff --git a/beamtime/plotting.py b/beamtime/plotting.py new file mode 100644 index 0000000..354b220 --- /dev/null +++ b/beamtime/plotting.py @@ -0,0 +1,278 @@ +import beamtime.auxillary as aux +import matplotlib.pyplot as plt +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) +from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition, mark_inset) +import importlib +import matplotlib.patches as mpatches +from matplotlib.lines import Line2D +import matplotlib.lines as mlines +from cycler import cycler +import itertools + + + +def prepare_plot(options={}): + ''' Prepares plot based on contents of options['rc_params'] and options['format_params']. + + rc_params is a dictionary with keyval-pairs corresponding to rcParams in matplotlib + + format_params will determine the size and aspect ratios of ''' + + rc_params = options['rc_params'] + format_params = options['format_params'] + + required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + + default_options = { + 'single_column_width': 8.3, + 'double_column_width': 17.1, + 'column_type': 'single', + 'width_ratio': '1:1', + 'aspect_ratio': '1:1', + 'compress_width': 1, + 'compress_height': 1, + 'upscaling_factor': 1.0, + 'dpi': 600, + } + + options = aux.update_options(format_params, required_options, default_options) + + + # Reset run commands + plt.rcdefaults() + + # Update run commands if any is passed (will pass an empty dictionary if not passed) + update_rc_params(rc_params) + + width = determine_width(options) + height = determine_height(options, width) + width, height = scale_figure(options=options, width=width, height=height) + + fig, ax = plt.subplots(figsize=(width, height), dpi=options['dpi']) + + return fig, ax + + +def prettify__plot(fig, ax, plot_data, options): + + required_options = ['plot_kind', 'hide_x_labels', 'hide_y_labels', 'rotation_x_ticks', 'rotation_y_ticks', 'xlabel', 'ylabel', 'yunit', 'xlim', 'ylim', 'x_tick_locators', 'y_tick_locators', 'xticks', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', + 'colours', 'palettes', 'title', 'legend', 'legend_position', 'subplots_adjust', 'text', 'legend_ncol'] + + default_options = { + 'plot_kind': None, # defaults to None, but should be utilised when + 'hide_x_labels': False, # Whether x labels should be hidden + 'hide_x_ticklabels': False, + 'hide_x_ticks': False, + 'rotation_x_ticks': 0, + 'hide_y_labels': False, # whether y labels should be hidden + 'hide_y_ticklabels': False, + 'hide_y_ticks': False, + 'rotation_y_ticks': 0, + 'xlabel': r'$x$ in Na$_{5-x}$FeO$_{4-\delta}$', + 'ylabel': 'Formation energy', + 'yunit': r'eV', # The unit of the y-values in the curve and bar plots + 'xlim': None, + 'ylim': None, + 'x_tick_locators': [.5, .25], # Major and minor tick locators + 'y_tick_locators': [.5, .25], + 'xticks': None, + 'labels': None, + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'title': None, + 'legend': False, + 'legend_position': ['lower center', (0.5, -0.1)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'legend_ncol': 1, + 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], + 'text': None + } + + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + # Set labels on x- and y-axes + if not options['hide_y_labels']: + ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + else: + ax.set_ylabel('') + + if not options['hide_x_labels']: + ax.set_xlabel(f'{options["xlabel"]}') + else: + ax.set_xlabel('') + + + # Set multiple locators + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + + ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) + ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + + if options['xticks']: + ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) + ax.set_xticklabels(options['xticks']) + else: + ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) + ax.set_xticklabels([x/2 for x in np.arange(plot_data['start'], plot_data['end']+1)]) + + # Hide x- and y- ticklabels + if options['hide_y_ticklabels']: + ax.tick_params(axis='y', direction='in', which='both', labelleft=False, labelright=False) + else: + plt.xticks(rotation=options['rotation_x_ticks']) + #ax.set_xticklabels(ax.get_xticks(), rotation = options['rotation_x_ticks']) + + if options['hide_x_ticklabels']: + ax.tick_params(axis='x', direction='in', which='both', labelbottom=False, labeltop=False) + else: + pass + #ax.set_yticklabels(ax.get_yticks(), rotation = options['rotation_y_ticks']) + + + # Hide x- and y-ticks: + if options['hide_y_ticks']: + ax.tick_params(axis='y', direction='in', which='both', left=False, right=False) + if options['hide_x_ticks']: + ax.tick_params(axis='x', direction='in', which='both', bottom=False, top=False) + + + + # Set title + if options['title']: + ax.set_title(options['title'], fontsize=plt.rcParams['font.size']) + + + + # Create legend + + if ax.get_legend(): + ax.get_legend().remove() + + + if options['legend']: + + + # Make palette and linestyles from original parameters + if not options['colours']: + colours = generate_colours(palettes=options['palettes']) + else: + colours = itertools.cycle(options['colours']) + + + markers = itertools.cycle(options['markers']) + + # Create legend + active_markers = [] + active_labels = [] + + for label in options['labels']: + + + # Discard next linestyle and colour if label is _ + if label == '_': + _ = next(colours) + _ = next(markers) + + else: + active_markers.append(mlines.Line2D([], [], markeredgecolor=next(colours), color=(1, 1, 1, 0), marker=next(markers))) + active_labels.append(label) + + + + ax.legend(active_markers, active_labels, frameon=False, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1], ncol=options['legend_ncol']) + #fig.legend(handles=patches, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1], frameon=False) + + + + # Adjust where the axes start within the figure. Default value is 10% in from the left and bottom edges. Used to make room for the plot within the figure size (to avoid using bbox_inches='tight' in the savefig-command, as this screws with plot dimensions) + plt.subplots_adjust(left=options['subplots_adjust'][0], bottom=options['subplots_adjust'][1], right=options['subplots_adjust'][2], top=options['subplots_adjust'][3]) + + + # If limits for x- and y-axes is passed, sets these. + if options['xlim'] is not None: + ax.set_xlim(options['xlim']) + + if options['ylim'] is not None: + ax.set_ylim(options['ylim']) + + + # Add custom text + if options['text']: + plt.text(x=options['text'][1][0], y=options['text'][1][1], s=options['text'][0]) + + return fig, ax + + + + +def ipywidgets_update(func, plot_data, options={}, **kwargs): + + for key in kwargs: + options[key] = kwargs[key] + + func(plot_data=plot_data, options=options) + + + + +def determine_width(options): + + conversion_cm_inch = 0.3937008 # cm to inch + + if options['column_type'] == 'single': + column_width = options['single_column_width'] + elif options['column_type'] == 'double': + column_width = options['double_column_width'] + + column_width *= conversion_cm_inch + + + width_ratio = [float(num) for num in options['width_ratio'].split(':')] + + + width = column_width * width_ratio[0]/width_ratio[1] + + + return width + + +def determine_height(options, width): + + aspect_ratio = [float(num) for num in options['aspect_ratio'].split(':')] + + height = width/(aspect_ratio[0] / aspect_ratio[1]) + + return height + + +def scale_figure(options, width, height): + width = width * options['upscaling_factor'] * options['compress_width'] + height = height * options['upscaling_factor'] * options['compress_height'] + + return width, height + + + +def update_rc_params(rc_params): + ''' Update all passed run commands in matplotlib''' + + if rc_params: + for key in rc_params.keys(): + plt.rcParams.update({key: rc_params[key]}) + + +def generate_colours(palettes): + + # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. + colour_collection = [] + for palette in palettes: + mod = importlib.import_module("palettable.colorbrewer.%s" % palette[0]) + colour = getattr(mod, palette[1]).mpl_colors + colour_collection = colour_collection + colour + + colour_cycle = itertools.cycle(colour_collection) + + + return colour_cycle \ No newline at end of file From e378e63971e80c16384fe9d930c8ab82e62ade1c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 11 Mar 2022 10:00:44 +0100 Subject: [PATCH 067/355] Add updated requirements --- beamtime/reqirements2.txt | 141 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 beamtime/reqirements2.txt diff --git a/beamtime/reqirements2.txt b/beamtime/reqirements2.txt new file mode 100644 index 0000000..5cef354 --- /dev/null +++ b/beamtime/reqirements2.txt @@ -0,0 +1,141 @@ +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: win-64 +argon2-cffi=21.3.0=pyhd3eb1b0_0 +argon2-cffi-bindings=21.2.0=py39h2bbff1b_0 +attrs=21.4.0=pyhd3eb1b0_0 +backcall=0.2.0=pyhd3eb1b0_0 +beamtime=0.1=pypi_0 +blas=1.0=mkl +bleach=4.1.0=pyhd3eb1b0_0 +bottleneck=1.3.2=py39h7cc1a96_1 +ca-certificates=2022.2.1=haa95532_0 +cached-property=1.5.2=hd8ed1ab_1 +cached_property=1.5.2=pyha770c72_1 +certifi=2021.10.8=py39haa95532_2 +cffi=1.15.0=py39h2bbff1b_1 +colorama=0.4.4=pyhd3eb1b0_0 +cycler=0.10.0=py_2 +debugpy=1.4.1=py39hd77b12b_0 +decorator=5.1.0=pyhd3eb1b0_0 +defusedxml=0.7.1=pyhd3eb1b0_0 +entrypoints=0.3=py39haa95532_0 +fabio=0.12.0=py39h5d4886f_0 +freetype=2.10.4=h546665d_1 +glymur=0.9.4=pyhd8ed1ab_0 +h5py=3.2.1=nompi_py39hf27771d_100 +hdf5=1.10.6=nompi_h5268f04_1114 +hdf5plugin=3.1.1=py39h71586dd_0 +icc_rt=2019.0.0=h0cc432a_1 +icu=68.1=h6c2663c_0 +importlib-metadata=4.8.2=py39haa95532_0 +importlib_metadata=4.8.2=hd3eb1b0_0 +intel-openmp=2021.3.0=haa95532_3372 +ipykernel=6.4.1=py39haa95532_1 +ipython=7.27.0=py39hd4e2768_0 +ipython_genutils=0.2.0=pyhd3eb1b0_1 +ipywidgets=7.6.5=pyhd3eb1b0_1 +jbig=2.1=h8d14728_2003 +jedi=0.18.0=py39haa95532_1 +jinja2=3.0.2=pyhd3eb1b0_0 +jpeg=9d=h2bbff1b_0 +jsonschema=3.2.0=pyhd3eb1b0_2 +jupyter_client=7.0.1=pyhd3eb1b0_0 +jupyter_core=4.8.1=py39haa95532_0 +jupyterlab_pygments=0.1.2=py_0 +jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 +kiwisolver=1.3.2=py39h2e07f2f_0 +krb5=1.19.2=hbae68bd_2 +lerc=3.0=h0e60522_0 +libclang=11.1.0=default_h5c34c98_1 +libcurl=7.79.1=h789b8ee_1 +libdeflate=1.8=h2bbff1b_5 +libiconv=1.16=he774522_0 +libpng=1.6.37=h1d00b33_2 +libssh2=1.10.0=h680486a_2 +libtiff=4.3.0=hd413186_2 +libwebp=1.2.0=h2bbff1b_0 +libxml2=2.9.12=h0ad7f3c_0 +libxslt=1.1.34=he774522_0 +libzlib=1.2.11=h8ffe710_1013 +lxml=4.6.3=py39h4fd7cdf_0 +lz4-c=1.9.3=h8ffe710_1 +mako=1.1.5=pyhd8ed1ab_0 +markupsafe=2.0.1=py39h2bbff1b_0 +matplotlib=3.4.3=py39hcbf5309_1 +matplotlib-base=3.4.3=py39h581301d_1 +matplotlib-inline=0.1.2=pyhd3eb1b0_2 +mistune=0.8.4=py39h2bbff1b_1000 +mkl=2021.3.0=haa95532_524 +mkl-service=2.4.0=py39h2bbff1b_0 +mkl_fft=1.3.1=py39h277e83a_0 +mkl_random=1.2.2=py39hf11a4ad_0 +nbclient=0.5.11=pyhd3eb1b0_0 +nbconvert=6.1.0=py39haa95532_0 +nbformat=5.1.3=pyhd3eb1b0_0 +nest-asyncio=1.5.1=pyhd3eb1b0_0 +notebook=6.4.8=py39haa95532_0 +numexpr=2.7.3=py39hb80d3ca_1 +numpy=1.21.2=py39hfca59bb_0 +numpy-base=1.21.2=py39h0829f74_0 +olefile=0.46=pyh9f0ad1d_1 +openjpeg=2.4.0=hb211442_1 +openssl=1.1.1m=h2bbff1b_0 +packaging=21.3=pyhd3eb1b0_0 +pandas=1.3.3=py39h6214cd6_0 +pandocfilters=1.5.0=pyhd3eb1b0_0 +parso=0.8.2=pyhd3eb1b0_0 +pickleshare=0.7.5=pyhd3eb1b0_1003 +pillow=8.4.0=py39hd45dc43_0 +pip=21.2.4=py39haa95532_0 +prometheus_client=0.13.1=pyhd3eb1b0_0 +prompt-toolkit=3.0.20=pyhd3eb1b0_0 +pycparser=2.21=pyhd3eb1b0_0 +pyfai=0.20.0=hd8ed1ab_0 +pyfai-base=0.20.0=py39h2e25243_0 +pygments=2.10.0=pyhd3eb1b0_0 +pyparsing=2.4.7=pyhd3eb1b0_0 +pyqt=5.12.3=py39hcbf5309_7 +pyqt-impl=5.12.3=py39h415ef7b_7 +pyqt5-sip=4.19.18=py39h415ef7b_7 +pyqtchart=5.12=py39h415ef7b_7 +pyqtwebengine=5.12.1=py39h415ef7b_7 +pyreadline=2.1=py39hcbf5309_1004 +pyrsistent=0.18.0=py39h196d8e1_0 +python=3.9.7=h6244533_1 +python-dateutil=2.8.2=pyhd3eb1b0_0 +python_abi=3.9=2_cp39 +pytz=2021.3=pyhd3eb1b0_0 +pywin32=228=py39hbaba5e8_1 +pywinpty=2.0.2=py39h5da7b33_0 +pyzmq=22.2.1=py39hd77b12b_1 +qt=5.12.9=h5909a2a_4 +qtconsole=5.1.1=pyhd3eb1b0_0 +qtpy=1.11.2=pyhd8ed1ab_0 +scipy=1.7.1=py39hbe87c03_2 +send2trash=1.8.0=pyhd3eb1b0_1 +setuptools=58.0.4=py39haa95532_0 +silx=0.15.2=hd8ed1ab_0 +silx-base=0.15.2=py39h2e25243_0 +six=1.16.0=pyhd3eb1b0_0 +sqlite=3.36.0=h2bbff1b_0 +terminado=0.13.1=py39haa95532_0 +testpath=0.5.0=pyhd3eb1b0_0 +tk=8.6.11=h8ffe710_1 +tornado=6.1=py39h2bbff1b_0 +traitlets=5.1.0=pyhd3eb1b0_0 +typing-extensions=3.10.0.2=hd3eb1b0_0 +typing_extensions=3.10.0.2=pyh06a4308_0 +tzdata=2021a=h5d7bf9c_0 +vc=14.2=h21ff451_1 +vs2015_runtime=14.27.29016=h5e58377_2 +wcwidth=0.2.5=pyhd3eb1b0_0 +webencodings=0.5.1=py39haa95532_1 +wheel=0.37.0=pyhd3eb1b0_1 +widgetsnbextension=3.5.2=py39haa95532_0 +wincertstore=0.2=py39haa95532_2 +winpty=0.4.3=4 +xz=5.2.5=h62dcd97_1 +zipp=3.7.0=pyhd3eb1b0_0 +zlib=1.2.11=h8ffe710_1013 +zstd=1.5.0=h6255e5f_0 From d993663c7c30c797d0c3cacbaad290f10cd3f456 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 11 Mar 2022 11:58:58 +0100 Subject: [PATCH 068/355] Make change to prettify_plot() --- beamtime/plotting.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index 354b220..ea6cc51 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -53,13 +53,18 @@ def prepare_plot(options={}): return fig, ax -def prettify__plot(fig, ax, plot_data, options): +def prettify_plot(fig, ax, plot_data, options): - required_options = ['plot_kind', 'hide_x_labels', 'hide_y_labels', 'rotation_x_ticks', 'rotation_y_ticks', 'xlabel', 'ylabel', 'yunit', 'xlim', 'ylim', 'x_tick_locators', 'y_tick_locators', 'xticks', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', + required_options = ['plot_kind', + 'hide_x_labels', 'hide_y_labels', + 'rotation_x_ticks', 'rotation_y_ticks', + 'xlim', 'ylim', + 'x_tick_locators', 'y_tick_locators', + 'xticks', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', 'colours', 'palettes', 'title', 'legend', 'legend_position', 'subplots_adjust', 'text', 'legend_ncol'] default_options = { - 'plot_kind': None, # defaults to None, but should be utilised when + 'plot_kind': None, # defaults to None, but should be utilised when requiring special formatting for a particular plot 'hide_x_labels': False, # Whether x labels should be hidden 'hide_x_ticklabels': False, 'hide_x_ticks': False, @@ -68,9 +73,6 @@ def prettify__plot(fig, ax, plot_data, options): 'hide_y_ticklabels': False, 'hide_y_ticks': False, 'rotation_y_ticks': 0, - 'xlabel': r'$x$ in Na$_{5-x}$FeO$_{4-\delta}$', - 'ylabel': 'Formation energy', - 'yunit': r'eV', # The unit of the y-values in the curve and bar plots 'xlim': None, 'ylim': None, 'x_tick_locators': [.5, .25], # Major and minor tick locators From 67ea048380745083d3fe4563480eb2c52bdbc199 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 11 Mar 2022 17:52:01 +0100 Subject: [PATCH 069/355] Generalise prepare_plot and adjust_plot --- beamtime/plotting.py | 141 +++++++++++++++++++++++++------------------ beamtime/xrd/plot.py | 40 +++++++----- 2 files changed, 107 insertions(+), 74 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index ea6cc51..da47dee 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -1,4 +1,5 @@ import beamtime.auxillary as aux + import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition, mark_inset) @@ -10,20 +11,23 @@ from cycler import cycler import itertools +import numpy as np + + def prepare_plot(options={}): - ''' Prepares plot based on contents of options['rc_params'] and options['format_params']. + ''' A general function to prepare a plot based on contents of options['rc_params'] and options['format_params']. - rc_params is a dictionary with keyval-pairs corresponding to rcParams in matplotlib + rc_params is a dictionary with keyval-pairs corresponding to rcParams in matplotlib, to give the user full control over this. Please consult the matplotlib-documentation - format_params will determine the size and aspect ratios of ''' + format_params will determine the size, aspect ratio, resolution etc. of the figure. Should be modified to conform with any requirements from a journal.''' rc_params = options['rc_params'] format_params = options['format_params'] - required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + required_format_params = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] - default_options = { + default_format_params = { 'single_column_width': 8.3, 'double_column_width': 17.1, 'column_type': 'single', @@ -35,7 +39,7 @@ def prepare_plot(options={}): 'dpi': 600, } - options = aux.update_options(format_params, required_options, default_options) + format_params = aux.update_options(format_params, required_format_params, default_format_params) # Reset run commands @@ -44,54 +48,50 @@ def prepare_plot(options={}): # Update run commands if any is passed (will pass an empty dictionary if not passed) update_rc_params(rc_params) - width = determine_width(options) - height = determine_height(options, width) - width, height = scale_figure(options=options, width=width, height=height) + width = determine_width(format_params=format_params) + height = determine_height(format_params=format_params, width=width) + width, height = scale_figure(format_params=format_params, width=width, height=height) - fig, ax = plt.subplots(figsize=(width, height), dpi=options['dpi']) + fig, ax = plt.subplots(figsize=(width, height), dpi=format_params['dpi']) return fig, ax -def prettify_plot(fig, ax, plot_data, options): +def adjust_plot(fig, ax, options): + ''' A general function to adjust plot according to contents of the options-dictionary ''' - required_options = ['plot_kind', - 'hide_x_labels', 'hide_y_labels', - 'rotation_x_ticks', 'rotation_y_ticks', + required_options = [ + 'plot_kind', + 'hide_x_labels', 'hide_y_labels', + 'hide_x_ticklabels', 'hide_y_ticklabels', + 'hide_x_ticks', 'hide_y_ticks', + 'x_tick_locators', 'y_tick_locators', + 'rotation_x_ticks', 'rotation_y_ticks', + 'xticks', 'yticks', 'xlim', 'ylim', - 'x_tick_locators', 'y_tick_locators', - 'xticks', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', - 'colours', 'palettes', 'title', 'legend', 'legend_position', 'subplots_adjust', 'text', 'legend_ncol'] + 'title', + 'legend', 'legend_position', 'legend_ncol', + 'subplots_adjust', + 'text'] default_options = { 'plot_kind': None, # defaults to None, but should be utilised when requiring special formatting for a particular plot - 'hide_x_labels': False, # Whether x labels should be hidden - 'hide_x_ticklabels': False, - 'hide_x_ticks': False, - 'rotation_x_ticks': 0, - 'hide_y_labels': False, # whether y labels should be hidden - 'hide_y_ticklabels': False, - 'hide_y_ticks': False, - 'rotation_y_ticks': 0, - 'xlim': None, - 'ylim': None, - 'x_tick_locators': [.5, .25], # Major and minor tick locators - 'y_tick_locators': [.5, .25], - 'xticks': None, - 'labels': None, - 'colours': None, - 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], - 'title': None, - 'legend': False, - 'legend_position': ['lower center', (0.5, -0.1)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively - 'legend_ncol': 1, - 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], - 'text': None + 'hide_x_labels': False, 'hide_y_labels': False, # Whether the main labels on the x- and/or y-axes should be hidden + 'hide_x_ticklabels': False, 'hide_y_ticklabels': False, # Whether ticklabels on the x- and/or y-axes should be hidden + 'hide_x_ticks': False, 'hide_y_ticks': False, # Whether the ticks on the x- and/or y-axes should be hidden + 'x_tick_locators': None, 'y_tick_locators': None, # The major and minor tick locators for the x- and y-axes + 'rotation_x_ticks': 0, 'rotation_y_ticks': 0, # Degrees the x- and/or y-ticklabels should be rotated + 'xticks': None, 'yticks': None, # Custom definition of the xticks and yticks. This is not properly implemented now. + 'xlim': None, 'ylim': None, # Limits to the x- and y-axes + 'title': None, # Title of the plot + 'legend': False, 'legend_position': ['lower center', (0.5, -0.1)], 'legend_ncol': 1, # Toggles on/off legend. Specifices legend position and the number of columns the legend should appear as. + 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], # Adjustment of the Axes-object within the Figure-object. Fraction of the Figure-object the left, bottom, right and top edges of the Axes-object will start. + 'text': None # Text to show in the plot. Should be a list where the first element is the string, and the second is a tuple with x- and y-coordinates. Could also be a list of lists to show more strings of text. } - options = update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) # Set labels on x- and y-axes if not options['hide_y_labels']: @@ -106,18 +106,22 @@ def prettify_plot(fig, ax, plot_data, options): # Set multiple locators - ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) - ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + if options['y_tick_locators']: + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) - ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) - ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + if options['x_tick_locators']: + ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) + ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + + # THIS NEEDS REWORK FOR IT TO FUNCTION PROPERLY! if options['xticks']: ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) ax.set_xticklabels(options['xticks']) - else: - ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) - ax.set_xticklabels([x/2 for x in np.arange(plot_data['start'], plot_data['end']+1)]) + # else: + # ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) + # ax.set_xticklabels([x/2 for x in np.arange(plot_data['start'], plot_data['end']+1)]) # Hide x- and y- ticklabels if options['hide_y_ticklabels']: @@ -202,7 +206,14 @@ def prettify_plot(fig, ax, plot_data, options): # Add custom text if options['text']: - plt.text(x=options['text'][1][0], y=options['text'][1][1], s=options['text'][0]) + + # If only a single element, put it into a list so the below for-loop works. + if isinstance(options['text'][0], str): + options['text'] = [options['text']] + + # Plot all passed texts + for text in options['text']: + plt.text(x=text[1][0], y=text[1][1], s=text[0]) return fig, ax @@ -210,28 +221,40 @@ def prettify_plot(fig, ax, plot_data, options): def ipywidgets_update(func, plot_data, options={}, **kwargs): + ''' A general ipywidgets update function that can be passed to ipywidgets.interactive. To use this, you can run: + import ipywidgets as widgets + import beamtime.plotting as btp + + w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(my_func), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), key1=widget1, key2=widget2, key3=widget3) + + where key1, key2, key3 etc. are the values in the options-dictionary you want widget control of, and widget1, widget2, widget3 etc. are widgets to control these values, e.g. widgets.IntSlider(value=1, min=0, max=10) + ''' + + # Update the options-dictionary with the values from the widgets for key in kwargs: options[key] = kwargs[key] + # Call the function with the plot_data and options-dictionaries func(plot_data=plot_data, options=options) -def determine_width(options): +def determine_width(format_params): + ''' ''' conversion_cm_inch = 0.3937008 # cm to inch - if options['column_type'] == 'single': - column_width = options['single_column_width'] - elif options['column_type'] == 'double': - column_width = options['double_column_width'] + if format_params['column_type'] == 'single': + column_width = format_params['single_column_width'] + elif format_params['column_type'] == 'double': + column_width = format_params['double_column_width'] column_width *= conversion_cm_inch - width_ratio = [float(num) for num in options['width_ratio'].split(':')] + width_ratio = [float(num) for num in format_params['width_ratio'].split(':')] width = column_width * width_ratio[0]/width_ratio[1] @@ -240,18 +263,18 @@ def determine_width(options): return width -def determine_height(options, width): +def determine_height(format_params, width): - aspect_ratio = [float(num) for num in options['aspect_ratio'].split(':')] + aspect_ratio = [float(num) for num in format_params['aspect_ratio'].split(':')] height = width/(aspect_ratio[0] / aspect_ratio[1]) return height -def scale_figure(options, width, height): - width = width * options['upscaling_factor'] * options['compress_width'] - height = height * options['upscaling_factor'] * options['compress_height'] +def scale_figure(format_params, width, height): + width = width * format_params['upscaling_factor'] * format_params['compress_width'] + height = height * format_params['upscaling_factor'] * format_params['compress_height'] return width, height diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 23b143b..9fb016c 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -7,38 +7,48 @@ import math import beamtime.xrd as xrd +import beamtime.auxillary as aux +import beamtime.plotting as btp - -def plot_diffractogram(path, kind, options=None): - - # Prepare plot, and read and process data - fig, ax = prepare_diffractogram_plot(options=options) - diffractogram = xrd.io.read_data(path=path, kind=kind, options=options) - +def plot_diffractogram(plot_data, options={}): + ''' Plots a diffractogram. + + Input: + plot_data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options - required_options = ['x_vals', 'y_vals', 'scatter'] - - + required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'scatter', 'plot_kind', 'rc_params', 'format_params'] default_options = { 'x_vals': '2th', - 'y_vals': 'I', - 'scatter': False + 'y_vals': 'I', + 'ylabel': 'Intensity', 'xlabel': '2theta', + 'xunit': 'deg', 'yunit': 'a.u.', + 'scatter': False, + 'plot_kind': None, + 'rc_params': {}, + 'format_params': {} } - options = update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + # Prepare plot, and read and process data + fig, ax = btp.prepare_plot(options=options) + diffractogram = xrd.io.read_data(path=plot_data['path'], kind=plot_data['plot_kind'], options=options) if options['scatter']: - diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, kind='scatter') + ax.scatter(x= diffractogram[options['x_vals']], y = diffractogram[options['y_vals']]) + + #diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, kind='scatter') else: diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax) - fig, ax = prettify_diffractogram_plot(fig=fig, ax=ax, options=options) + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) return diffractogram, fig, ax From a5c845fa541e56c18fcc475dec8c186ef05809e8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sat, 12 Mar 2022 22:26:06 +0100 Subject: [PATCH 070/355] Add reflection table and interactive mode to xrd --- beamtime/plotting.py | 79 +++++++++++++++++++++-- beamtime/xrd/io.py | 6 +- beamtime/xrd/plot.py | 147 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 213 insertions(+), 19 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index da47dee..f1c4bbe 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -25,7 +25,9 @@ def prepare_plot(options={}): rc_params = options['rc_params'] format_params = options['format_params'] - required_format_params = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + required_format_params = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', + 'width', 'height', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi', + 'nrows', 'ncols', 'grid_ratio_height', 'grid_ratio_width'] default_format_params = { 'single_column_width': 8.3, @@ -33,10 +35,16 @@ def prepare_plot(options={}): 'column_type': 'single', 'width_ratio': '1:1', 'aspect_ratio': '1:1', + 'width': None, + 'height': None, 'compress_width': 1, 'compress_height': 1, 'upscaling_factor': 1.0, - 'dpi': 600, + 'dpi': 600, + 'nrows': 1, + 'ncols': 1, + 'grid_ratio_height': None, + 'grid_ratio_width': None } format_params = aux.update_options(format_params, required_format_params, default_format_params) @@ -48,13 +56,72 @@ def prepare_plot(options={}): # Update run commands if any is passed (will pass an empty dictionary if not passed) update_rc_params(rc_params) - width = determine_width(format_params=format_params) - height = determine_height(format_params=format_params, width=width) + if not format_params['width']: + width = determine_width(format_params=format_params) + + if not format_params['height']: + height = determine_height(format_params=format_params, width=width) + width, height = scale_figure(format_params=format_params, width=width, height=height) - fig, ax = plt.subplots(figsize=(width, height), dpi=format_params['dpi']) + if format_params['nrows'] == 1 and format_params['ncols'] == 1: + fig, ax = plt.subplots(figsize=(width, height), dpi=format_params['dpi']) + + return fig, ax + + else: + if not format_params['grid_ratio_height']: + format_params['grid_ratio_height'] = [1 for i in range(format_params['nrows'])] + + if not format_params['grid_ratio_width']: + format_params['grid-ratio_width'] = [1 for i in range(format_params['ncols'])] + + fig, axes = plt.subplots(nrows=format_params['nrows'], ncols=format_params['ncols'], figsize=(width,height), + gridspec_kw={'height_ratios': format_params['grid_ratio_height'], 'width_ratios': format_params['grid_ratio_width']}, + facecolor='w', dpi=format_params['dpi']) + + return fig, axes + +def prepare_plots(options={}): + + rc_params = options['rc_params'] + format_params = options['format_params'] - return fig, ax + required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + + default_options = { + 'single_column_width': 8.3, + 'double_column_width': 17.1, + 'column_type': 'single', + 'width_ratio': '1:1', + 'aspect_ratio': '1:1', + 'compress_width': 1, + 'compress_height': 1, + 'upscaling_factor': 1.0, + 'dpi': 600, + } + + format_params = aux.update_options(format_params, required_options, default_options) + + + # Reset run commands + plt.rcdefaults() + + # Update run commands if any is passed (will pass an empty dictionary if not passed) + update_rc_params(rc_params) + + width = determine_width(format_params) + height = determine_height(format_params, width) + width, height = scale_figure(options=format_params, width=width, height=height) + + + if options['plot_kind'] == 'relative': + fig, axes = plt.subplots(nrows=1, ncols=options['number_of_frames'], figsize=(width,height), facecolor='w', dpi=format_params['dpi']) + + elif options['plot_kind'] == 'absolute': + fig, axes = plt.subplots(nrows=2, ncols=options['number_of_frames'], figsize=(width,height), gridspec_kw={'height_ratios': [1,5]}, facecolor='w', dpi=format_params['dpi']) + + return fig, axes def adjust_plot(fig, ax, options): diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 2afe41c..a0b4f9f 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -87,7 +87,7 @@ def integrate_1d(calibrant, bins, path=None, image=None, options=None): res = ai.integrate1d(image, bins, unit=options['unit'], filename=filename) if options['return']: - return open_1d_data(filename) + return read_diffractogram(filename) @@ -222,7 +222,7 @@ def read_brml(path, options=None): return diffractogram -def read_diffractogram(path, options=None): +def read_diffractogram(path): @@ -248,7 +248,7 @@ def read_diffractogram(path, options=None): def read_data(path, kind, options=None): if kind == 'beamline': - diffractogram = read_diffractogram(path, options=options) + diffractogram = read_diffractogram(path) elif kind == 'recx': diffractogram = read_brml(path, options=options) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 9fb016c..f8dd518 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -5,8 +5,9 @@ import pandas as pd import numpy as np import math -import beamtime.xrd as xrd +import ipywidgets as widgets +import beamtime.xrd as xrd import beamtime.auxillary as aux import beamtime.plotting as btp @@ -18,42 +19,168 @@ def plot_diffractogram(plot_data, options={}): plot_data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options - required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'scatter', 'plot_kind', 'rc_params', 'format_params'] + required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', + 'reflections', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] default_options = { 'x_vals': '2th', 'y_vals': 'I', 'ylabel': 'Intensity', 'xlabel': '2theta', 'xunit': 'deg', 'yunit': 'a.u.', + 'line': True, 'scatter': False, + 'reflections': False, + 'reflection_data': None, # Should be passed as a dictionary on the form {path: rel_path, colour: [r,g,b], min_alpha: 0-1] 'plot_kind': None, + 'palettes': [('qualitative', 'Dark2_8')], + 'interactive': False, 'rc_params': {}, - 'format_params': {} + 'format_params': {}, } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + if options['interactive']: + options['interactive'] = False + plot_diffractogram_interactive(plot_data=plot_data, options=options) + return + + # Make adjustments to parameters if reflections data is passed + if options['reflections']: + options['format_params']['nrows'] = 2 + + if not 'grid_ratio_height' in options['format_params'].keys(): + options['format_params']['grid_ratio_height'] = [1,10] + + else: + options['format_params']['nrows'] = 1 + + # Prepare plot, and read and process data + fig, ax = btp.prepare_plot(options=options) + + if options['reflections']: + reflection_ax = ax[0] + ax = ax[1] + + colours = btp.generate_colours(options['palettes']) + + diffractogram = xrd.io.read_data(path=plot_data['path'], kind=plot_data['plot_kind'], options=options) - if options['scatter']: - ax.scatter(x= diffractogram[options['x_vals']], y = diffractogram[options['y_vals']]) - - #diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, kind='scatter') - - else: - diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax) + if options['line']: + diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) + + if options['scatter']: + ax.scatter(x= diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + if options['reflections'] and options['reflection_data']: + plot_reflection_table(plot_data=options['reflection_data'], ax=reflection_ax, options=options) + return diffractogram, fig, ax +def plot_diffractogram_interactive(plot_data, options): + + if options['reflections']: + w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), + scatter=widgets.ToggleButton(value=False), + line=widgets.ToggleButton(value=True), + reflections=widgets.ToggleButton(value=True)) + + else: + w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), + scatter=widgets.ToggleButton(value=False), + line=widgets.ToggleButton(value=True)) + + + display(w) + + + +def plot_reflection_table(plot_data, ax=None, options={}): + ''' Plots a reflection table from output generated by VESTA. + + Required contents of plot_data: + path (str): relative path to reflection table file''' + + required_options = ['reflections_colour', 'min_alpha', 'format_params', 'rc_params'] + + default_options = { + 'reflections_colour': [0,0,0], + 'min_alpha': 0, + 'format_params': {}, + 'rc_params': {} + } + + if 'colour' in plot_data.keys(): + options['reflections_colour'] = plot_data['colour'] + if 'min_alpha' in plot_data.keys(): + options['min_alpha'] = plot_data['min_alpha'] + + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + + if not ax: + _, ax = btp.prepare_plot(options) + + reflection_table = load_reflection_table(plot_data['path']) + + reflections, intensities = reflection_table['2th'], reflection_table['I'] + + for ref, intensity in zip(reflections, intensities): + + rel_intensity = (intensity / intensities.max())*(1-options['min_alpha']) + options['min_alpha'] + + + ax.axvline(x=ref, c=options['reflections_colour'], alpha=rel_intensity) + + + + ax.tick_params(which='both', bottom=False, labelbottom=False, right=False, labelright=False, left=False, labelleft=False, top=False, labeltop=False) + + if options['xlim']: + ax.set_xlim(options['xlim']) + + + + +def load_reflection_table(path): + + # VESTA outputs the file with a header that has a space between the parameter and units - so there is some extra code to rectify the issue + # that ensues from this formatting + reflections = pd.read_csv(path, delim_whitespace=True) + + # Remove the extra column that appears from the headers issue + reflections.drop(reflections.columns[-1], axis=1, inplace=True) + + with open(path, 'r') as f: + line = f.readline() + + headers = line.split() + + # Delete the fourth element which is '(Å)' + del headers[4] + + # Change name of column to avoid using greek letters + headers[7] = '2th' + + # Set the new modified headers as the headers of + reflections.columns = headers + + return reflections + + + def prepare_diffractogram_plot(options=None): # First take care of the options for plotting - set any values not specified to the default values From e6c48c1e5452c8d4ccc3ca6f064676f1399d25ea Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sat, 12 Mar 2022 22:50:09 +0100 Subject: [PATCH 071/355] Speed up reflection table plotting --- beamtime/xrd/io.py | 24 ++++++++++++++++++++++++ beamtime/xrd/plot.py | 35 ++++++++--------------------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index a0b4f9f..347eed1 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -262,3 +262,27 @@ def read_data(path, kind, options=None): +def load_reflection_table(path): + + # VESTA outputs the file with a header that has a space between the parameter and units - so there is some extra code to rectify the issue + # that ensues from this formatting + reflections = pd.read_csv(path, delim_whitespace=True) + + # Remove the extra column that appears from the headers issue + reflections.drop(reflections.columns[-1], axis=1, inplace=True) + + with open(path, 'r') as f: + line = f.readline() + + headers = line.split() + + # Delete the fourth element which is '(Å)' + del headers[4] + + # Change name of column to avoid using greek letters + headers[7] = '2th' + + # Set the new modified headers as the headers of + reflections.columns = headers + + return reflections \ No newline at end of file diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index f8dd518..a6697c9 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -133,16 +133,21 @@ def plot_reflection_table(plot_data, ax=None, options={}): if not ax: _, ax = btp.prepare_plot(options) - reflection_table = load_reflection_table(plot_data['path']) + reflection_table = xrd.io.load_reflection_table(plot_data['path']) reflections, intensities = reflection_table['2th'], reflection_table['I'] + colours = [] for ref, intensity in zip(reflections, intensities): + colour = list(options['reflections_colour']) rel_intensity = (intensity / intensities.max())*(1-options['min_alpha']) + options['min_alpha'] + colour.append(rel_intensity) + colours.append(colour) - - ax.axvline(x=ref, c=options['reflections_colour'], alpha=rel_intensity) + + ax.vlines(x=reflections, ymin=-1, ymax=1, colors=colours) + ax.set_ylim([-0.5,0.5]) @@ -154,30 +159,6 @@ def plot_reflection_table(plot_data, ax=None, options={}): -def load_reflection_table(path): - - # VESTA outputs the file with a header that has a space between the parameter and units - so there is some extra code to rectify the issue - # that ensues from this formatting - reflections = pd.read_csv(path, delim_whitespace=True) - - # Remove the extra column that appears from the headers issue - reflections.drop(reflections.columns[-1], axis=1, inplace=True) - - with open(path, 'r') as f: - line = f.readline() - - headers = line.split() - - # Delete the fourth element which is '(Å)' - del headers[4] - - # Change name of column to avoid using greek letters - headers[7] = '2th' - - # Set the new modified headers as the headers of - reflections.columns = headers - - return reflections From 545c8212c5251ee70107b8f8f33728411d16cdfa Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 13 Mar 2022 13:58:28 +0100 Subject: [PATCH 072/355] Add indices function to xrd plot --- beamtime/xrd/plot.py | 109 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 19 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index a6697c9..bd6062b 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -20,7 +20,7 @@ def plot_diffractogram(plot_data, options={}): # Update options required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', - 'reflections', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] + 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] default_options = { 'x_vals': '2th', @@ -29,8 +29,9 @@ def plot_diffractogram(plot_data, options={}): 'xunit': 'deg', 'yunit': 'a.u.', 'line': True, 'scatter': False, - 'reflections': False, - 'reflection_data': None, # Should be passed as a dictionary on the form {path: rel_path, colour: [r,g,b], min_alpha: 0-1] + 'reflections_plot': False, + 'reflections_indices': False, + 'reflections_data': None, # Should be passed as a dictionary on the form {path: rel_path, reflection_indices: number of indices, plot: boolean, colour: [r,g,b], min_alpha: 0-1] 'plot_kind': None, 'palettes': [('qualitative', 'Dark2_8')], 'interactive': False, @@ -46,24 +47,46 @@ def plot_diffractogram(plot_data, options={}): plot_diffractogram_interactive(plot_data=plot_data, options=options) return - # Make adjustments to parameters if reflections data is passed - if options['reflections']: - options['format_params']['nrows'] = 2 - - if not 'grid_ratio_height' in options['format_params'].keys(): + + if options['reflections_plot'] or options['reflections_indices']: + if options['reflections_plot'] and options['reflections_indices']: + options['format_params']['nrows'] = 3 + options['format_params']['grid_ratio_height'] = [2,1,10] + + elif options['reflections_plot']: + options['format_params']['nrows'] = 2 options['format_params']['grid_ratio_height'] = [1,10] + elif options['reflections_indices']: + options['format_params']['nrows'] = 2 + options['format_params']['grid_ratio_height'] = [2,10] + else: options['format_params']['nrows'] = 1 + + # Prepare plot, and read and process data fig, ax = btp.prepare_plot(options=options) - if options['reflections']: - reflection_ax = ax[0] - ax = ax[1] + + # Assign the correct axes + if options['reflections_plot'] or options['reflections_indices']: + if options['reflections_plot'] and options['reflections_indices']: + indices_ax = ax[0] + reflection_ax = ax[1] + ax = ax[2] + + elif options['reflections_plot']: + reflection_ax = ax[0] + ax = ax[1] + + elif options['reflections_indices']: + indices_ax = ax[0] + ax = ax[1] + colours = btp.generate_colours(options['palettes']) @@ -80,8 +103,13 @@ def plot_diffractogram(plot_data, options={}): fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) - if options['reflections'] and options['reflection_data']: - plot_reflection_table(plot_data=options['reflection_data'], ax=reflection_ax, options=options) + if options['reflections_plot'] and options['reflections_data']: + options['xlim'] = ax.get_xlim() + plot_reflection_table(plot_data=options['reflections_data'], ax=reflection_ax, options=options) + + if options['reflections_indices'] and options['reflections_data']: + options['xlim'] = ax.get_xlim() + plot_reflection_indices(plot_data=options['reflections_data'], ax=indices_ax, options=options) return diffractogram, fig, ax @@ -89,11 +117,12 @@ def plot_diffractogram(plot_data, options={}): def plot_diffractogram_interactive(plot_data, options): - if options['reflections']: + if options['reflections_data']: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), - reflections=widgets.ToggleButton(value=True)) + reflections_plot=widgets.ToggleButton(value=True), + reflections_indices=widgets.ToggleButton(value=False)) else: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), @@ -103,7 +132,34 @@ def plot_diffractogram_interactive(plot_data, options): display(w) +def plot_reflection_indices(plot_data, ax, options={}): + required_options = ['reflection_indices'] + + default_options = { + 'reflection_indices': 3 + } + + plot_data = update_options(options=plot_data, required_options=required_options, default_options=default_options) + + + reflection_table = xrd.io.load_reflection_table(plot_data['path']) + + if plot_data['reflection_indices'] > 0: + reflection_indices = reflection_table.nlargest(options['reflection_indices'], 'I') + + + for i in range(plot_data['reflection_indices']): + ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices['2th'].iloc[i], y=0, fontsize=5, rotation=90, va='bottom', ha='center') + + + if options['xlim']: + ax.set_xlim(options['xlim']) + + ax.axis('off') + + + return def plot_reflection_table(plot_data, ax=None, options={}): ''' Plots a reflection table from output generated by VESTA. @@ -111,31 +167,39 @@ def plot_reflection_table(plot_data, ax=None, options={}): Required contents of plot_data: path (str): relative path to reflection table file''' - required_options = ['reflections_colour', 'min_alpha', 'format_params', 'rc_params'] + required_options = ['reflection_indices', 'reflections_colour', 'min_alpha', 'format_params', 'rc_params', 'label'] default_options = { + 'reflection_indices': 0, # Number of indices to print 'reflections_colour': [0,0,0], 'min_alpha': 0, 'format_params': {}, - 'rc_params': {} + 'rc_params': {}, + 'label': None } if 'colour' in plot_data.keys(): options['reflections_colour'] = plot_data['colour'] if 'min_alpha' in plot_data.keys(): options['min_alpha'] = plot_data['min_alpha'] + if 'reflection_indices' in plot_data.keys(): + options['reflection_indices'] = plot_data['reflection_indices'] + if 'label' in plot_data.keys(): + options['label'] = plot_data['label'] options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - if not ax: _, ax = btp.prepare_plot(options) reflection_table = xrd.io.load_reflection_table(plot_data['path']) reflections, intensities = reflection_table['2th'], reflection_table['I'] + + + colours = [] for ref, intensity in zip(reflections, intensities): @@ -146,9 +210,10 @@ def plot_reflection_table(plot_data, ax=None, options={}): colours.append(colour) + + ax.vlines(x=reflections, ymin=-1, ymax=1, colors=colours) ax.set_ylim([-0.5,0.5]) - ax.tick_params(which='both', bottom=False, labelbottom=False, right=False, labelright=False, left=False, labelleft=False, top=False, labeltop=False) @@ -157,6 +222,12 @@ def plot_reflection_table(plot_data, ax=None, options={}): ax.set_xlim(options['xlim']) + if options['label']: + xlim_range = ax.get_xlim()[1] - ax.get_xlim()[0] + ylim_avg = (ax.get_ylim()[0]+ax.get_ylim()[1])/2 + + ax.text(s=plot_data['label'], x=(ax.get_xlim()[0]-0.01*xlim_range), y=ylim_avg, ha = 'right', va = 'center') + From 1126809c5acbbfcf0b85ad436458a7d1ea9709e2 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 15 Mar 2022 15:51:39 +0100 Subject: [PATCH 073/355] Plot multiple simulated diffractograms --- beamtime/xrd/plot.py | 129 ++++++++++++++++++++++++++----------------- 1 file changed, 78 insertions(+), 51 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index bd6062b..c9ecc77 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -27,14 +27,16 @@ def plot_diffractogram(plot_data, options={}): 'y_vals': 'I', 'ylabel': 'Intensity', 'xlabel': '2theta', 'xunit': 'deg', 'yunit': 'a.u.', - 'line': True, - 'scatter': False, - 'reflections_plot': False, - 'reflections_indices': False, - 'reflections_data': None, # Should be passed as a dictionary on the form {path: rel_path, reflection_indices: number of indices, plot: boolean, colour: [r,g,b], min_alpha: 0-1] + 'xlim': None, 'ylim': None, + 'line': True, # whether or not to plot diffractogram as a line plot + 'scatter': False, # whether or not to plot individual data points + 'reflections_plot': False, # whether to plot reflections as a plot + 'reflections_indices': False, # whether to plot the reflection indices + 'reflections_data': None, # Should be passed as a list of dictionaries on the form {path: rel_path, reflection_indices: number of indices, colour: [r,g,b], min_alpha: 0-1] 'plot_kind': None, 'palettes': [('qualitative', 'Dark2_8')], 'interactive': False, + 'interactive_session_active': False, 'rc_params': {}, 'format_params': {}, } @@ -44,27 +46,20 @@ def plot_diffractogram(plot_data, options={}): if options['interactive']: options['interactive'] = False + options['interactive_session_active'] = True plot_diffractogram_interactive(plot_data=plot_data, options=options) return - if options['reflections_plot'] or options['reflections_indices']: - if options['reflections_plot'] and options['reflections_indices']: - options['format_params']['nrows'] = 3 - options['format_params']['grid_ratio_height'] = [2,1,10] - - elif options['reflections_plot']: - options['format_params']['nrows'] = 2 - options['format_params']['grid_ratio_height'] = [1,10] - - elif options['reflections_indices']: - options['format_params']['nrows'] = 2 - options['format_params']['grid_ratio_height'] = [2,10] - - else: - options['format_params']['nrows'] = 1 + if options['reflections_data']: + if not isinstance(options['reflections_data'], list): + options['reflections_data'] = [options['reflections_data']] + # Determine number of subplots and height ratios between them + if len(options['reflections_data']) >= 1: + options = determine_grid_layout(options=options) + print(options['format_params']['nrows']) # Prepare plot, and read and process data @@ -74,19 +69,17 @@ def plot_diffractogram(plot_data, options={}): # Assign the correct axes if options['reflections_plot'] or options['reflections_indices']: - if options['reflections_plot'] and options['reflections_indices']: + + if options['reflections_indices']: indices_ax = ax[0] - reflection_ax = ax[1] - ax = ax[2] - elif options['reflections_plot']: - reflection_ax = ax[0] - ax = ax[1] - - elif options['reflections_indices']: - indices_ax = ax[0] - ax = ax[1] - + if options['reflections_plot']: + ref_axes = [axx for axx in ax[range(1,len(options['reflections_data'])+1)]] + + else: + ref_axes = [axx for axx in ax[range(0,len(options['reflections_data']))]] + + ax = ax[-1] colours = btp.generate_colours(options['palettes']) @@ -98,23 +91,50 @@ def plot_diffractogram(plot_data, options={}): diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) if options['scatter']: - ax.scatter(x= diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) + ax.scatter(x=diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) + + if not options['xlim']: + options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + + # Make the reflection plots if options['reflections_plot'] and options['reflections_data']: options['xlim'] = ax.get_xlim() - plot_reflection_table(plot_data=options['reflections_data'], ax=reflection_ax, options=options) + + for reference, axis in zip(options['reflections_data'], ref_axes): + plot_reflection_table(plot_data=reference, ax=axis, options=options) + # Print the reflection indices if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() - plot_reflection_indices(plot_data=options['reflections_data'], ax=indices_ax, options=options) + for reference in options['reflections_data']: + plot_reflection_indices(plot_data=reference, ax=indices_ax, options=options) return diffractogram, fig, ax +def determine_grid_layout(options): + + + nrows = 1 if not options['reflections_indices'] else 2 + + if options['reflections_plot']: + for reference in options['reflections_data']: + nrows += 1 + + options['format_params']['nrows'] = nrows + options['format_params']['grid_ratio_height'] = [1 for i in range(nrows-1)]+[10] + + return options + + + def plot_diffractogram_interactive(plot_data, options): if options['reflections_data']: @@ -132,31 +152,38 @@ def plot_diffractogram_interactive(plot_data, options): display(w) -def plot_reflection_indices(plot_data, ax, options={}): - required_options = ['reflection_indices'] +def plot_reflection_indices(plot_data, ax, options={}): + ''' Print reflection indices from output generated by VESTA. + + Required contents of plot_data: + path (str): relative path to reflection table file''' + + required_options = ['reflection_indices', 'text_colour', 'hide_indices'] default_options = { - 'reflection_indices': 3 + 'reflection_indices': 3, # Number of reflection indices to plot, from highest intensity and working its way down + 'text_colour': 'black', + 'hide_indices': False } plot_data = update_options(options=plot_data, required_options=required_options, default_options=default_options) + if not plot_data['hide_indices']: + reflection_table = xrd.io.load_reflection_table(plot_data['path']) + + if plot_data['reflection_indices'] > 0: + reflection_indices = reflection_table.nlargest(options['reflection_indices'], 'I') - reflection_table = xrd.io.load_reflection_table(plot_data['path']) - - if plot_data['reflection_indices'] > 0: - reflection_indices = reflection_table.nlargest(options['reflection_indices'], 'I') + + for i in range(plot_data['reflection_indices']): + ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices['2th'].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=plot_data['text_colour']) - - for i in range(plot_data['reflection_indices']): - ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices['2th'].iloc[i], y=0, fontsize=5, rotation=90, va='bottom', ha='center') - - - if options['xlim']: - ax.set_xlim(options['xlim']) - - ax.axis('off') + + if options['xlim']: + ax.set_xlim(options['xlim']) + + ax.axis('off') return @@ -212,7 +239,7 @@ def plot_reflection_table(plot_data, ax=None, options={}): - ax.vlines(x=reflections, ymin=-1, ymax=1, colors=colours) + ax.vlines(x=reflections, ymin=-1, ymax=1, colors=colours, lw=0.5) ax.set_ylim([-0.5,0.5]) From ebb98debffb2a4166e3aed5f3bbecbd126ccfe98 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 15 Mar 2022 17:13:17 +0100 Subject: [PATCH 074/355] Open beamline data straight from raw file --- beamtime/xrd/io.py | 49 ++++++++++++++++++++++++-------------------- beamtime/xrd/plot.py | 6 ++---- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 347eed1..43ae2f7 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -23,15 +23,14 @@ def get_image_headers(path): return image.header -def integrate_1d(calibrant, bins, path=None, image=None, options=None): +def integrate_1d(data, options={}): ''' Integrates an image file to a 1D diffractogram. - Input: - calibrant: path to .poni-file - bins: Number of bins to divide image into - path (optional): path to image file - either this or image must be specified. If both is passed, image is prioritsed - image (optional): image array (Numpy) as extracted from get_image_array - options (optional): dictionary of options + Required content of data: + calibrant (str): path to .poni-file + nbins (int): Number of bins to divide image into + path (str) (optional): path to image file - either this or image must be specified. If both is passed, image is prioritsed + image (str) (optional): image array (Numpy) as extracted from get_image_array Output: df: DataFrame contianing 1D diffractogram if option 'return' is True @@ -56,15 +55,15 @@ def integrate_1d(calibrant, bins, path=None, image=None, options=None): options[option] = default_options[option] - if not image: - image = get_image_array(path) + if 'image' not in data.keys(): + data['image'] = get_image_array(data['path']) - ai = pyFAI.load(calibrant) + ai = pyFAI.load(data['calibrant']) if not options['filename']: - if path: - filename = os.path.join(options['save_folder'], os.path.split(path)[-1].split('.')[0] + options['extension']) + if data['path']: + filename = os.path.join(options['save_folder'], os.path.split(data['path'])[-1].split('.')[0] + options['extension']) else: filename = os.path.join(options['save_folder'], 'integrated.dat') @@ -84,10 +83,9 @@ def integrate_1d(calibrant, bins, path=None, image=None, options=None): os.makedirs(options['save_folder']) - res = ai.integrate1d(image, bins, unit=options['unit'], filename=filename) + res = ai.integrate1d(data['image'], data['nbins'], unit=options['unit'], filename=filename) - if options['return']: - return read_diffractogram(filename) + return read_diffractogram(filename) @@ -245,16 +243,23 @@ def read_diffractogram(path): return diffractogram -def read_data(path, kind, options=None): +def read_data(data, options={}): - if kind == 'beamline': - diffractogram = read_diffractogram(path) + if data['plot_kind'] == 'beamline': - elif kind == 'recx': - diffractogram = read_brml(path, options=options) + beamline = ['mar3450', 'edf', 'cbf'] - elif kind == 'image': - diffractogram = get_image_array(path) + if data['path'].split('.')[-1] in beamline: + diffractogram = integrate_1d(data=data, options=options) + + else: + diffractogram = read_diffractogram(data['path']) + + elif data['plot_kind'] == 'recx': + diffractogram = read_brml(data['path'], options=options) + + elif data['plot_kind'] == 'image': + diffractogram = get_image_array(data['path']) return diffractogram diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index c9ecc77..25a91cf 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -19,7 +19,7 @@ def plot_diffractogram(plot_data, options={}): plot_data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options - required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', + required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] default_options = { @@ -59,8 +59,6 @@ def plot_diffractogram(plot_data, options={}): if len(options['reflections_data']) >= 1: options = determine_grid_layout(options=options) - print(options['format_params']['nrows']) - # Prepare plot, and read and process data @@ -84,7 +82,7 @@ def plot_diffractogram(plot_data, options={}): colours = btp.generate_colours(options['palettes']) - diffractogram = xrd.io.read_data(path=plot_data['path'], kind=plot_data['plot_kind'], options=options) + diffractogram = xrd.io.read_data(data=plot_data) if options['line']: From f2902aed3a97261eaa5e4161c85549ddb8ce3a09 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 15 Mar 2022 21:17:15 +0100 Subject: [PATCH 075/355] Speed up replotting and add xlim slider --- beamtime/plotting.py | 14 ++++---- beamtime/xrd/plot.py | 80 +++++++++++++++++++++++++------------------- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index f1c4bbe..c78a989 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -57,15 +57,15 @@ def prepare_plot(options={}): update_rc_params(rc_params) if not format_params['width']: - width = determine_width(format_params=format_params) + format_params['width'] = determine_width(format_params=format_params) if not format_params['height']: - height = determine_height(format_params=format_params, width=width) + format_params['height'] = determine_height(format_params=format_params, width=format_params['width']) - width, height = scale_figure(format_params=format_params, width=width, height=height) + format_params['width'], format_params['height'] = scale_figure(format_params=format_params, width=format_params['width'], height=format_params['height']) if format_params['nrows'] == 1 and format_params['ncols'] == 1: - fig, ax = plt.subplots(figsize=(width, height), dpi=format_params['dpi']) + fig, ax = plt.subplots(figsize=(format_params['width'], format_params['height']), dpi=format_params['dpi']) return fig, ax @@ -76,7 +76,7 @@ def prepare_plot(options={}): if not format_params['grid_ratio_width']: format_params['grid-ratio_width'] = [1 for i in range(format_params['ncols'])] - fig, axes = plt.subplots(nrows=format_params['nrows'], ncols=format_params['ncols'], figsize=(width,height), + fig, axes = plt.subplots(nrows=format_params['nrows'], ncols=format_params['ncols'], figsize=(format_params['width'],format_params['height']), gridspec_kw={'height_ratios': format_params['grid_ratio_height'], 'width_ratios': format_params['grid_ratio_width']}, facecolor='w', dpi=format_params['dpi']) @@ -287,7 +287,7 @@ def adjust_plot(fig, ax, options): -def ipywidgets_update(func, plot_data, options={}, **kwargs): +def ipywidgets_update(func, data, options={}, **kwargs): ''' A general ipywidgets update function that can be passed to ipywidgets.interactive. To use this, you can run: import ipywidgets as widgets @@ -303,7 +303,7 @@ def ipywidgets_update(func, plot_data, options={}, **kwargs): options[key] = kwargs[key] # Call the function with the plot_data and options-dictionaries - func(plot_data=plot_data, options=options) + func(data=data, options=options) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 25a91cf..7c89c43 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -12,11 +12,11 @@ import beamtime.auxillary as aux import beamtime.plotting as btp -def plot_diffractogram(plot_data, options={}): +def plot_diffractogram(data, options={}): ''' Plots a diffractogram. Input: - plot_data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' + data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', @@ -43,11 +43,22 @@ def plot_diffractogram(plot_data, options={}): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + if not 'diffractogram' in data.keys(): + diffractogram = xrd.io.read_data(data=data) + data['diffractogram'] = diffractogram + else: + diffractogram = data['diffractogram'] + + if not options['xlim']: + options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] + + + # Start inteactive session with ipywidgets if options['interactive']: options['interactive'] = False options['interactive_session_active'] = True - plot_diffractogram_interactive(plot_data=plot_data, options=options) + plot_diffractogram_interactive(data=data, options=options) return @@ -63,7 +74,7 @@ def plot_diffractogram(plot_data, options={}): # Prepare plot, and read and process data fig, ax = btp.prepare_plot(options=options) - + # Assign the correct axes if options['reflections_plot'] or options['reflections_indices']: @@ -82,7 +93,7 @@ def plot_diffractogram(plot_data, options={}): colours = btp.generate_colours(options['palettes']) - diffractogram = xrd.io.read_data(data=plot_data) + if options['line']: @@ -93,8 +104,7 @@ def plot_diffractogram(plot_data, options={}): - if not options['xlim']: - options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) @@ -105,13 +115,13 @@ def plot_diffractogram(plot_data, options={}): options['xlim'] = ax.get_xlim() for reference, axis in zip(options['reflections_data'], ref_axes): - plot_reflection_table(plot_data=reference, ax=axis, options=options) + plot_reflection_table(data=reference, ax=axis, options=options) # Print the reflection indices if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() for reference in options['reflections_data']: - plot_reflection_indices(plot_data=reference, ax=indices_ax, options=options) + plot_reflection_indices(data=reference, ax=indices_ax, options=options) return diffractogram, fig, ax @@ -133,28 +143,30 @@ def determine_grid_layout(options): -def plot_diffractogram_interactive(plot_data, options): +def plot_diffractogram_interactive(data, options): if options['reflections_data']: - w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), + w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), reflections_plot=widgets.ToggleButton(value=True), - reflections_indices=widgets.ToggleButton(value=False)) + reflections_indices=widgets.ToggleButton(value=False), + xlim=widgets.IntRangeSlider(value=options['xlim'], min=options['xlim'][0], max=options['xlim'][1], step=1, description='xlim', layout=widgets.Layout(width='1000px'))) else: - w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), plot_data=widgets.fixed(plot_data), options=widgets.fixed(options), + w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), - line=widgets.ToggleButton(value=True)) + line=widgets.ToggleButton(value=True), + xlim=widgets.IntRangeSlider(value=options['xlim'], min=options['xlim'][0], max=options['xlim'][1], step=1, description='xlim')) display(w) -def plot_reflection_indices(plot_data, ax, options={}): +def plot_reflection_indices(data, ax, options={}): ''' Print reflection indices from output generated by VESTA. - Required contents of plot_data: + Required contents of data: path (str): relative path to reflection table file''' required_options = ['reflection_indices', 'text_colour', 'hide_indices'] @@ -165,17 +177,17 @@ def plot_reflection_indices(plot_data, ax, options={}): 'hide_indices': False } - plot_data = update_options(options=plot_data, required_options=required_options, default_options=default_options) + data = update_options(options=data, required_options=required_options, default_options=default_options) - if not plot_data['hide_indices']: - reflection_table = xrd.io.load_reflection_table(plot_data['path']) + if not data['hide_indices']: + reflection_table = xrd.io.load_reflection_table(data['path']) - if plot_data['reflection_indices'] > 0: + if data['reflection_indices'] > 0: reflection_indices = reflection_table.nlargest(options['reflection_indices'], 'I') - for i in range(plot_data['reflection_indices']): - ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices['2th'].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=plot_data['text_colour']) + for i in range(data['reflection_indices']): + ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices['2th'].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=data['text_colour']) if options['xlim']: @@ -186,10 +198,10 @@ def plot_reflection_indices(plot_data, ax, options={}): return -def plot_reflection_table(plot_data, ax=None, options={}): +def plot_reflection_table(data, ax=None, options={}): ''' Plots a reflection table from output generated by VESTA. - Required contents of plot_data: + Required contents of data: path (str): relative path to reflection table file''' required_options = ['reflection_indices', 'reflections_colour', 'min_alpha', 'format_params', 'rc_params', 'label'] @@ -203,14 +215,14 @@ def plot_reflection_table(plot_data, ax=None, options={}): 'label': None } - if 'colour' in plot_data.keys(): - options['reflections_colour'] = plot_data['colour'] - if 'min_alpha' in plot_data.keys(): - options['min_alpha'] = plot_data['min_alpha'] - if 'reflection_indices' in plot_data.keys(): - options['reflection_indices'] = plot_data['reflection_indices'] - if 'label' in plot_data.keys(): - options['label'] = plot_data['label'] + if 'colour' in data.keys(): + options['reflections_colour'] = data['colour'] + if 'min_alpha' in data.keys(): + options['min_alpha'] = data['min_alpha'] + if 'reflection_indices' in data.keys(): + options['reflection_indices'] = data['reflection_indices'] + if 'label' in data.keys(): + options['label'] = data['label'] options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -219,7 +231,7 @@ def plot_reflection_table(plot_data, ax=None, options={}): if not ax: _, ax = btp.prepare_plot(options) - reflection_table = xrd.io.load_reflection_table(plot_data['path']) + reflection_table = xrd.io.load_reflection_table(data['path']) reflections, intensities = reflection_table['2th'], reflection_table['I'] @@ -251,7 +263,7 @@ def plot_reflection_table(plot_data, ax=None, options={}): xlim_range = ax.get_xlim()[1] - ax.get_xlim()[0] ylim_avg = (ax.get_ylim()[0]+ax.get_ylim()[1])/2 - ax.text(s=plot_data['label'], x=(ax.get_xlim()[0]-0.01*xlim_range), y=ylim_avg, ha = 'right', va = 'center') + ax.text(s=data['label'], x=(ax.get_xlim()[0]-0.01*xlim_range), y=ylim_avg, ha = 'right', va = 'center') From 0e4f14a6e3f35ba046b9fe1417f1c22d7b4c83f6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 16 Mar 2022 14:16:41 +0100 Subject: [PATCH 076/355] Add translation functions --- beamtime/xrd/io.py | 185 ++++++++++++++++++++++++++++++------------- beamtime/xrd/plot.py | 11 ++- 2 files changed, 137 insertions(+), 59 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 43ae2f7..b5a479c 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -8,6 +8,9 @@ import zipfile import xml.etree.ElementTree as ET +import beamtime.auxillary as aux + + def get_image_array(path): image = fabio.open(path) @@ -29,66 +32,95 @@ def integrate_1d(data, options={}): Required content of data: calibrant (str): path to .poni-file nbins (int): Number of bins to divide image into - path (str) (optional): path to image file - either this or image must be specified. If both is passed, image is prioritsed - image (str) (optional): image array (Numpy) as extracted from get_image_array + path (str) (optional, dependent on image): path to image file - either this or image must be specified. If both is passed, image is prioritsed + image (NumPy 2D Array) (optional, dependent on path): image array as extracted from get_image_array Output: df: DataFrame contianing 1D diffractogram if option 'return' is True ''' - required_options = ['unit', 'extension', 'filename', 'save_folder', 'overwrite', 'return'] + required_options = ['unit', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite'] default_options = { 'unit': '2th_deg', - 'extension': '_integrated.dat', - 'filename': None, + 'save': False, + 'save_filename': None, + 'save_extension': '_integrated.xy', 'save_folder': '.', - 'overwrite': False, - 'return': False} + 'overwrite': False} - if not options: - options = default_options - - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] - + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + # Get image array from filename if not passed if 'image' not in data.keys(): data['image'] = get_image_array(data['path']) + # Instanciate the azimuthal integrator from pyFAI from the calibrant (.poni-file) ai = pyFAI.load(data['calibrant']) - - if not options['filename']: - if data['path']: - filename = os.path.join(options['save_folder'], os.path.split(data['path'])[-1].split('.')[0] + options['extension']) - else: - filename = os.path.join(options['save_folder'], 'integrated.dat') - - - if not options['overwrite']: - trunk = os.path.join(options['save_folder'], filename.split('\\')[-1].split('.')[0]) - extension = filename.split('.')[-1] - counter = 0 - - while os.path.isfile(filename): - counter_string = str(counter) - filename = trunk + '_' + counter_string.zfill(4) + '.' + extension - counter += 1 - - + # Determine filename + filename = make_filename(data=data, options=options) + + # Make save_folder if this does not exist already if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) res = ai.integrate1d(data['image'], data['nbins'], unit=options['unit'], filename=filename) - return read_diffractogram(filename) + diffractogram = read_xy(filename) + + if not options['save']: + os.remove(filename) + shutil.rmtree('tmp') + + return diffractogram +def make_filename(data, options): + + # Define save location for integrated diffractogram data + if not options['save']: + options['save_folder'] = 'tmp' + filename = os.path.join(options['save_folder'], 'tmp_diff.dat') + + elif options['save']: + + # Case 1: No filename is given. + if not options['save_filename']: + # If a path is given instead of an image array, the path is taken as the trunk of the savename + if data['path']: + # Make filename by joining the save_folder, the filename (with extension deleted) and adding the save_extension + filename = os.path.join(options['save_folder'], os.path.split(data['path'])[-1].split('.')[0] + options['save_extension']) + else: + # Make filename just "integrated.dat" in the save_folder + filename = os.path.join(options['save_folder'], 'integrated.xy') + + + else: + filename = os.path.join(options['save_folder'], options['save_filename']) + + + if not options['overwrite']: + trunk = filename.split('.')[0] + extension = filename.split('.')[-1] + counter = 0 + + while os.path.isfile(filename): + + # Rename first file to match naming scheme if already exists + if counter == 0: + os.rename(filename, trunk + '_' + str(counter).zfill(4) + '.' + extension) + + # Increment counter and make new filename + counter += 1 + counter_string = str(counter) + filename = trunk + '_' + counter_string.zfill(4) + '.' + extension + + + return filename @@ -220,16 +252,15 @@ def read_brml(path, options=None): return diffractogram -def read_diffractogram(path): +def read_xy(data): - - - with open(path, 'r') as f: + with open(data['path'], 'r') as f: position = 0 current_line = f.readline() - while current_line[0] == '#': + while current_line[0] == '#' or "\'": + find_wavelength(line=current_line, data=data) position = f.tell() current_line = f.readline() @@ -237,7 +268,10 @@ def read_diffractogram(path): diffractogram = pd.read_csv(f, header=None, delim_whitespace=True) - diffractogram.columns = ['2th', 'I'] + if diffractogram.shape[1] == 2: + diffractogram.columns = ['2th', 'I'] + elif diffractogram.shape[1] == 3: + diffractogram.columns = ['2th', 'I', 'sigma'] return diffractogram @@ -245,22 +279,17 @@ def read_diffractogram(path): def read_data(data, options={}): - if data['plot_kind'] == 'beamline': + beamline_extensions = ['mar3450', 'edf', 'cbf'] + file_extension = data['path'].split('.')[-1] - beamline = ['mar3450', 'edf', 'cbf'] - - if data['path'].split('.')[-1] in beamline: - diffractogram = integrate_1d(data=data, options=options) + if file_extension in beamline_extensions: + diffractogram = integrate_1d(data=data, options=options) - else: - diffractogram = read_diffractogram(data['path']) - - elif data['plot_kind'] == 'recx': - diffractogram = read_brml(data['path'], options=options) - - elif data['plot_kind'] == 'image': - diffractogram = get_image_array(data['path']) + elif file_extension == 'brml': + diffractogram = read_brml(path=data['path'], options=options) + elif file_extension in['xy', 'xye']: + diffractogram = read_xy(data['path']) return diffractogram @@ -290,4 +319,50 @@ def load_reflection_table(path): # Set the new modified headers as the headers of reflections.columns = headers - return reflections \ No newline at end of file + return reflections + + + +def translate_wavelengths(diffractogram, wavelength): + + # Translate to CuKalpha + cuka = 1.54059 # Å + + if cuka > wavelength: + max_2th_cuka = 2*np.arcsin(wavelength/cuka) * 180/np.pi + else: + max_2th_cuka = diffractogram['2th'].max() + + diffractogram['2th_cuka'] = np.NAN + + diffractogram['2th_cuka'].loc[diffractogram['2th'] <= max_2th_cuka] = 2*np.arcsin(cuka/wavelength * np.sin((diffractogram['2th']/2) * np.pi/180)) * 180/np.pi + + # Translate to MoKalpha + moka = 0.71073 # Å + + if moka > wavelength: + max_2th_moka = 2*np.arcsin(wavelength/moka) * 180/np.pi + else: + max_2th_moka = diffractogram['2th'].max() + + diffractogram['2th_moka'] = np.NAN + + diffractogram['2th_moka'].loc[diffractogram['2th'] <= max_2th_moka] = 2*np.arcsin(moka/wavelength * np.sin((diffractogram['2th']/2) * np.pi/180)) * 180/np.pi + + + # Convert to other parameters + diffractogram['d'] = wavelength / (2*np.sin(2*diffractogram['2th']) * 180/np.pi) + diffractogram['1/d'] = 1/diffractogram['d'] + diffractogram['q'] = np.abs((4*np.pi/wavelength)*np.sin(diffractogram['2th']/2 * np.pi/180)) + diffractogram['q2'] = diffractogram['q']**2 + diffractogram['q4'] = diffractogram['q']**4 + + + +def find_wavelength(line, data): + + # Find from EVA-exports + wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} + + if 'Anode' in line: + \ No newline at end of file diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 7c89c43..9e9ecee 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -183,11 +183,14 @@ def plot_reflection_indices(data, ax, options={}): reflection_table = xrd.io.load_reflection_table(data['path']) if data['reflection_indices'] > 0: - reflection_indices = reflection_table.nlargest(options['reflection_indices'], 'I') - - for i in range(data['reflection_indices']): - ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices['2th'].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=data['text_colour']) + # Get the data['reflection_indices'] number of highest reflections within the subrange options['xlim'] + reflection_indices = reflection_table.loc[(reflection_table[options['x_vals']] > options['xlim'][0]) & (reflection_table[options['x_vals']] < options['xlim'][1])].nlargest(options['reflection_indices'], 'I') + + # Plot the indices + for i in range(data['reflection_indices']): + if reflection_indices.shape[0] > i: + ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices[options['x_vals']].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=data['text_colour']) if options['xlim']: From decb69a59958d2c3e750257f0e0d11cfe671cb19 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 16 Mar 2022 15:21:05 +0100 Subject: [PATCH 077/355] Fix find_wavelength --- beamtime/xrd/io.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index b5a479c..d6d35c7 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -252,7 +252,10 @@ def read_brml(path, options=None): return diffractogram -def read_xy(data): +def read_xy(data, options): + + if 'wavelength' not in data.keys(): + find_wavelength(data=data, file_ext='xy') with open(data['path'], 'r') as f: position = 0 @@ -260,7 +263,6 @@ def read_xy(data): current_line = f.readline() while current_line[0] == '#' or "\'": - find_wavelength(line=current_line, data=data) position = f.tell() current_line = f.readline() @@ -286,10 +288,10 @@ def read_data(data, options={}): diffractogram = integrate_1d(data=data, options=options) elif file_extension == 'brml': - diffractogram = read_brml(path=data['path'], options=options) + diffractogram = read_brml(data=data, options=options) elif file_extension in['xy', 'xye']: - diffractogram = read_xy(data['path']) + diffractogram = read_xy(data=data, options=options) return diffractogram @@ -359,10 +361,29 @@ def translate_wavelengths(diffractogram, wavelength): -def find_wavelength(line, data): +def find_wavelength(data, file_ext): + + # Find from EVA-exports (.xy) + if file_ext == 'xy': + wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} + + with open(data['path'], 'r') as f: + lines = f.readlines() + + for line in lines: + if 'Anode' in line: + anode = line.split()[8].strip('"') + data['wavelength'] = wavelength_dict[anode] + + + # Find from .poni-file + if file_ext in ['mar3450', 'edf', 'cbf']: + if 'calibrant' in data.keys(): + with open(data['calibrant'], 'r') as f: + lines = f.readlines() + + for line in lines: + if 'Wavelength' in line: + data['wavelength'] = float(line.split[-1]) - # Find from EVA-exports - wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} - if 'Anode' in line: - \ No newline at end of file From 17404a36e4d7c426df4239d60a089c25a5e45a80 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 16 Mar 2022 15:23:55 +0100 Subject: [PATCH 078/355] Fix integrate_1d() --- beamtime/xrd/io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index d6d35c7..aa9d09a 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -69,7 +69,8 @@ def integrate_1d(data, options={}): res = ai.integrate1d(data['image'], data['nbins'], unit=options['unit'], filename=filename) - diffractogram = read_xy(filename) + diff_data = {'path': filename} + diffractogram = read_xy(data=diff_data, options=options) if not options['save']: os.remove(filename) From 6f2d96005d3d7628504c52103d521a500632fb24 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 16 Mar 2022 15:36:40 +0100 Subject: [PATCH 079/355] Fix read_xy() --- beamtime/xrd/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index aa9d09a..da45aad 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -260,10 +260,10 @@ def read_xy(data, options): with open(data['path'], 'r') as f: position = 0 - + current_line = f.readline() - while current_line[0] == '#' or "\'": + while current_line[0] == '#' or current_line[0] == '\'': position = f.tell() current_line = f.readline() From a96210cf075b40cff5855afb4ed37c69f64920b6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 16 Mar 2022 16:16:05 +0100 Subject: [PATCH 080/355] Add change modes in interactive mode --- beamtime/xrd/io.py | 88 +++++++++++++++++++++----------------------- beamtime/xrd/plot.py | 3 +- 2 files changed, 43 insertions(+), 48 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index da45aad..9f80e76 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -69,8 +69,8 @@ def integrate_1d(data, options={}): res = ai.integrate1d(data['image'], data['nbins'], unit=options['unit'], filename=filename) - diff_data = {'path': filename} - diffractogram = read_xy(data=diff_data, options=options) + data['path'] = filename + diffractogram = read_xy(data=data, options=options) if not options['save']: os.remove(filename) @@ -175,24 +175,17 @@ def view_integrator(calibrant): -def read_brml(path, options=None): +def read_brml(data, options={}): required_options = ['extract_folder', 'save_folder'] default_options = { - 'extract_folder': 'temp', + 'extract_folder': 'tmp', 'save_folder': None } - if not options: - options = default_options - - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] - + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) if not os.path.isdir(options['extract_folder']): @@ -200,7 +193,7 @@ def read_brml(path, options=None): # Extract the RawData0.xml file from the brml-file - with zipfile.ZipFile(path, 'r') as brml: + with zipfile.ZipFile(data['path'], 'r') as brml: for info in brml.infolist(): if "RawData" in info.filename: brml.extract(info.filename, options['extract_folder']) @@ -223,21 +216,26 @@ def read_brml(path, options=None): if scantype.text == 'StillScan': if chain.get('Description') == 'Originally measured data.': - for data in chain.findall('Datum'): - data = data.text.split(',') - data = [float(i) for i in data] - twotheta, intensity = float(data[2]), float(data[3]) + for scandata in chain.findall('Datum'): + scandata = scandata.text.split(',') + scandata = [float(i) for i in scandata] + twotheta, intensity = float(scandata[2]), float(scandata[3]) else: if chain.get('Description') == 'Originally measured data.': - for data in chain.findall('Datum'): - data = data.text.split(',') - twotheta, intensity = float(data[2]), float(data[3]) + for scandata in chain.findall('Datum'): + scandata = scandata.text.split(',') + twotheta, intensity = float(scandata[2]), float(scandata[3]) if twotheta > 0: diffractogram.append({'2th': twotheta, 'I': intensity}) + + if 'wavelength' not in data.keys(): + for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): + data['wavelength'] = float(chain.attrib['Value']) + diffractogram = pd.DataFrame(diffractogram) @@ -253,10 +251,10 @@ def read_brml(path, options=None): return diffractogram -def read_xy(data, options): +def read_xy(data, options={}): if 'wavelength' not in data.keys(): - find_wavelength(data=data, file_ext='xy') + find_wavelength_from_xy(data=data) with open(data['path'], 'r') as f: position = 0 @@ -294,6 +292,9 @@ def read_data(data, options={}): elif file_extension in['xy', 'xye']: diffractogram = read_xy(data=data, options=options) + + diffractogram = translate_wavelengths(diffractogram=diffractogram, wavelength=data['wavelength']) + return diffractogram @@ -361,30 +362,23 @@ def translate_wavelengths(diffractogram, wavelength): diffractogram['q4'] = diffractogram['q']**4 - -def find_wavelength(data, file_ext): - - # Find from EVA-exports (.xy) - if file_ext == 'xy': - wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} - - with open(data['path'], 'r') as f: - lines = f.readlines() - - for line in lines: - if 'Anode' in line: - anode = line.split()[8].strip('"') - data['wavelength'] = wavelength_dict[anode] - - - # Find from .poni-file - if file_ext in ['mar3450', 'edf', 'cbf']: - if 'calibrant' in data.keys(): - with open(data['calibrant'], 'r') as f: - lines = f.readlines() - - for line in lines: - if 'Wavelength' in line: - data['wavelength'] = float(line.split[-1]) + return diffractogram + +def find_wavelength_from_xy(data): + + + wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} + + with open(data['path'], 'r') as f: + lines = f.readlines() + + for line in lines: + if 'Anode' in line: + anode = line.split()[8].strip('"') + data['wavelength'] = wavelength_dict[anode] + + elif 'Wavelength' in line: + data['wavelength'] = float(line.split()[2])*10**10 + diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 9e9ecee..4fdb5df 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -151,7 +151,8 @@ def plot_diffractogram_interactive(data, options): line=widgets.ToggleButton(value=True), reflections_plot=widgets.ToggleButton(value=True), reflections_indices=widgets.ToggleButton(value=False), - xlim=widgets.IntRangeSlider(value=options['xlim'], min=options['xlim'][0], max=options['xlim'][1], step=1, description='xlim', layout=widgets.Layout(width='1000px'))) + x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q4', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), + xlim=widgets.FloatRangeSlider(value=options['xlim'], min=options['xlim'][0], max=options['xlim'][1], step=1, description='xlim', layout=widgets.Layout(width='1000px'))) else: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), From 9bb52fe819157d0f5f2f8a3cc8ec27d96267b83c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 18 Mar 2022 16:55:43 +0100 Subject: [PATCH 081/355] Add translation of wavelengths and xlim updates --- beamtime/plotting.py | 11 ++ beamtime/xrd/io.py | 65 ++++++--- beamtime/xrd/plot.py | 331 +++++++------------------------------------ 3 files changed, 107 insertions(+), 300 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index c78a989..3430a2a 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -298,6 +298,7 @@ def ipywidgets_update(func, data, options={}, **kwargs): where key1, key2, key3 etc. are the values in the options-dictionary you want widget control of, and widget1, widget2, widget3 etc. are widgets to control these values, e.g. widgets.IntSlider(value=1, min=0, max=10) ''' + # Update the options-dictionary with the values from the widgets for key in kwargs: options[key] = kwargs[key] @@ -306,6 +307,16 @@ def ipywidgets_update(func, data, options={}, **kwargs): func(data=data, options=options) +def update_widgets(options): + + for widget in options['widgets'].values(): + + if widget['state'] != options['x_vals']: + for arg in widget[f'{options["x_vals"]}_default']: + setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) + + widget['state'] = options['x_vals'] + def determine_width(format_params): diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 9f80e76..6b0bb7a 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -1,3 +1,4 @@ +from sympy import re import fabio, pyFAI import pandas as pd import numpy as np @@ -252,7 +253,7 @@ def read_brml(data, options={}): def read_xy(data, options={}): - + if 'wavelength' not in data.keys(): find_wavelength_from_xy(data=data) @@ -293,23 +294,32 @@ def read_data(data, options={}): diffractogram = read_xy(data=data, options=options) - diffractogram = translate_wavelengths(diffractogram=diffractogram, wavelength=data['wavelength']) + diffractogram = translate_wavelengths(data=diffractogram, wavelength=data['wavelength']) return diffractogram -def load_reflection_table(path): +def load_reflection_table(data, options={}): + + required_options = ['wavelength', 'to_wavelength'] + + default_options = { + 'wavelength': 1.54059, + 'to_wavelength': None + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) # VESTA outputs the file with a header that has a space between the parameter and units - so there is some extra code to rectify the issue # that ensues from this formatting - reflections = pd.read_csv(path, delim_whitespace=True) + reflections = pd.read_csv(data['path'], delim_whitespace=True) # Remove the extra column that appears from the headers issue reflections.drop(reflections.columns[-1], axis=1, inplace=True) - with open(path, 'r') as f: + with open(data['path'], 'r') as f: line = f.readline() headers = line.split() @@ -323,11 +333,16 @@ def load_reflection_table(path): # Set the new modified headers as the headers of reflections.columns = headers + reflections = translate_wavelengths(data=reflections, wavelength=options['wavelength'], to_wavelength=options['to_wavelength']) + + #print(reflections) + return reflections -def translate_wavelengths(diffractogram, wavelength): +def translate_wavelengths(data, wavelength, to_wavelength=None): + pd.options.mode.chained_assignment = None # Translate to CuKalpha cuka = 1.54059 # Å @@ -335,11 +350,11 @@ def translate_wavelengths(diffractogram, wavelength): if cuka > wavelength: max_2th_cuka = 2*np.arcsin(wavelength/cuka) * 180/np.pi else: - max_2th_cuka = diffractogram['2th'].max() + max_2th_cuka = data['2th'].max() - diffractogram['2th_cuka'] = np.NAN + data['2th_cuka'] = np.NAN - diffractogram['2th_cuka'].loc[diffractogram['2th'] <= max_2th_cuka] = 2*np.arcsin(cuka/wavelength * np.sin((diffractogram['2th']/2) * np.pi/180)) * 180/np.pi + data['2th_cuka'].loc[data['2th'] <= max_2th_cuka] = 2*np.arcsin(cuka/wavelength * np.sin((data['2th']/2) * np.pi/180)) * 180/np.pi # Translate to MoKalpha moka = 0.71073 # Å @@ -347,22 +362,36 @@ def translate_wavelengths(diffractogram, wavelength): if moka > wavelength: max_2th_moka = 2*np.arcsin(wavelength/moka) * 180/np.pi else: - max_2th_moka = diffractogram['2th'].max() + max_2th_moka = data['2th'].max() - diffractogram['2th_moka'] = np.NAN + data['2th_moka'] = np.NAN - diffractogram['2th_moka'].loc[diffractogram['2th'] <= max_2th_moka] = 2*np.arcsin(moka/wavelength * np.sin((diffractogram['2th']/2) * np.pi/180)) * 180/np.pi + data['2th_moka'].loc[data['2th'] <= max_2th_moka] = 2*np.arcsin(moka/wavelength * np.sin((data['2th']/2) * np.pi/180)) * 180/np.pi # Convert to other parameters - diffractogram['d'] = wavelength / (2*np.sin(2*diffractogram['2th']) * 180/np.pi) - diffractogram['1/d'] = 1/diffractogram['d'] - diffractogram['q'] = np.abs((4*np.pi/wavelength)*np.sin(diffractogram['2th']/2 * np.pi/180)) - diffractogram['q2'] = diffractogram['q']**2 - diffractogram['q4'] = diffractogram['q']**4 + data['d'] = wavelength / (2*np.sin((2*data['2th']*np.pi/180)/2)) + data['1/d'] = 1/data['d'] + data['q'] = np.abs((4*np.pi/wavelength)*np.sin(data['2th']/2 * np.pi/180)) + data['q2'] = data['q']**2 + data['q4'] = data['q']**4 - return diffractogram + if to_wavelength: + + if to_wavelength > cuka: + max_2th = 2*np.arcsin(cuka/to_wavelength) * 180/np.pi + else: + max_2th = data['2th_cuka'].max() + + + data['2th'] = np.NAN + data['2th'].loc[data['2th_cuka'] <= max_2th] = 2*np.arcsin(to_wavelength/cuka * np.sin((data['2th_cuka']/2) * np.pi/180)) * 180/np.pi + + + + + return data diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 4fdb5df..daa7eaf 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -44,12 +44,13 @@ def plot_diffractogram(data, options={}): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) if not 'diffractogram' in data.keys(): - diffractogram = xrd.io.read_data(data=data) + diffractogram = xrd.io.read_data(data=data, options=options) data['diffractogram'] = diffractogram else: diffractogram = data['diffractogram'] + # Sets the xlim if this has not bee specified if not options['xlim']: options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] @@ -62,6 +63,7 @@ def plot_diffractogram(data, options={}): return + # Makes a list out of reflections_data if it only passed as a dict, as it will be looped through later if options['reflections_data']: if not isinstance(options['reflections_data'], list): options['reflections_data'] = [options['reflections_data']] @@ -93,9 +95,6 @@ def plot_diffractogram(data, options={}): colours = btp.generate_colours(options['palettes']) - - - if options['line']: diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) @@ -104,8 +103,6 @@ def plot_diffractogram(data, options={}): - - fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) @@ -113,6 +110,7 @@ def plot_diffractogram(data, options={}): # Make the reflection plots if options['reflections_plot'] and options['reflections_data']: options['xlim'] = ax.get_xlim() + options['to_wavelength'] = data['wavelength'] for reference, axis in zip(options['reflections_data'], ref_axes): plot_reflection_table(data=reference, ax=axis, options=options) @@ -120,10 +118,16 @@ def plot_diffractogram(data, options={}): # Print the reflection indices if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() + options['to_wavelength'] = data['wavelength'] + for reference in options['reflections_data']: plot_reflection_indices(data=reference, ax=indices_ax, options=options) + if options['interactive_session_active']: + btp.update_widgets(options=options) + + return diffractogram, fig, ax @@ -145,25 +149,53 @@ def determine_grid_layout(options): def plot_diffractogram_interactive(data, options): + options['widgets'] = { + 'xlim': { + 'w': widgets.FloatRangeSlider(value=[data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], min=data['diffractogram']['2th'].min(), max=data['diffractogram']['2th'].max(), step=0.5, layout=widgets.Layout(width='95%')), + '2th_default': {'min': data['diffractogram']['2th'].min(), 'max': data['diffractogram']['2th'].max(), 'value': [data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], 'step': 0.5}, + '2th_cuka_default': {'min': data['diffractogram']['2th_cuka'].min(), 'max': data['diffractogram']['2th_cuka'].max(), 'value': [data['diffractogram']['2th_cuka'].min(), data['diffractogram']['2th_cuka'].max()], 'step': 0.5}, + '2th_moka_default': {'min': data['diffractogram']['2th_moka'].min(), 'max': data['diffractogram']['2th_moka'].max(), 'value': [data['diffractogram']['2th_moka'].min(), data['diffractogram']['2th_moka'].max()], 'step': 0.5}, + 'd_default': {'min': data['diffractogram']['d'].min(), 'max': data['diffractogram']['d'].max(), 'value': [data['diffractogram']['d'].min(), data['diffractogram']['d'].max()], 'step': 0.5}, + '1/d_default': {'min': data['diffractogram']['1/d'].min(), 'max': data['diffractogram']['1/d'].max(), 'value': [data['diffractogram']['1/d'].min(), data['diffractogram']['1/d'].max()], 'step': 0.5}, + 'q_default': {'min': data['diffractogram']['q'].min(), 'max': data['diffractogram']['q'].max(), 'value': [data['diffractogram']['q'].min(), data['diffractogram']['q'].max()], 'step': 0.5}, + 'q2_default': {'min': data['diffractogram']['q2'].min(), 'max': data['diffractogram']['q2'].max(), 'value': [data['diffractogram']['q2'].min(), data['diffractogram']['q2'].max()], 'step': 0.5}, + 'q4_default': {'min': data['diffractogram']['q4'].min(), 'max': data['diffractogram']['q4'].max(), 'value': [data['diffractogram']['q4'].min(), data['diffractogram']['q4'].max()], 'step': 0.5}, + 'state': '2th' + } + } + + if options['reflections_data']: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), reflections_plot=widgets.ToggleButton(value=True), reflections_indices=widgets.ToggleButton(value=False), - x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q4', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), - xlim=widgets.FloatRangeSlider(value=options['xlim'], min=options['xlim'][0], max=options['xlim'][1], step=1, description='xlim', layout=widgets.Layout(width='1000px'))) + x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), + xlim=options['widgets']['xlim']['w']) else: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), - xlim=widgets.IntRangeSlider(value=options['xlim'], min=options['xlim'][0], max=options['xlim'][1], step=1, description='xlim')) + xlim=options['widgets']['xlim']['w']) display(w) +def update_widgets(options): + + for widget in options['widgets'].values(): + + if widget['state'] != options['x_vals']: + for arg in widget[f'{options["x_vals"]}_default']: + setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) + + widget['state'] = options['x_vals'] + + + def plot_reflection_indices(data, ax, options={}): ''' Print reflection indices from output generated by VESTA. @@ -178,10 +210,10 @@ def plot_reflection_indices(data, ax, options={}): 'hide_indices': False } - data = update_options(options=data, required_options=required_options, default_options=default_options) + data = aux.update_options(options=data, required_options=required_options, default_options=default_options) if not data['hide_indices']: - reflection_table = xrd.io.load_reflection_table(data['path']) + reflection_table = xrd.io.load_reflection_table(data=data, options=options) if data['reflection_indices'] > 0: @@ -208,12 +240,13 @@ def plot_reflection_table(data, ax=None, options={}): Required contents of data: path (str): relative path to reflection table file''' - required_options = ['reflection_indices', 'reflections_colour', 'min_alpha', 'format_params', 'rc_params', 'label'] + required_options = ['reflection_indices', 'reflections_colour', 'min_alpha', 'wavelength', 'format_params', 'rc_params', 'label'] default_options = { 'reflection_indices': 0, # Number of indices to print 'reflections_colour': [0,0,0], 'min_alpha': 0, + 'wavelength': 1.54059, # CuKalpha, [Å] 'format_params': {}, 'rc_params': {}, 'label': None @@ -227,7 +260,8 @@ def plot_reflection_table(data, ax=None, options={}): options['reflection_indices'] = data['reflection_indices'] if 'label' in data.keys(): options['label'] = data['label'] - + if 'wavelength' in data.keys(): + options['wavelength'] = data['wavelength'] options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -235,9 +269,9 @@ def plot_reflection_table(data, ax=None, options={}): if not ax: _, ax = btp.prepare_plot(options) - reflection_table = xrd.io.load_reflection_table(data['path']) + reflection_table = xrd.io.load_reflection_table(data=data, options=options) - reflections, intensities = reflection_table['2th'], reflection_table['I'] + reflections, intensities = reflection_table[options['x_vals']], reflection_table['I'] @@ -272,257 +306,6 @@ def plot_reflection_table(data, ax=None, options={}): - - - -def prepare_diffractogram_plot(options=None): - # First take care of the options for plotting - set any values not specified to the default values - required_options = ['columns', 'width', 'height', 'format', 'dpi', 'facecolor'] - default_options = {'columns': 1, 'width': 14, 'format': 'golden_ratio', 'dpi': None, 'facecolor': 'w'} - - - # Define the required sizes - required_sizes = ['lines', 'axes'] - - - - - # If none are set at all, just pass the default_options - if not options: - options = default_options - options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 - options['figsize'] = (options['width'], options['height']) - - - # Define default sizes - default_sizes = { - 'lines': 3*options['columns'], - 'axes': 3*options['columns'] - } - - # Initialise dictionary if it doesn't exist - if not 'sizes' in options.keys(): - options['sizes'] = {} - - - # Update dictionary with default values where none is supplied - for size in required_sizes: - if size not in options['sizes']: - options['sizes'][size] = default_sizes[size] - - - # If options is passed, go through to fill out the rest. - else: - # Start by setting the width: - if 'width' not in options.keys(): - options['width'] = default_options['width'] - - # Then set height - check options for format. If not given, set the height to the width scaled by the golden ratio - if the format is square, set the same. This should possibly allow for the tweaking of custom ratios later. - if 'height' not in options.keys(): - if 'format' not in options.keys(): - options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 - elif options['format'] == 'square': - options['height'] = options['width'] - - options['figsize'] = (options['width'], options['height']) - - # After height and width are set, go through the rest of the options to make sure that all the required options are filled - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] - - fig, ax = plt.subplots(figsize=(options['figsize']), dpi=options['dpi'], facecolor=options['facecolor']) - - - plt.rc('lines', linewidth=options['sizes']['lines']) - plt.rc('axes', linewidth=options['sizes']['axes']) - - return fig, ax - -def prettify_diffractogram_plot(fig, ax, options=None): - - ################################################################## - ######################### UPDATE OPTIONS ######################### - ################################################################## - - # Define the required options - required_options = [ - 'columns', - 'xticks', 'yticks', - 'units', - 'show_major_ticks', 'show_minor_ticks', 'show_tick_labels', - 'xlim', 'ylim', - 'hide_x_axis', 'hide_y_axis', - 'positions', - 'xlabel', 'ylabel', - 'sizes', - 'title' - ] - - - # Define the default options - default_options = { - 'columns': 1, - 'xticks': [10, 5], 'yticks': [10000, 5000], - 'units': {'2th': '$^o$', 'I': 'arb. u.'}, - 'show_major_ticks': [True, False, True, False], 'show_minor_ticks': [True, False, True, False], 'show_tick_labels': [True, False, False, False], - 'xlim': None,'ylim': None, - 'hide_x_axis': False, 'hide_y_axis': False, - 'positions': {'xaxis': 'bottom', 'yaxis': 'left'}, - 'xlabel': None, 'ylabel': None, - 'sizes': None, - 'title': None - } - - options = update_options(options, required_options, default_options) - - - - ################################################################## - ########################## DEFINE SIZES ########################## - ################################################################## - - # Define the required sizes - required_sizes = [ - 'labels', - 'legend', - 'title', - 'line', 'axes', - 'tick_labels', - 'major_ticks', 'minor_ticks'] - - - - # Define default sizes - default_sizes = { - 'labels': 30*options['columns'], - 'legend': 30*options['columns'], - 'title': 30*options['columns'], - 'line': 3*options['columns'], - 'axes': 3*options['columns'], - 'tick_labels': 30*options['columns'], - 'major_ticks': 20*options['columns'], - 'minor_ticks': 10*options['columns'] - } - - # Initialise dictionary if it doesn't exist - if not options['sizes']: - options['sizes'] = {} - - - # Update dictionary with default values where none is supplied - for size in required_sizes: - if size not in options['sizes']: - options['sizes'][size] = default_sizes[size] - - - ################################################################## - ########################## AXIS LABELS ########################### - ################################################################## - - - if not options['xlabel']: - options['xlabel'] = prettify_labels(options['x_vals']) + ' [{}]'.format(options['units'][options['x_vals']]) - - else: - options['xlabel'] = options['xlabel'] + ' [{}]'.format(options['units'][options['x_vals']]) - - - if not options['ylabel']: - options['ylabel'] = prettify_labels(options['y_vals']) + ' [{}]'.format(options['units'][options['y_vals']]) - - else: - options['ylabel'] = options['ylabel'] + ' [{}]'.format(options['units'][options['y_vals']]) - - ax.set_xlabel(options['xlabel'], size=options['sizes']['labels']) - ax.set_ylabel(options['ylabel'], size=options['sizes']['labels']) - - ################################################################## - ###################### TICK MARKS & LABELS ####################### - ################################################################## - - ax.tick_params( - direction='in', - which='major', - bottom=options['show_major_ticks'][0], labelbottom=options['show_tick_labels'][0], - left=options['show_major_ticks'][1], labelleft=options['show_tick_labels'][1], - top=options['show_major_ticks'][2], labeltop=options['show_tick_labels'][2], - right=options['show_major_ticks'][3], labelright=options['show_tick_labels'][3], - length=options['sizes']['major_ticks'], - width=options['sizes']['axes']) - - - ax.tick_params(direction='in', which='minor', bottom=options['show_minor_ticks'][0], left=options['show_minor_ticks'][1], top=options['show_minor_ticks'][2], right=options['show_minor_ticks'][3], length=options['sizes']['minor_ticks'], width=options['sizes']['axes']) - - - - if options['positions']['yaxis'] == 'right': - ax.yaxis.set_label_position("right") - ax.yaxis.tick_right() - - - if options['hide_x_axis']: - ax.axes.xaxis.set_visible(False) - - if options['hide_y_axis']: - ax.axes.yaxis.set_visible(False) - - - - # Otherwise apply user input - if options['xticks']: - major_xtick = options['xticks'][0] - minor_xtick = options['xticks'][1] - - - if options['yticks']: - - major_ytick = options['yticks'][0] - minor_ytick = options['yticks'][1] - - - # Apply values - ax.xaxis.set_major_locator(MultipleLocator(major_xtick)) - ax.xaxis.set_minor_locator(MultipleLocator(minor_xtick)) - - ax.yaxis.set_major_locator(MultipleLocator(major_ytick)) - ax.yaxis.set_minor_locator(MultipleLocator(minor_ytick)) - - - - - # SET FONTSIZE OF TICK LABELS - - plt.xticks(fontsize=options['sizes']['tick_labels']) - plt.yticks(fontsize=options['sizes']['tick_labels']) - - ################################################################## - ########################## AXES LIMITS ########################### - ################################################################## - - if options['xlim']: - plt.xlim(options['xlim']) - - if options['ylim']: - plt.ylim(options['ylim']) - - ################################################################## - ############################# TITLE ############################## - ################################################################## - - if options['title']: - ax.set_title(options['title'], size=options['sizes']['title']) - - ################################################################## - ############################# LEGEND ############################# - ################################################################## - - if ax.get_legend(): - ax.get_legend().remove() - - return fig, ax - - def prettify_labels(label): @@ -593,19 +376,3 @@ def reverse_diffractograms(diffractograms): return rev_diffractograms #def plot_heatmap(): - - - - - -def update_options(options, required_options, default_options): - - if not options: - options = default_options - - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] - - return options \ No newline at end of file From da2b2f855b2002358b94e6533fdf19f5d2dad6dc Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 19:57:21 +0100 Subject: [PATCH 082/355] Change plotting method to allow for multiple diffractograms --- beamtime/plotting.py | 11 ++++++ beamtime/xrd/io.py | 77 ++++++++++++++++++++++----------------- beamtime/xrd/plot.py | 87 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 51 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index 3430a2a..dd9a481 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -313,6 +313,17 @@ def update_widgets(options): if widget['state'] != options['x_vals']: for arg in widget[f'{options["x_vals"]}_default']: + + # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first + if arg == 'min': + if widget[f'{options["x_vals"]}_default']['min'] > getattr(widget['w'], 'max'): + setattr(widget['w'], 'max', widget[f'{options["x_vals"]}_default']['max']) + + elif arg == 'max': + if widget[f'{options["x_vals"]}_default']['max'] < getattr(widget['w'], 'min'): + setattr(widget['w'], 'min', widget[f'{options["x_vals"]}_default']['min']) + + setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) widget['state'] = options['x_vals'] diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 6b0bb7a..3e29b80 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -27,7 +27,7 @@ def get_image_headers(path): return image.header -def integrate_1d(data, options={}): +def integrate_1d(data, options={}, index=0): ''' Integrates an image file to a 1D diffractogram. Required content of data: @@ -51,11 +51,14 @@ def integrate_1d(data, options={}): 'overwrite': False} options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + if not isinstance(data['path'], list): + data['path'] = [data['path']] # Get image array from filename if not passed if 'image' not in data.keys(): - data['image'] = get_image_array(data['path']) + data['image'] = get_image_array(data['path'][index]) # Instanciate the azimuthal integrator from pyFAI from the calibrant (.poni-file) ai = pyFAI.load(data['calibrant']) @@ -70,18 +73,18 @@ def integrate_1d(data, options={}): res = ai.integrate1d(data['image'], data['nbins'], unit=options['unit'], filename=filename) - data['path'] = filename - diffractogram = read_xy(data=data, options=options) + data['path'][index] = filename + diffractogram, wavelength = read_xy(data=data, options=options) if not options['save']: os.remove(filename) shutil.rmtree('tmp') - return diffractogram + return diffractogram, wavelength -def make_filename(data, options): +def make_filename(options, path=None): # Define save location for integrated diffractogram data if not options['save']: @@ -93,7 +96,7 @@ def make_filename(data, options): # Case 1: No filename is given. if not options['save_filename']: # If a path is given instead of an image array, the path is taken as the trunk of the savename - if data['path']: + if path: # Make filename by joining the save_folder, the filename (with extension deleted) and adding the save_extension filename = os.path.join(options['save_folder'], os.path.split(data['path'])[-1].split('.')[0] + options['save_extension']) else: @@ -176,12 +179,12 @@ def view_integrator(calibrant): -def read_brml(data, options={}): +def read_brml(data, options={}, index=0): required_options = ['extract_folder', 'save_folder'] default_options = { - 'extract_folder': 'tmp', + 'extract_folder': 'tmp/', 'save_folder': None } @@ -194,7 +197,7 @@ def read_brml(data, options={}): # Extract the RawData0.xml file from the brml-file - with zipfile.ZipFile(data['path'], 'r') as brml: + with zipfile.ZipFile(data['path'][index], 'r') as brml: for info in brml.infolist(): if "RawData" in info.filename: brml.extract(info.filename, options['extract_folder']) @@ -233,9 +236,10 @@ def read_brml(data, options={}): diffractogram.append({'2th': twotheta, 'I': intensity}) - if 'wavelength' not in data.keys(): - for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): - data['wavelength'] = float(chain.attrib['Value']) + #if 'wavelength' not in data.keys(): + # Find wavelength + for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): + wavelength = float(chain.attrib['Value']) diffractogram = pd.DataFrame(diffractogram) @@ -249,15 +253,16 @@ def read_brml(data, options={}): - return diffractogram + return diffractogram, wavelength -def read_xy(data, options={}): +def read_xy(data, options={}, index=0): - if 'wavelength' not in data.keys(): - find_wavelength_from_xy(data=data) + #if 'wavelength' not in data.keys(): + # Get wavelength from scan + wavelength = find_wavelength_from_xy(path=data['path'][index]) - with open(data['path'], 'r') as f: + with open(data['path'][index], 'r') as f: position = 0 current_line = f.readline() @@ -276,37 +281,37 @@ def read_xy(data, options={}): diffractogram.columns = ['2th', 'I', 'sigma'] - return diffractogram + return diffractogram, wavelength -def read_data(data, options={}): +def read_data(data, options={}, index=0): beamline_extensions = ['mar3450', 'edf', 'cbf'] - file_extension = data['path'].split('.')[-1] + file_extension = data['path'][index].split('.')[-1] if file_extension in beamline_extensions: - diffractogram = integrate_1d(data=data, options=options) + diffractogram, wavelength = integrate_1d(data=data, options=options, index=index) elif file_extension == 'brml': - diffractogram = read_brml(data=data, options=options) + diffractogram, wavelength = read_brml(data=data, options=options, index=index) elif file_extension in['xy', 'xye']: - diffractogram = read_xy(data=data, options=options) + diffractogram, wavelength = read_xy(data=data, options=options, index=index) - diffractogram = translate_wavelengths(data=diffractogram, wavelength=data['wavelength']) + diffractogram = translate_wavelengths(data=diffractogram, wavelength=wavelength) - return diffractogram + return diffractogram, wavelength def load_reflection_table(data, options={}): - required_options = ['wavelength', 'to_wavelength'] + required_options = ['ref_wavelength', 'to_wavelength'] default_options = { - 'wavelength': 1.54059, + 'ref_wavelength': 1.54059, 'to_wavelength': None } @@ -333,7 +338,7 @@ def load_reflection_table(data, options={}): # Set the new modified headers as the headers of reflections.columns = headers - reflections = translate_wavelengths(data=reflections, wavelength=options['wavelength'], to_wavelength=options['to_wavelength']) + reflections = translate_wavelengths(data=reflections, wavelength=options['ref_wavelength'], to_wavelength=options['to_wavelength']) #print(reflections) @@ -378,6 +383,7 @@ def translate_wavelengths(data, wavelength, to_wavelength=None): if to_wavelength: + if to_wavelength > cuka: max_2th = 2*np.arcsin(cuka/to_wavelength) * 180/np.pi @@ -395,19 +401,24 @@ def translate_wavelengths(data, wavelength, to_wavelength=None): -def find_wavelength_from_xy(data): +def find_wavelength_from_xy(path): wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} - with open(data['path'], 'r') as f: + with open(path, 'r') as f: lines = f.readlines() for line in lines: + # For .xy-files output from EVA if 'Anode' in line: anode = line.split()[8].strip('"') - data['wavelength'] = wavelength_dict[anode] + wavelength = wavelength_dict[anode] + # For .xy-files output from pyFAI integration elif 'Wavelength' in line: - data['wavelength'] = float(line.split()[2])*10**10 + wavelength = float(line.split()[2])*10**10 + + + return wavelength diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index daa7eaf..cec2d1e 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -43,12 +43,25 @@ def plot_diffractogram(data, options={}): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + # Convert data['path'] to list to allow iteration over this to accommodate both single and multiple diffractograms + if not isinstance(data['path'], list): + data['path'] = [data['path']] + + if not 'diffractogram' in data.keys(): - diffractogram = xrd.io.read_data(data=data, options=options) - data['diffractogram'] = diffractogram + # Initialise empty list for diffractograms and wavelengths + data['diffractogram'] = [None for _ in range(len(data['path']))] + data['wavelength'] = [None for _ in range(len(data['path']))] + + for index in range(len(data['path'])): + diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) + data['diffractogram'][index] = diffractogram + data['wavelength'][index] = wavelength else: - diffractogram = data['diffractogram'] + if not isinstance(data['diffractogram'], list): + data['diffractogram'] = [data['diffractogram']] + data['wavelength'] = [data['wavelength']] # Sets the xlim if this has not bee specified if not options['xlim']: @@ -95,11 +108,12 @@ def plot_diffractogram(data, options={}): colours = btp.generate_colours(options['palettes']) - if options['line']: - diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) + for diffractogram in data['diffractogram']: + if options['line']: + diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) - if options['scatter']: - ax.scatter(x=diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) + if options['scatter']: + ax.scatter(x=diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) @@ -110,7 +124,7 @@ def plot_diffractogram(data, options={}): # Make the reflection plots if options['reflections_plot'] and options['reflections_data']: options['xlim'] = ax.get_xlim() - options['to_wavelength'] = data['wavelength'] + options['to_wavelength'] = data['wavelength'][0] for reference, axis in zip(options['reflections_data'], ref_axes): plot_reflection_table(data=reference, ax=axis, options=options) @@ -118,7 +132,7 @@ def plot_diffractogram(data, options={}): # Print the reflection indices if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() - options['to_wavelength'] = data['wavelength'] + options['to_wavelength'] = data['wavelength'][0] for reference in options['reflections_data']: plot_reflection_indices(data=reference, ax=indices_ax, options=options) @@ -149,17 +163,54 @@ def determine_grid_layout(options): def plot_diffractogram_interactive(data, options): + + minmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None]} + + for index, diffractogram in enumerate(data['diffractogram']): + if not minmax['2th'][0] or diffractogram['2th'].min() < minmax['2th'][0]: + minmax['2th'][0] = diffractogram['2th'].min() + min_index = index + + if not minmax['2th'][1] or diffractogram['2th'].max() > minmax['2th'][1]: + minmax['2th'][1] = diffractogram['2th'].max() + max_index = index + + minmax['2th_cuka'][0], minmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() + minmax['2th_moka'][0], minmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() + minmax['d'][0], minmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() + minmax['1/d'][0], minmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() + minmax['q'][0], minmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() + minmax['q2'][0], minmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() + minmax['q4'][0], minmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() + + + + # options['widgets'] = { + # 'xlim': { + # 'w': widgets.FloatRangeSlider(value=[data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], min=data['diffractogram']['2th'].min(), max=data['diffractogram']['2th'].max(), step=0.5, layout=widgets.Layout(width='95%')), + # '2th_default': {'min': data['diffractogram']['2th'].min(), 'max': data['diffractogram']['2th'].max(), 'value': [data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], 'step': 0.5}, + # '2th_cuka_default': {'min': data['diffractogram']['2th_cuka'].min(), 'max': data['diffractogram']['2th_cuka'].max(), 'value': [data['diffractogram']['2th_cuka'].min(), data['diffractogram']['2th_cuka'].max()], 'step': 0.5}, + # '2th_moka_default': {'min': data['diffractogram']['2th_moka'].min(), 'max': data['diffractogram']['2th_moka'].max(), 'value': [data['diffractogram']['2th_moka'].min(), data['diffractogram']['2th_moka'].max()], 'step': 0.5}, + # 'd_default': {'min': data['diffractogram']['d'].min(), 'max': data['diffractogram']['d'].max(), 'value': [data['diffractogram']['d'].min(), data['diffractogram']['d'].max()], 'step': 0.5}, + # '1/d_default': {'min': data['diffractogram']['1/d'].min(), 'max': data['diffractogram']['1/d'].max(), 'value': [data['diffractogram']['1/d'].min(), data['diffractogram']['1/d'].max()], 'step': 0.5}, + # 'q_default': {'min': data['diffractogram']['q'].min(), 'max': data['diffractogram']['q'].max(), 'value': [data['diffractogram']['q'].min(), data['diffractogram']['q'].max()], 'step': 0.5}, + # 'q2_default': {'min': data['diffractogram']['q2'].min(), 'max': data['diffractogram']['q2'].max(), 'value': [data['diffractogram']['q2'].min(), data['diffractogram']['q2'].max()], 'step': 0.5}, + # 'q4_default': {'min': data['diffractogram']['q4'].min(), 'max': data['diffractogram']['q4'].max(), 'value': [data['diffractogram']['q4'].min(), data['diffractogram']['q4'].max()], 'step': 0.5}, + # 'state': '2th' + # } + # } + options['widgets'] = { 'xlim': { - 'w': widgets.FloatRangeSlider(value=[data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], min=data['diffractogram']['2th'].min(), max=data['diffractogram']['2th'].max(), step=0.5, layout=widgets.Layout(width='95%')), - '2th_default': {'min': data['diffractogram']['2th'].min(), 'max': data['diffractogram']['2th'].max(), 'value': [data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], 'step': 0.5}, - '2th_cuka_default': {'min': data['diffractogram']['2th_cuka'].min(), 'max': data['diffractogram']['2th_cuka'].max(), 'value': [data['diffractogram']['2th_cuka'].min(), data['diffractogram']['2th_cuka'].max()], 'step': 0.5}, - '2th_moka_default': {'min': data['diffractogram']['2th_moka'].min(), 'max': data['diffractogram']['2th_moka'].max(), 'value': [data['diffractogram']['2th_moka'].min(), data['diffractogram']['2th_moka'].max()], 'step': 0.5}, - 'd_default': {'min': data['diffractogram']['d'].min(), 'max': data['diffractogram']['d'].max(), 'value': [data['diffractogram']['d'].min(), data['diffractogram']['d'].max()], 'step': 0.5}, - '1/d_default': {'min': data['diffractogram']['1/d'].min(), 'max': data['diffractogram']['1/d'].max(), 'value': [data['diffractogram']['1/d'].min(), data['diffractogram']['1/d'].max()], 'step': 0.5}, - 'q_default': {'min': data['diffractogram']['q'].min(), 'max': data['diffractogram']['q'].max(), 'value': [data['diffractogram']['q'].min(), data['diffractogram']['q'].max()], 'step': 0.5}, - 'q2_default': {'min': data['diffractogram']['q2'].min(), 'max': data['diffractogram']['q2'].max(), 'value': [data['diffractogram']['q2'].min(), data['diffractogram']['q2'].max()], 'step': 0.5}, - 'q4_default': {'min': data['diffractogram']['q4'].min(), 'max': data['diffractogram']['q4'].max(), 'value': [data['diffractogram']['q4'].min(), data['diffractogram']['q4'].max()], 'step': 0.5}, + 'w': widgets.FloatRangeSlider(value=[minmax['2th'][0], minmax['2th'][1]], min=minmax['2th'][0], max=minmax['2th'][1], step=0.5, layout=widgets.Layout(width='95%')), + '2th_default': {'min': minmax['2th'][0], 'max': minmax['2th'][1], 'value': [minmax['2th'][0], minmax['2th'][1]], 'step': 0.5}, + '2th_cuka_default': {'min': minmax['2th_cuka'][0], 'max': minmax['2th_cuka'][1], 'value': [minmax['2th_cuka'][0], minmax['2th_cuka'][1]], 'step': 0.5}, + '2th_moka_default': {'min': minmax['2th_moka'][0], 'max': minmax['2th_moka'][1], 'value': [minmax['2th_moka'][0], minmax['2th_moka'][1]], 'step': 0.5}, + 'd_default': {'min': minmax['d'][0], 'max': minmax['d'][1], 'value': [minmax['d'][0], minmax['d'][1]], 'step': 0.5}, + '1/d_default': {'min': minmax['1/d'][0], 'max': minmax['1/d'][1], 'value': [minmax['1/d'][0], minmax['1/d'][1]], 'step': 0.5}, + 'q_default': {'min': minmax['q'][0], 'max': minmax['q'][1], 'value': [minmax['q'][0], minmax['q'][1]], 'step': 0.5}, + 'q2_default': {'min': minmax['q2'][0], 'max': minmax['q2'][1], 'value': [minmax['q2'][0], minmax['q2'][1]], 'step': 0.5}, + 'q4_default': {'min': minmax['q4'][0], 'max': minmax['q4'][1], 'value': [minmax['q4'][0], minmax['q4'][1]], 'step': 0.5}, 'state': '2th' } } From f26153757a9527449d6e082d72bcd25141328af4 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 20:03:23 +0100 Subject: [PATCH 083/355] testing this branching stuff --- beamtime/test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..30a1328 --- /dev/null +++ b/beamtime/test.txt @@ -0,0 +1 @@ +something something test From 05c06351c0d6ceafadfccd0bef6133ab53671adf Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 20:06:26 +0100 Subject: [PATCH 084/355] test2.txt --- beamtime/test2.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 beamtime/test2.txt diff --git a/beamtime/test2.txt b/beamtime/test2.txt new file mode 100644 index 0000000..beb7fce --- /dev/null +++ b/beamtime/test2.txt @@ -0,0 +1 @@ +something more test From 0631399d7740ee9a0fdb8b634bded9bd3347f588 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 20:07:43 +0100 Subject: [PATCH 085/355] removing test-file --- beamtime/test2.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 beamtime/test2.txt diff --git a/beamtime/test2.txt b/beamtime/test2.txt deleted file mode 100644 index beb7fce..0000000 --- a/beamtime/test2.txt +++ /dev/null @@ -1 +0,0 @@ -something more test From b2ee68859a93b41a3d0b85f1bc203896ccadaeb2 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 20:08:11 +0100 Subject: [PATCH 086/355] removing test-file --- beamtime/test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt deleted file mode 100644 index 30a1328..0000000 --- a/beamtime/test.txt +++ /dev/null @@ -1 +0,0 @@ -something something test From 86a2ae2379dfcc71e5761a215b4ba2179a7b96e6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 20:25:53 +0100 Subject: [PATCH 087/355] Fix bug to iterate correctly through filenames --- beamtime/xrd/io.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 3e29b80..16991a2 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -40,10 +40,11 @@ def integrate_1d(data, options={}, index=0): df: DataFrame contianing 1D diffractogram if option 'return' is True ''' - required_options = ['unit', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite'] + required_options = ['unit', 'nbins', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite'] default_options = { 'unit': '2th_deg', + 'nbins': 3000, 'save': False, 'save_filename': None, 'save_extension': '_integrated.xy', @@ -64,17 +65,17 @@ def integrate_1d(data, options={}, index=0): ai = pyFAI.load(data['calibrant']) # Determine filename - filename = make_filename(data=data, options=options) + filename = make_filename(options=options, path=data['path'][index]) # Make save_folder if this does not exist already if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) - res = ai.integrate1d(data['image'], data['nbins'], unit=options['unit'], filename=filename) + res = ai.integrate1d(data['image'], options['nbins'], unit=options['unit'], filename=filename) data['path'][index] = filename - diffractogram, wavelength = read_xy(data=data, options=options) + diffractogram, wavelength = read_xy(data=data, options=options, index=index) if not options['save']: os.remove(filename) @@ -98,7 +99,7 @@ def make_filename(options, path=None): # If a path is given instead of an image array, the path is taken as the trunk of the savename if path: # Make filename by joining the save_folder, the filename (with extension deleted) and adding the save_extension - filename = os.path.join(options['save_folder'], os.path.split(data['path'])[-1].split('.')[0] + options['save_extension']) + filename = os.path.join(options['save_folder'], os.path.split(path)[-1].split('.')[0] + options['save_extension']) else: # Make filename just "integrated.dat" in the save_folder filename = os.path.join(options['save_folder'], 'integrated.xy') @@ -403,6 +404,8 @@ def translate_wavelengths(data, wavelength, to_wavelength=None): def find_wavelength_from_xy(path): + print(path) + wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} From d31adb9585e95400191d79ca0b2fa5ef441fe969 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 22 Mar 2022 20:26:38 +0100 Subject: [PATCH 088/355] Add feature list --- beamtime/feature_list.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 beamtime/feature_list.txt diff --git a/beamtime/feature_list.txt b/beamtime/feature_list.txt new file mode 100644 index 0000000..f4a0efd --- /dev/null +++ b/beamtime/feature_list.txt @@ -0,0 +1,2 @@ +- Must allow for automatic normalisation between different diffractograms, must only happen upon reading data +- From 06753ab6b2d0f685e21929f63dbb762b2008690a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 23 Mar 2022 14:20:19 +0100 Subject: [PATCH 089/355] Move minmax-determiniation to own function --- beamtime/xrd/plot.py | 61 +++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index cec2d1e..173da03 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -166,43 +166,12 @@ def plot_diffractogram_interactive(data, options): minmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None]} - for index, diffractogram in enumerate(data['diffractogram']): - if not minmax['2th'][0] or diffractogram['2th'].min() < minmax['2th'][0]: - minmax['2th'][0] = diffractogram['2th'].min() - min_index = index - - if not minmax['2th'][1] or diffractogram['2th'].max() > minmax['2th'][1]: - minmax['2th'][1] = diffractogram['2th'].max() - max_index = index - - minmax['2th_cuka'][0], minmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() - minmax['2th_moka'][0], minmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() - minmax['d'][0], minmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() - minmax['1/d'][0], minmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() - minmax['q'][0], minmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() - minmax['q2'][0], minmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() - minmax['q4'][0], minmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() - - - - # options['widgets'] = { - # 'xlim': { - # 'w': widgets.FloatRangeSlider(value=[data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], min=data['diffractogram']['2th'].min(), max=data['diffractogram']['2th'].max(), step=0.5, layout=widgets.Layout(width='95%')), - # '2th_default': {'min': data['diffractogram']['2th'].min(), 'max': data['diffractogram']['2th'].max(), 'value': [data['diffractogram']['2th'].min(), data['diffractogram']['2th'].max()], 'step': 0.5}, - # '2th_cuka_default': {'min': data['diffractogram']['2th_cuka'].min(), 'max': data['diffractogram']['2th_cuka'].max(), 'value': [data['diffractogram']['2th_cuka'].min(), data['diffractogram']['2th_cuka'].max()], 'step': 0.5}, - # '2th_moka_default': {'min': data['diffractogram']['2th_moka'].min(), 'max': data['diffractogram']['2th_moka'].max(), 'value': [data['diffractogram']['2th_moka'].min(), data['diffractogram']['2th_moka'].max()], 'step': 0.5}, - # 'd_default': {'min': data['diffractogram']['d'].min(), 'max': data['diffractogram']['d'].max(), 'value': [data['diffractogram']['d'].min(), data['diffractogram']['d'].max()], 'step': 0.5}, - # '1/d_default': {'min': data['diffractogram']['1/d'].min(), 'max': data['diffractogram']['1/d'].max(), 'value': [data['diffractogram']['1/d'].min(), data['diffractogram']['1/d'].max()], 'step': 0.5}, - # 'q_default': {'min': data['diffractogram']['q'].min(), 'max': data['diffractogram']['q'].max(), 'value': [data['diffractogram']['q'].min(), data['diffractogram']['q'].max()], 'step': 0.5}, - # 'q2_default': {'min': data['diffractogram']['q2'].min(), 'max': data['diffractogram']['q2'].max(), 'value': [data['diffractogram']['q2'].min(), data['diffractogram']['q2'].max()], 'step': 0.5}, - # 'q4_default': {'min': data['diffractogram']['q4'].min(), 'max': data['diffractogram']['q4'].max(), 'value': [data['diffractogram']['q4'].min(), data['diffractogram']['q4'].max()], 'step': 0.5}, - # 'state': '2th' - # } - # } + update_minmax(minmax, data) options['widgets'] = { 'xlim': { 'w': widgets.FloatRangeSlider(value=[minmax['2th'][0], minmax['2th'][1]], min=minmax['2th'][0], max=minmax['2th'][1], step=0.5, layout=widgets.Layout(width='95%')), + 'state': '2th', '2th_default': {'min': minmax['2th'][0], 'max': minmax['2th'][1], 'value': [minmax['2th'][0], minmax['2th'][1]], 'step': 0.5}, '2th_cuka_default': {'min': minmax['2th_cuka'][0], 'max': minmax['2th_cuka'][1], 'value': [minmax['2th_cuka'][0], minmax['2th_cuka'][1]], 'step': 0.5}, '2th_moka_default': {'min': minmax['2th_moka'][0], 'max': minmax['2th_moka'][1], 'value': [minmax['2th_moka'][0], minmax['2th_moka'][1]], 'step': 0.5}, @@ -210,8 +179,7 @@ def plot_diffractogram_interactive(data, options): '1/d_default': {'min': minmax['1/d'][0], 'max': minmax['1/d'][1], 'value': [minmax['1/d'][0], minmax['1/d'][1]], 'step': 0.5}, 'q_default': {'min': minmax['q'][0], 'max': minmax['q'][1], 'value': [minmax['q'][0], minmax['q'][1]], 'step': 0.5}, 'q2_default': {'min': minmax['q2'][0], 'max': minmax['q2'][1], 'value': [minmax['q2'][0], minmax['q2'][1]], 'step': 0.5}, - 'q4_default': {'min': minmax['q4'][0], 'max': minmax['q4'][1], 'value': [minmax['q4'][0], minmax['q4'][1]], 'step': 0.5}, - 'state': '2th' + 'q4_default': {'min': minmax['q4'][0], 'max': minmax['q4'][1], 'value': [minmax['q4'][0], minmax['q4'][1]], 'step': 0.5} } } @@ -235,6 +203,29 @@ def plot_diffractogram_interactive(data, options): display(w) +def update_minmax(minmax, data): + ''' Finds minimum and maximum values of each column and updates the minmax dictionary to contain the correct values. + + Input: + minmax (dict): contains ''' + + for index, diffractogram in enumerate(data['diffractogram']): + if not minmax['2th'][0] or diffractogram['2th'].min() < minmax['2th'][0]: + minmax['2th'][0] = diffractogram['2th'].min() + min_index = index + + if not minmax['2th'][1] or diffractogram['2th'].max() > minmax['2th'][1]: + minmax['2th'][1] = diffractogram['2th'].max() + max_index = index + + minmax['2th_cuka'][0], minmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() + minmax['2th_moka'][0], minmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() + minmax['d'][0], minmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() # swapped, intended + minmax['1/d'][0], minmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() + minmax['q'][0], minmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() + minmax['q2'][0], minmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() + minmax['q4'][0], minmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() + def update_widgets(options): for widget in options['widgets'].values(): From 0fb8883d198ac6d57ac9a6dde3f2a6ed32084731 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 23 Mar 2022 14:26:43 +0100 Subject: [PATCH 090/355] Add normalisation of diffractograms (default: True= --- beamtime/xrd/io.py | 4 ++++ beamtime/xrd/plot.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 16991a2..fc05654 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -300,6 +300,10 @@ def read_data(data, options={}, index=0): diffractogram, wavelength = read_xy(data=data, options=options, index=index) + if options['normalise']: + diffractogram['I'] = diffractogram['I'] / diffractogram['I'].max() + + diffractogram = translate_wavelengths(data=diffractogram, wavelength=wavelength) return diffractogram, wavelength diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 173da03..d196f2c 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -19,7 +19,7 @@ def plot_diffractogram(data, options={}): data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options - required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', + required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] default_options = { @@ -28,6 +28,7 @@ def plot_diffractogram(data, options={}): 'ylabel': 'Intensity', 'xlabel': '2theta', 'xunit': 'deg', 'yunit': 'a.u.', 'xlim': None, 'ylim': None, + 'normalise': True, 'line': True, # whether or not to plot diffractogram as a line plot 'scatter': False, # whether or not to plot individual data points 'reflections_plot': False, # whether to plot reflections as a plot @@ -55,9 +56,12 @@ def plot_diffractogram(data, options={}): for index in range(len(data['path'])): diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) + data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength + + else: if not isinstance(data['diffractogram'], list): data['diffractogram'] = [data['diffractogram']] From 2424d89156adc085a4fb5c2a82ee632469ccbedf Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 23 Mar 2022 15:31:47 +0100 Subject: [PATCH 091/355] Add offset of x- and y-values for stacked diffractograms --- beamtime/xrd/io.py | 50 +++++++++++++++++++++++++++++++++++++------- beamtime/xrd/plot.py | 14 ++++++++----- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index fc05654..263eaae 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -45,6 +45,7 @@ def integrate_1d(data, options={}, index=0): default_options = { 'unit': '2th_deg', 'nbins': 3000, + 'extract_folder': 'tmp', 'save': False, 'save_filename': None, 'save_extension': '_integrated.xy', @@ -68,8 +69,8 @@ def integrate_1d(data, options={}, index=0): filename = make_filename(options=options, path=data['path'][index]) # Make save_folder if this does not exist already - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) + if not os.path.isdir(options['extract_folder']): + os.makedirs(options['extract_folder']) res = ai.integrate1d(data['image'], options['nbins'], unit=options['unit'], filename=filename) @@ -79,7 +80,11 @@ def integrate_1d(data, options={}, index=0): if not options['save']: os.remove(filename) - shutil.rmtree('tmp') + shutil.rmtree(f'tmp') + + + # Reset this option + options['save_folder'] = None return diffractogram, wavelength @@ -89,8 +94,7 @@ def make_filename(options, path=None): # Define save location for integrated diffractogram data if not options['save']: - options['save_folder'] = 'tmp' - filename = os.path.join(options['save_folder'], 'tmp_diff.dat') + filename = os.path.join(options['extract_folder'], 'tmp_diff.dat') elif options['save']: @@ -183,9 +187,12 @@ def view_integrator(calibrant): def read_brml(data, options={}, index=0): + # FIXME: Can't read RECX1-data, apparently must be formatted differently from RECX2. Check the RawData-files and compare between the two files. + + required_options = ['extract_folder', 'save_folder'] default_options = { - 'extract_folder': 'tmp/', + 'extract_folder': 'tmp', 'save_folder': None } @@ -304,12 +311,41 @@ def read_data(data, options={}, index=0): diffractogram['I'] = diffractogram['I'] / diffractogram['I'].max() + if options['offset']: + diffractogram = apply_offset(diffractogram, wavelength, index, options) + + diffractogram = translate_wavelengths(data=diffractogram, wavelength=wavelength) return diffractogram, wavelength - +def apply_offset(diffractogram, wavelength, index, options): + #Apply offset along y-axis + diffractogram['I_org'] = diffractogram['I'] # make copy of original intensities + diffractogram['I'] = diffractogram['I'] + index*options['offset_y'] + + # Apply offset along x-axis + relative_shift = (wavelength / 1.54059)*options['offset_x'] # Adjusts the offset-factor to account for wavelength, so that offset_x given is given in 2th_cuka-units + diffractogram['2th_org'] = diffractogram['2th'] + diffractogram['2th'] = diffractogram['2th'] + index*relative_shift + + + return diffractogram + +def revert_offset(diffractogram,which=None): + + if which == 'both': + diffractogram['2th'] = diffractogram['2th_org'] + diffractogram['I'] = diffractogram['I_org'] + + if which == 'y': + diffractogram['I'] = diffractogram['I_org'] + + if which == 'x': + diffractogram['2th'] = diffractogram['2th_org'] + + return diffractogram def load_reflection_table(data, options={}): diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index d196f2c..17ad70b 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -19,7 +19,7 @@ def plot_diffractogram(data, options={}): data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options - required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', + required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] default_options = { @@ -29,6 +29,9 @@ def plot_diffractogram(data, options={}): 'xunit': 'deg', 'yunit': 'a.u.', 'xlim': None, 'ylim': None, 'normalise': True, + 'offset': True, + 'offset_x': 0, + 'offset_y': 1, 'line': True, # whether or not to plot diffractogram as a line plot 'scatter': False, # whether or not to plot individual data points 'reflections_plot': False, # whether to plot reflections as a plot @@ -49,6 +52,8 @@ def plot_diffractogram(data, options={}): data['path'] = [data['path']] + + # Check if there is some data stored already, load in data if not. This speeds up replotting in interactive mode. if not 'diffractogram' in data.keys(): # Initialise empty list for diffractograms and wavelengths data['diffractogram'] = [None for _ in range(len(data['path']))] @@ -72,7 +77,7 @@ def plot_diffractogram(data, options={}): options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] - # Start inteactive session with ipywidgets + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session if options['interactive']: options['interactive'] = False options['interactive_session_active'] = True @@ -91,7 +96,6 @@ def plot_diffractogram(data, options={}): # Prepare plot, and read and process data - fig, ax = btp.prepare_plot(options=options) @@ -125,7 +129,7 @@ def plot_diffractogram(data, options={}): - # Make the reflection plots + # Make the reflection plots. By default, the wavelength of the first diffractogram will be used for these. if options['reflections_plot'] and options['reflections_data']: options['xlim'] = ax.get_xlim() options['to_wavelength'] = data['wavelength'][0] @@ -133,7 +137,7 @@ def plot_diffractogram(data, options={}): for reference, axis in zip(options['reflections_data'], ref_axes): plot_reflection_table(data=reference, ax=axis, options=options) - # Print the reflection indices + # Print the reflection indices. By default, the wavelength of the first diffractogram will be used for this. if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() options['to_wavelength'] = data['wavelength'][0] From 606bfc180d13e5a5df3436b3dc08c5368ac6b286 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 23 Mar 2022 16:03:34 +0100 Subject: [PATCH 092/355] Add first stab at ylim widget, should be improved --- beamtime/xrd/io.py | 6 +++--- beamtime/xrd/plot.py | 21 ++++++++++++++++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 263eaae..612cc16 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -388,6 +388,8 @@ def load_reflection_table(data, options={}): def translate_wavelengths(data, wavelength, to_wavelength=None): + # FIXME Somewhere here there is an invalid arcsin-argument. Not sure where. + pd.options.mode.chained_assignment = None # Translate to CuKalpha @@ -424,9 +426,7 @@ def translate_wavelengths(data, wavelength, to_wavelength=None): if to_wavelength: - - - if to_wavelength > cuka: + if to_wavelength >= cuka: max_2th = 2*np.arcsin(cuka/to_wavelength) * 180/np.pi else: max_2th = data['2th_cuka'].max() diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 17ad70b..770b5a8 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -176,6 +176,24 @@ def plot_diffractogram_interactive(data, options): update_minmax(minmax, data) + ymin, ymax = None, None + for index, diffractogram in enumerate(data['diffractogram']): + if not ymin or (ymin > (diffractogram['I'].min())): #+index*options['offset_y'])): + ymin = diffractogram['I'].min()#+index*options['offset_y'] + + if not ymax or (ymax < (diffractogram['I'].max())):#+index*options['offset_y'])): + ymax = diffractogram['I'].max()#+index*options['offset_y'] + print(ymax) + + + ymin_start = ymin - 0.1*ymax + ymax_start = ymax+0.2*ymax + ymin = ymin - 5*ymax + ymax = ymax*5 + + + + options['widgets'] = { 'xlim': { 'w': widgets.FloatRangeSlider(value=[minmax['2th'][0], minmax['2th'][1]], min=minmax['2th'][0], max=minmax['2th'][1], step=0.5, layout=widgets.Layout(width='95%')), @@ -199,7 +217,8 @@ def plot_diffractogram_interactive(data, options): reflections_plot=widgets.ToggleButton(value=True), reflections_indices=widgets.ToggleButton(value=False), x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), - xlim=options['widgets']['xlim']['w']) + xlim=options['widgets']['xlim']['w'], + ylim=widgets.FloatRangeSlider(value=[ymin_start, ymax_start], min=ymin, max=ymax, step=0.5, layout=widgets.Layout(width='95%'))) else: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), From 0fbfd20a74cebea5d14daaf207ff7ba51edae2b0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 30 Mar 2022 16:11:34 +0200 Subject: [PATCH 093/355] Enable reading of CoupledTwoTheta-scans --- beamtime/xrd/io.py | 64 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 612cc16..5aa2fb1 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -224,24 +224,56 @@ def read_brml(data, options={}, index=0): for chain in root.findall('./DataRoutes/DataRoute'): - for scantype in chain.findall('ScanInformation/ScanMode'): - if scantype.text == 'StillScan': - if chain.get('Description') == 'Originally measured data.': - for scandata in chain.findall('Datum'): - scandata = scandata.text.split(',') - scandata = [float(i) for i in scandata] - twotheta, intensity = float(scandata[2]), float(scandata[3]) + # Get the scan type to be able to handle different data formats + scantype = chain.findall('ScanInformation')[0].get('VisibleName') - - else: - if chain.get('Description') == 'Originally measured data.': - for scandata in chain.findall('Datum'): + # Check if the chain is the right one to extract the data from + if chain.get('Description') == 'Originally measured data.': + + + if scantype == 'TwoTheta': + for scandata in chain.findall('Datum'): + scandata = scandata.text.split(',') + twotheta, intensity = float(scandata[2]), float(scandata[3]) + + if twotheta > 0: + diffractogram.append({'2th': twotheta, 'I': intensity}) + + elif scantype == 'Coupled TwoTheta/Theta': + for scandata in chain.findall('Datum'): + scandata = scandata.text.split(',') + twotheta, intensity = float(scandata[2]), float(scandata[4]) + + if twotheta > 0: + diffractogram.append({'2th': twotheta, 'I': intensity}) + + elif scantype == 'Still (Eiger2R_500K (1D mode))': + + start = float(chain.findall('ScanInformation/ScaleAxes/ScaleAxisInfo/Start')[0].text) + stop = float(chain.findall('ScanInformation/ScaleAxes/ScaleAxisInfo/Stop')[0].text) + increment = float(chain.findall('ScanInformation/ScaleAxes/ScaleAxisInfo/Increment')[0].text) + + + + for scandata in chain.findall('Datum'): scandata = scandata.text.split(',') - twotheta, intensity = float(scandata[2]), float(scandata[3]) - - if twotheta > 0: - diffractogram.append({'2th': twotheta, 'I': intensity}) + raw = [float(i) for i in scandata] + + intensity = [] + for r in raw: + if r > 600: + intensity.append(r) + + intensity = np.array(intensity) + + + + + twotheta = np.linspace(start, stop, len(intensity)) + + diffractogram = {'2th': twotheta, 'I': intensity} + #if 'wavelength' not in data.keys(): @@ -249,7 +281,9 @@ def read_brml(data, options={}, index=0): for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): wavelength = float(chain.attrib['Value']) + diffractogram = pd.DataFrame(diffractogram) + From 223be18c3e451b8be2a962bba126f44c9eab41db Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 30 Mar 2022 17:40:06 +0200 Subject: [PATCH 094/355] Change default behaviour with 10+ plots --- beamtime/plotting.py | 20 ++++++++++++-------- beamtime/xrd/io.py | 6 ++---- beamtime/xrd/plot.py | 13 ++++++++++--- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index dd9a481..4e058ac 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -377,16 +377,20 @@ def update_rc_params(rc_params): plt.rcParams.update({key: rc_params[key]}) -def generate_colours(palettes): +def generate_colours(palettes, kind=None): - # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. - colour_collection = [] - for palette in palettes: - mod = importlib.import_module("palettable.colorbrewer.%s" % palette[0]) - colour = getattr(mod, palette[1]).mpl_colors - colour_collection = colour_collection + colour + if kind == 'single': + colour_cycle = itertools.cycle(palettes) - colour_cycle = itertools.cycle(colour_collection) + else: + # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. + colour_collection = [] + for palette in palettes: + mod = importlib.import_module("palettable.colorbrewer.%s" % palette[0]) + colour = getattr(mod, palette[1]).mpl_colors + colour_collection = colour_collection + colour + + colour_cycle = itertools.cycle(colour_collection) return colour_cycle \ No newline at end of file diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 5aa2fb1..14c0f98 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -252,7 +252,7 @@ def read_brml(data, options={}, index=0): start = float(chain.findall('ScanInformation/ScaleAxes/ScaleAxisInfo/Start')[0].text) stop = float(chain.findall('ScanInformation/ScaleAxes/ScaleAxisInfo/Stop')[0].text) - increment = float(chain.findall('ScanInformation/ScaleAxes/ScaleAxisInfo/Increment')[0].text) + @@ -262,7 +262,7 @@ def read_brml(data, options={}, index=0): intensity = [] for r in raw: - if r > 600: + if r > 601: intensity.append(r) intensity = np.array(intensity) @@ -478,9 +478,7 @@ def translate_wavelengths(data, wavelength, to_wavelength=None): def find_wavelength_from_xy(path): - print(path) - wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} with open(path, 'r') as f: diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 770b5a8..1a9429c 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -20,7 +20,7 @@ def plot_diffractogram(data, options={}): # Update options required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', - 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params'] + 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] default_options = { 'x_vals': '2th', @@ -45,6 +45,10 @@ def plot_diffractogram(data, options={}): 'format_params': {}, } + if 'offset_y' not in options.keys(): + if len(data['path']) > 10: + default_options['offset_y'] = 0.05 + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) # Convert data['path'] to list to allow iteration over this to accommodate both single and multiple diffractograms @@ -62,6 +66,7 @@ def plot_diffractogram(data, options={}): for index in range(len(data['path'])): diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) + data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength @@ -113,7 +118,10 @@ def plot_diffractogram(data, options={}): ax = ax[-1] - colours = btp.generate_colours(options['palettes']) + if len(data['path']) < 10: + colours = btp.generate_colours(options['palettes']) + else: + colours = btp.generate_colours(['black'], kind='single') for diffractogram in data['diffractogram']: @@ -183,7 +191,6 @@ def plot_diffractogram_interactive(data, options): if not ymax or (ymax < (diffractogram['I'].max())):#+index*options['offset_y'])): ymax = diffractogram['I'].max()#+index*options['offset_y'] - print(ymax) ymin_start = ymin - 0.1*ymax From 3e4c0a9fc2bbc95b8b6525902727dd30863c30b7 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 30 Mar 2022 17:50:41 +0200 Subject: [PATCH 095/355] Add slider for offset_y (not working yet) --- beamtime/xrd/plot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 1a9429c..78c3906 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -66,7 +66,6 @@ def plot_diffractogram(data, options={}): for index in range(len(data['path'])): diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) - data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength @@ -157,6 +156,8 @@ def plot_diffractogram(data, options={}): if options['interactive_session_active']: btp.update_widgets(options=options) + xrd.io.up + return diffractogram, fig, ax @@ -225,7 +226,9 @@ def plot_diffractogram_interactive(data, options): reflections_indices=widgets.ToggleButton(value=False), x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], - ylim=widgets.FloatRangeSlider(value=[ymin_start, ymax_start], min=ymin, max=ymax, step=0.5, layout=widgets.Layout(width='95%'))) + ylim=widgets.FloatRangeSlider(value=[ymin_start, ymax_start], min=ymin, max=ymax, step=0.5, layout=widgets.Layout(width='95%')), + offset_y=widgets.FloatSlider(value=options['offset_y'], min=-5, max=5) + ) else: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), From 7c95135c33e30f91a3dadaeafef4792504951543 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 31 Mar 2022 11:04:20 +0200 Subject: [PATCH 096/355] Initial commit to XANES-module --- beamtime/xanes/io.py | 150 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index a818d86..2ed6d29 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -1,2 +1,148 @@ -#hello -#yeah \ No newline at end of file +import pandas as pd +import matplotlib.pyplot as plt +import os + + +def split_xanes_scan(root, destination=None, replace=False): + #root is the path to the beamtime-folder + #destination should be the path to the processed data + + #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw + + with open(filename, 'r') as f: + lines = f.readlines() + + datas = [] + data = [] + headers = [] + header = '' + start = False + + for line in lines: + if line[0:2] == "#L": + start = True + header = line[2:].split() + continue + + elif line[0:2] == "#C": + start = False + + if data: + datas.append(data) + data = [] + + if header: + headers.append(header) + header = '' + + + + if start == False: + continue + + else: + data.append(line.split()) + + + + + edges = {'Mn': [6.0, 6.1, 6.2, 6.3, 6.4, 6.5], 'Fe': [6.8, 6.9, 7.0, 7.1, 7.2], 'Co': [7.6, 7.7, 7.8, 7.9], 'Ni': [8.1, 8.2, 8.3, 8.4, 8.5]} + edge_count = {'Mn': 0, 'Fe': 0, 'Co': 0, 'Ni': 0} + + + for ind, data in enumerate(datas): + df = pd.DataFrame(data) + df.columns = headers[ind] + + edge_start = np.round((float(df["ZapEnergy"].min())), 1) + + for edge, energies in edges.items(): + if edge_start in energies: + edge_actual = edge + edge_count[edge] += 1 + + + + filename = filename.split('/')[-1] + count = str(edge_count[edge_actual]).zfill(4) + + + # Save + if destination: + cwd = os.getcwd() + + if not os.path.isdir(destination): + os.mkdir(destination) + + os.chdir(destination) + + df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + + os.chdir(cwd) + + else: + df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + + +#Function that "collects" all the files in a folder, only accepting .dat-files from xanes-measurements +def get_filenames(path): + + + cwd = os.getcwd() + + # Change into path provided + os.chdir(path) + + filenames = [os.path.join(path, filename) for filename in os.listdir() if os.path.isfile(filename) and filename[-4:] == '.dat'] #changed + + + + # Change directory back to where you ran the script from + os.chdir(cwd) + + return filenames + +def put_in_dataframe(path): + filenames = get_filenames(path) + + #making the column names to be used in the dataframe, making sure the first column is the ZapEnergy + column_names = ["ZapEnergy"] + + for i in range(len(filenames)): + column_names.append(filenames[i]) + + #Taking the first file in the folder and extracting ZapEnergies and intensity from that (only need the intensity from the rest) + first = pd.read_csv(filenames[0], skiprows=0) + + #Making a data frame with the correct columns, and will fill inn data afterwards + df = pd.DataFrame(columns = column_names) + #First putting in the 2theta-values + df["ZapEnergy"]=first["ZapEnergy"] + + #filling in the intensities from all files into the corresponding column in the dataframe + for i in range(len(filenames)): + df2 = pd.read_csv(filenames[i]) + df2 = df2.drop(['Mon','Det1','Det2','Det3','Det4','Det5', 'Det6','Ion1'], axis=1) #, axis=1) + df2 = df2.drop(['MonEx','Ion2','Htime','MusstEnc1','MusstEnc3','MusstEnc4', 'TwoTheta', 'ZCryo'], axis=1) + df2 = df2.drop(['ZBlower1', 'ZBlower2', 'ZSrcur'], axis=1)#, axis=19) #removing the sigma at this point + + ############## THIS PART PICKS OUT WHICH ROI IS OF INTEREST, BUT MUST BE FIXED IF LOOKING AT THREE EDGES (roi00,roi01,roi02) ##################### + if 'xmap_roi01' in df2.columns: + #Trying to pick the roi with the highest difference between maximum and minimum intensity --> biggest edge shift + if max(df2["xmap_roi00"])-min(df2["xmap_roi00"])>max(df2["xmap_roi01"])-min(df2["xmap_roi01"]): + df[filenames[i]]=df2["xmap_roi00"] #forMn + else: + df[filenames[i]]=df2["xmap_roi01"] #forNi + else: + df[filenames[i]]=df2["xmap_roi00"] + ############################################################################################### + + i=i+1 + + + #print(df) + #If I want to make a csv-file of the raw data. Decided that was not necessary: + #df.to_csv('static-Mn-edge.csv') #writing it to a csv, first row is datapoint (index), second column is 2theta, and from there the scans starts + + + return df \ No newline at end of file From f504c7dd691fa7fa360680e7dbad58e218ec866f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 31 Mar 2022 13:52:59 +0200 Subject: [PATCH 097/355] Add interactive offset_y --- beamtime/xrd/io.py | 20 +++++++++++++++----- beamtime/xrd/plot.py | 11 ++++++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 14c0f98..dc53b29 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -341,11 +341,13 @@ def read_data(data, options={}, index=0): diffractogram, wavelength = read_xy(data=data, options=options, index=index) - if options['normalise']: - diffractogram['I'] = diffractogram['I'] / diffractogram['I'].max() - if options['offset']: + if options['offset'] or options['normalise']: + # Make copy of the original intensities before any changes are made through normalisation or offset, to easily revert back if need to update. + diffractogram['I_org'] = diffractogram['I'] + diffractogram['2th_org'] = diffractogram['2th'] + diffractogram = apply_offset(diffractogram, wavelength, index, options) @@ -355,13 +357,21 @@ def read_data(data, options={}, index=0): def apply_offset(diffractogram, wavelength, index, options): + + options['current_offset_y'] = options['offset_y'] + options['current_offset_x'] = options['offset_x'] + #Apply offset along y-axis - diffractogram['I_org'] = diffractogram['I'] # make copy of original intensities + diffractogram['I'] = diffractogram['I_org'] # Reset intensities + + if options['normalise']: + diffractogram['I'] = diffractogram['I'] / diffractogram['I'].max() + diffractogram['I'] = diffractogram['I'] + index*options['offset_y'] # Apply offset along x-axis relative_shift = (wavelength / 1.54059)*options['offset_x'] # Adjusts the offset-factor to account for wavelength, so that offset_x given is given in 2th_cuka-units - diffractogram['2th_org'] = diffractogram['2th'] + diffractogram['2th'] = diffractogram['2th_org'] diffractogram['2th'] = diffractogram['2th'] + index*relative_shift diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 78c3906..d7a9a5c 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -81,6 +81,13 @@ def plot_diffractogram(data, options={}): options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] + if options['interactive_session_active']: + if options['offset']: + if (options['offset_x'] != options['current_offset_x']) or (options['offset_y'] != options['current_offset_y']): + for i, (diff, wl) in enumerate(zip(data['diffractogram'], data['wavelength'])): + xrd.io.apply_offset(diff, wl, i, options) + + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session if options['interactive']: options['interactive'] = False @@ -156,7 +163,9 @@ def plot_diffractogram(data, options={}): if options['interactive_session_active']: btp.update_widgets(options=options) - xrd.io.up + + + return diffractogram, fig, ax From 6f9fefae086b6729e7b68379eb969ee10542e493 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 31 Mar 2022 14:02:04 +0200 Subject: [PATCH 098/355] Add interactive offset_x --- beamtime/xrd/plot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index d7a9a5c..012e9b8 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -236,7 +236,8 @@ def plot_diffractogram_interactive(data, options): x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], ylim=widgets.FloatRangeSlider(value=[ymin_start, ymax_start], min=ymin, max=ymax, step=0.5, layout=widgets.Layout(width='95%')), - offset_y=widgets.FloatSlider(value=options['offset_y'], min=-5, max=5) + offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01), + offset_x=widgets.BoundedFloatText(value=options['offset_x'], min=-1, max=1, step=0.01) ) else: From b0130d49b88a4faa725fff54269e1239c089d2e1 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 31 Mar 2022 17:05:32 +0200 Subject: [PATCH 099/355] Working on the calibration of the XANES-data, subtracting background and defining post-edge --- beamtime/xanes/calib.py | 192 ++++++++++++++++++++++++---------------- 1 file changed, 118 insertions(+), 74 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 2ef6c4d..3c54dd1 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -1,6 +1,8 @@ import pandas as pd import numpy as np import os +import matplotlib.pyplot as plt +import beamtime.auxillary as aux def rbkerbest(): print("ROSENBORG!<3") @@ -13,82 +15,124 @@ def rbkerbest(): ##Better to make a new function that loops through the files, and performing the split_xanes_scan on -def split_xanes_scan(root, destination=None, replace=False): - #root is the path to the beamtime-folder - #destination should be the path to the processed data +def pre_edge_subtraction(df,filenames, options={}): + + required_options = ['edge', 'print'] + default_options = { + 'edge' : 'Mn', + 'print': False + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge + if str(options['edge']) == 'Mn': + edge_start = 6.45 + if str(options['edge']) == 'Ni': + edge_start = 8.3 + + + #making a function to check the difference between values in the list and the defined start of the edge (where background regression will stop): + absolute_difference_function = lambda list_value : abs(list_value - edge_start) + + #finding the energy data point value that is closest to what I defined as the end of the background + edge_start_value = min(df["ZapEnergy"], key=absolute_difference_function) + + #Finding what the index of the edge shift end point is + start_index=df[df["ZapEnergy"]==edge_start_value].index.values[0] + + #Defining x-range for linear background fit, ending at the edge start index + df_start=df[0:start_index] - #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw + #Making a new dataframe, with only the ZapEnergies as the first column + df_background = pd.DataFrame(df["ZapEnergy"]) + + for files in filenames: + + #Fitting linear function to the pre-edge + d = np.polyfit(df_start["ZapEnergy"],df_start[files],1) + function_pre = np.poly1d(d) + + #making a list, y_pre,so the background will be applied to all ZapEnergy-values + y_pre=function_pre(df["ZapEnergy"]) + + #adding a new column in df_background with the y-values of the background + df_background.insert(1,files,y_pre) - with open(filename, 'r') as f: - lines = f.readlines() - - datas = [] - data = [] - headers = [] - header = '' - start = False + #Plotting the calculated pre-edge background with the region used for the regression - for line in lines: - if line[0:2] == "#L": - start = True - header = line[2:].split() - continue - - elif line[0:2] == "#C": - start = False - - if data: - datas.append(data) - data = [] - - if header: - headers.append(header) - header = '' - - + ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILES IS CORRUPT + #ax = df.plot(x = "ZapEnergy",y=files) + + if options['print'] == True: + #Plotting an example of the edge_start region and the fitted background that will later be subtracted + ax = df.plot(x = "ZapEnergy",y=filenames[0]) #defining x and y + plt.axvline(x = edge_start_value) + fig = plt.figure(figsize=(15,15)) + df_background.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) + + ###################### Subtracting the pre edge from xmap_roi00 ################ + #making a new dataframe to insert the background subtracted intensities + df_new = pd.DataFrame(df["ZapEnergy"]) + #inserting the pre_edge-background subtracted original xmap_roi00 data - if start == False: - continue - - else: - data.append(line.split()) - - - - - edges = {'Mn': [6.0, 6.1, 6.2, 6.3, 6.4, 6.5], 'Fe': [6.8, 6.9, 7.0, 7.1, 7.2], 'Co': [7.6, 7.7, 7.8, 7.9], 'Ni': [8.1, 8.2, 8.3, 8.4, 8.5]} - edge_count = {'Mn': 0, 'Fe': 0, 'Co': 0, 'Ni': 0} + for files in filenames: + newintensity_calc=df[files]-df_background[files] + df_new.insert(1,files,newintensity_calc) + + if options['print'] == True: + #Plotting original data (black) and background subtracted data (red) + ax = df.plot(x = "ZapEnergy",y=filenames[0], color="Black") + plt.axvline(x = edge_start_value) + fig = plt.figure(figsize=(15,15)) + df_new.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) + return df_new + +def post_edge_normalization(df,df_new,filenames, options={}): + + required_options = ['edge', 'print'] + default_options = { + 'edge' : 'Mn', + 'print': False + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge + if str(options['edge']) == 'Mn': + edge_stop = 6.565 + if str(options['edge']) == 'Ni': + edge_stop = 8.361 + + absolute_difference_function = lambda list_value : abs(list_value - edge_stop) + edge_stop_value = min(df_new["ZapEnergy"], key=absolute_difference_function) + end_index=df_new[df_new["ZapEnergy"]==edge_stop_value].index.values[0] + #Defining x-range for linear fit + df_fix=df_new + df_fix.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + df_end=df_fix[end_index:] #The region of interest for the post edge + #print(df_end) + #Fitting linear function to the pre-edge using the background corrected intensities to make the post edge fit + df_postedge = pd.DataFrame(df["ZapEnergy"]) + + function_post_list=[] + for files in filenames: + d = np.polyfit(df_end["ZapEnergy"],df_end[files],1) + function_post = np.poly1d(d) + y_post=function_post(df["ZapEnergy"]) + function_post_list.append(function_post) + df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge + + #print(filenames[0]) + #print(df_postedge) + #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line + if options['print'] == True: + ax = df_new.plot(x = "ZapEnergy",y=filenames) #defining x and y + plt.axvline(x = edge_stop_value) + fig = plt.figure(figsize=(15,15)) + df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) + #print(function_post_list) + #print(function_post) + ax = df_new.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y + df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) + plt.axvline(x = edge_stop_value) - - for ind, data in enumerate(datas): - df = pd.DataFrame(data) - df.columns = headers[ind] - - edge_start = np.round((float(df["ZapEnergy"].min())), 1) - - for edge, energies in edges.items(): - if edge_start in energies: - edge_actual = edge - edge_count[edge] += 1 - - - - filename = filename.split('/')[-1] - count = str(edge_count[edge_actual]).zfill(4) - - - # Save - if destination: - cwd = os.getcwd() - - if not os.path.isdir(destination): - os.mkdir(destination) - - os.chdir(destination) - - df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) - - os.chdir(cwd) - - else: - df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) \ No newline at end of file + \ No newline at end of file From 6e851b494b39f93b87dfde610542a942fc4bd327 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 31 Mar 2022 17:29:10 +0200 Subject: [PATCH 100/355] Add heatmap, lacks mapping between x-value and 2th --- beamtime/xrd/plot.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 012e9b8..a073dd6 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -1,3 +1,4 @@ +import seaborn as sns import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) @@ -20,7 +21,7 @@ def plot_diffractogram(data, options={}): # Update options required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', - 'reflections_plot', 'reflections_indices', 'reflections_data', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] + 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] default_options = { 'x_vals': '2th', @@ -37,6 +38,8 @@ def plot_diffractogram(data, options={}): 'reflections_plot': False, # whether to plot reflections as a plot 'reflections_indices': False, # whether to plot the reflection indices 'reflections_data': None, # Should be passed as a list of dictionaries on the form {path: rel_path, reflection_indices: number of indices, colour: [r,g,b], min_alpha: 0-1] + 'heatmap': False, + 'cmap': 'viridis', 'plot_kind': None, 'palettes': [('qualitative', 'Dark2_8')], 'interactive': False, @@ -69,13 +72,20 @@ def plot_diffractogram(data, options={}): data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength - else: if not isinstance(data['diffractogram'], list): data['diffractogram'] = [data['diffractogram']] data['wavelength'] = [data['wavelength']] + if options['heatmap']: + data['heatmap'] = [] + for diff in data['diffractogram']: + data['heatmap'].append(np.array(diff['I'])) + + data['heatmap'] = np.array(data['heatmap']) + data['heatmap'] = np.flipud(data['heatmap']) + # Sets the xlim if this has not bee specified if not options['xlim']: options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] @@ -130,12 +140,17 @@ def plot_diffractogram(data, options={}): colours = btp.generate_colours(['black'], kind='single') - for diffractogram in data['diffractogram']: - if options['line']: - diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) - - if options['scatter']: - ax.scatter(x=diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) + # FIXME Must be changed to map the x-value to the 2th-value somehow + if options['heatmap']: + sns.heatmap(data['heatmap'], cmap=options['cmap']) + + else: + for diffractogram in data['diffractogram']: + if options['line']: + diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) + + if options['scatter']: + ax.scatter(x=diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) @@ -168,7 +183,7 @@ def plot_diffractogram(data, options={}): - return diffractogram, fig, ax + return data['diffractogram'], fig, ax def determine_grid_layout(options): From 567282b80b79d86892e62f144e465f20a5d6f528 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 4 Apr 2022 14:48:29 +0200 Subject: [PATCH 101/355] Fix bug where extract_folder had no default value --- beamtime/xrd/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index dc53b29..729b674 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -40,7 +40,7 @@ def integrate_1d(data, options={}, index=0): df: DataFrame contianing 1D diffractogram if option 'return' is True ''' - required_options = ['unit', 'nbins', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite'] + required_options = ['unit', 'nbins', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite', 'extract_folder'] default_options = { 'unit': '2th_deg', From b0629de9a3dd5e32e49ad59639d1f4ef21c35bf2 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 4 Apr 2022 14:48:59 +0200 Subject: [PATCH 102/355] Add functions to round up and down to nearest dec --- beamtime/auxillary.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/beamtime/auxillary.py b/beamtime/auxillary.py index 76ec551..68785f7 100644 --- a/beamtime/auxillary.py +++ b/beamtime/auxillary.py @@ -1,4 +1,5 @@ import json +import numpy as np def update_options(options, required_options, default_options): ''' Takes a dictionary of options along with a list of required options and dictionary of default options, and sets all keyval-pairs of options that is not already defined to the default values''' @@ -37,6 +38,18 @@ def swap_values(dict, key1, key2): -def hello_world2(a=1, b=2): +def ceil(a, roundto=1): - print(f'Halla, MAFAKKAS! a = {a} og b = {b}') \ No newline at end of file + fac = 1/roundto + + a = np.ceil(a*fac) / fac + + return a + +def floor(a, roundto=1): + + fac = 1/roundto + + a = np.floor(a*fac) / fac + + return a \ No newline at end of file From 59629fcb615a798a2ef13ad47d83394dfe859d95 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 4 Apr 2022 14:50:20 +0200 Subject: [PATCH 103/355] Add plotting of heatmaps with true xlim --- beamtime/plotting.py | 2 +- beamtime/xrd/plot.py | 88 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index 4e058ac..500f532 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -182,7 +182,7 @@ def adjust_plot(fig, ax, options): ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) - # THIS NEEDS REWORK FOR IT TO FUNCTION PROPERLY! + # FIXME THIS NEEDS REWORK FOR IT TO FUNCTION PROPERLY! if options['xticks']: ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) ax.set_xticklabels(options['xticks']) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index a073dd6..d2b7b51 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -72,23 +72,20 @@ def plot_diffractogram(data, options={}): data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength + # Sets the xlim if this has not bee specified + if not options['xlim']: + options['xlim'] = [data['diffractogram'][0][options['x_vals']].min(), data['diffractogram'][0][options['x_vals']].max()] + + + if options['heatmap']: + data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'] = generate_heatmap(data=data, options=options) + options['xlim'] = options['heatmap_xlim'] else: if not isinstance(data['diffractogram'], list): data['diffractogram'] = [data['diffractogram']] data['wavelength'] = [data['wavelength']] - if options['heatmap']: - data['heatmap'] = [] - for diff in data['diffractogram']: - data['heatmap'].append(np.array(diff['I'])) - - data['heatmap'] = np.array(data['heatmap']) - data['heatmap'] = np.flipud(data['heatmap']) - - # Sets the xlim if this has not bee specified - if not options['xlim']: - options['xlim'] = [diffractogram[options['x_vals']].min(), diffractogram[options['x_vals']].max()] if options['interactive_session_active']: @@ -142,7 +139,10 @@ def plot_diffractogram(data, options={}): # FIXME Must be changed to map the x-value to the 2th-value somehow if options['heatmap']: - sns.heatmap(data['heatmap'], cmap=options['cmap']) + sns.heatmap(data['heatmap'], cmap=options['cmap'], cbar=False, ax=ax) + ax.set_xticks(data['heatmap_xticks']) + ax.set_xticklabels(data['heatmap_xticklabels']) + ax.tick_params(axis='x', which='minor', bottom=False, top=False) else: for diffractogram in data['diffractogram']: @@ -186,6 +186,68 @@ def plot_diffractogram(data, options={}): return data['diffractogram'], fig, ax + +def generate_heatmap(data, options={}): + + required_options = ['x_tick_locators'] + + default_options = { + 'x_tick_locators': [0.5, 0.1] + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + twotheta = [] + intensities = [] + scans = [] + + for i, d in enumerate(data['diffractogram']): + twotheta = np.append(twotheta, d['2th'].to_numpy()) + intensities = np.append(intensities, d['I'].to_numpy()) + scans = np.append(scans, np.full(len(d['2th'].to_numpy()), int(i))) + + + # Generate ticks and xtick-labels + twotheta_max = data['diffractogram'][0]['2th'].max() + twotheta_min = data['diffractogram'][0]['2th'].min() + twotheta_span = twotheta_max - twotheta_min + ndatapoints = len(data['diffractogram'][0]['2th']) + steps = twotheta_span / ndatapoints + + twotheta_label_max = aux.floor(twotheta_max, roundto=options['x_tick_locators'][0]) + twotheta_label_min = aux.ceil(twotheta_min, roundto=options['x_tick_locators'][0]) + label_steps = (twotheta_label_max - twotheta_label_min)/options['x_tick_locators'][0] + + + + xtick_labels = np.linspace(twotheta_label_min, twotheta_label_max, num=int(label_steps)+1) + + options['x_tick_locators'] = None + + xticks = [] + for tick in xtick_labels: + xticks.append((tick - twotheta_min)/steps) + + heatmap = pd.DataFrame({'2th': twotheta, 'scan': scans, 'I': intensities}) + heatmap = heatmap.reset_index().pivot_table(index='scan', columns='2th', values='I') + + options['heatmap_xlim'] = [(options['xlim'][0] - twotheta_min)/steps, (options['xlim'][1] - twotheta_min)/steps] + + + print(xticks, xtick_labels) + + + return heatmap, xticks, xtick_labels + + + + + + + +# #results = np.transpose(np.vstack([twotheta, scans, intensities])) + + def determine_grid_layout(options): @@ -202,6 +264,8 @@ def determine_grid_layout(options): + + def plot_diffractogram_interactive(data, options): From 5541f44a58b4cbc96ba2c55f3ce1b16fcc255bb4 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 4 Apr 2022 16:47:01 +0200 Subject: [PATCH 104/355] Add automatic change of xlim range with heatmaps --- beamtime/xrd/plot.py | 107 ++++++++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 36 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index d2b7b51..df17e8d 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -76,9 +76,9 @@ def plot_diffractogram(data, options={}): if not options['xlim']: options['xlim'] = [data['diffractogram'][0][options['x_vals']].min(), data['diffractogram'][0][options['x_vals']].max()] - - if options['heatmap']: - data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'] = generate_heatmap(data=data, options=options) + # Generate heatmap data + data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'] = generate_heatmap(data=data, options=options) + if options['heatmap']: options['xlim'] = options['heatmap_xlim'] else: @@ -136,12 +136,10 @@ def plot_diffractogram(data, options={}): else: colours = btp.generate_colours(['black'], kind='single') - - # FIXME Must be changed to map the x-value to the 2th-value somehow if options['heatmap']: sns.heatmap(data['heatmap'], cmap=options['cmap'], cbar=False, ax=ax) - ax.set_xticks(data['heatmap_xticks']) - ax.set_xticklabels(data['heatmap_xticklabels']) + ax.set_xticks(data['heatmap_xticks'][options['x_vals']]) + ax.set_xticklabels(data['heatmap_xticklabels'][options['x_vals']]) ax.tick_params(axis='x', which='minor', bottom=False, top=False) else: @@ -176,7 +174,7 @@ def plot_diffractogram(data, options={}): if options['interactive_session_active']: - btp.update_widgets(options=options) + update_widgets(options=options) @@ -207,37 +205,52 @@ def generate_heatmap(data, options={}): scans = np.append(scans, np.full(len(d['2th'].to_numpy()), int(i))) - # Generate ticks and xtick-labels - twotheta_max = data['diffractogram'][0]['2th'].max() - twotheta_min = data['diffractogram'][0]['2th'].min() - twotheta_span = twotheta_max - twotheta_min + heatmap = pd.DataFrame({'2th': twotheta, 'scan': scans, 'I': intensities}) + xrd.io.translate_wavelengths(data=heatmap, wavelength=data['wavelength'][0]) + min_dict = {'2th': heatmap['2th'].min(), '2th_cuka': heatmap['2th_cuka'].min(), '2th_moka': heatmap['2th_moka'].min(), + 'q': heatmap['q'].min(), 'q2': heatmap['q2'].min(), 'q4': heatmap['q4'].min(), '1/d': heatmap['1/d'].min()} + + max_dict = {'2th': heatmap['2th'].max(), '2th_cuka': heatmap['2th_cuka'].max(), '2th_moka': heatmap['2th_moka'].max(), + 'q': heatmap['q'].max(), 'q2': heatmap['q2'].max(), 'q4': heatmap['q4'].max(), '1/d': heatmap['1/d'].max()} + + ndatapoints = len(data['diffractogram'][0]['2th']) - steps = twotheta_span / ndatapoints - twotheta_label_max = aux.floor(twotheta_max, roundto=options['x_tick_locators'][0]) - twotheta_label_min = aux.ceil(twotheta_min, roundto=options['x_tick_locators'][0]) - label_steps = (twotheta_label_max - twotheta_label_min)/options['x_tick_locators'][0] + xlims = [0, ndatapoints] + xticks = {} + xticklabels = {} - + for xval in min_dict.keys(): + + # Add xticks labels + + label_max = aux.floor(max_dict[xval], roundto=options['x_tick_locators'][0]) + label_min = aux.ceil(min_dict[xval], roundto=options['x_tick_locators'][0]) + label_steps = (label_max - label_min)/options['x_tick_locators'][0] + + xticklabels[xval] = np.linspace(label_min, label_max, num=int(label_steps)+1) + + # Add xticks + xval_span = max_dict[xval] - min_dict[xval] + steps = xval_span / ndatapoints + + + xticks_xval = [] + + for tick in xticklabels[xval]: + xticks_xval.append((tick-min_dict[xval])/steps) + + xticks[xval] = xticks_xval - xtick_labels = np.linspace(twotheta_label_min, twotheta_label_max, num=int(label_steps)+1) options['x_tick_locators'] = None - xticks = [] - for tick in xtick_labels: - xticks.append((tick - twotheta_min)/steps) - - heatmap = pd.DataFrame({'2th': twotheta, 'scan': scans, 'I': intensities}) heatmap = heatmap.reset_index().pivot_table(index='scan', columns='2th', values='I') - options['heatmap_xlim'] = [(options['xlim'][0] - twotheta_min)/steps, (options['xlim'][1] - twotheta_min)/steps] + options['heatmap_xlim'] = xlims - print(xticks, xtick_labels) - - - return heatmap, xticks, xtick_labels + return heatmap, xticks, xticklabels @@ -269,9 +282,9 @@ def determine_grid_layout(options): def plot_diffractogram_interactive(data, options): - minmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None]} + minmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None], 'heatmap': [None, None]} - update_minmax(minmax, data) + update_minmax(minmax=minmax, data=data, options=options) ymin, ymax = None, None for index, diffractogram in enumerate(data['diffractogram']): @@ -288,8 +301,6 @@ def plot_diffractogram_interactive(data, options): ymax = ymax*5 - - options['widgets'] = { 'xlim': { 'w': widgets.FloatRangeSlider(value=[minmax['2th'][0], minmax['2th'][1]], min=minmax['2th'][0], max=minmax['2th'][1], step=0.5, layout=widgets.Layout(width='95%')), @@ -301,17 +312,18 @@ def plot_diffractogram_interactive(data, options): '1/d_default': {'min': minmax['1/d'][0], 'max': minmax['1/d'][1], 'value': [minmax['1/d'][0], minmax['1/d'][1]], 'step': 0.5}, 'q_default': {'min': minmax['q'][0], 'max': minmax['q'][1], 'value': [minmax['q'][0], minmax['q'][1]], 'step': 0.5}, 'q2_default': {'min': minmax['q2'][0], 'max': minmax['q2'][1], 'value': [minmax['q2'][0], minmax['q2'][1]], 'step': 0.5}, - 'q4_default': {'min': minmax['q4'][0], 'max': minmax['q4'][1], 'value': [minmax['q4'][0], minmax['q4'][1]], 'step': 0.5} + 'q4_default': {'min': minmax['q4'][0], 'max': minmax['q4'][1], 'value': [minmax['q4'][0], minmax['q4'][1]], 'step': 0.5}, + 'heatmap_default': {'min': minmax['heatmap'][0], 'max': minmax['heatmap'][1], 'value': [minmax['heatmap'][0], minmax['heatmap'][1]], 'step': 10} } } - if options['reflections_data']: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), reflections_plot=widgets.ToggleButton(value=True), reflections_indices=widgets.ToggleButton(value=False), + heatmap=widgets.ToggleButton(value=options['heatmap']), x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], ylim=widgets.FloatRangeSlider(value=[ymin_start, ymax_start], min=ymin, max=ymax, step=0.5, layout=widgets.Layout(width='95%')), @@ -329,7 +341,7 @@ def plot_diffractogram_interactive(data, options): display(w) -def update_minmax(minmax, data): +def update_minmax(minmax, data, options={}): ''' Finds minimum and maximum values of each column and updates the minmax dictionary to contain the correct values. Input: @@ -351,15 +363,38 @@ def update_minmax(minmax, data): minmax['q'][0], minmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() minmax['q2'][0], minmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() minmax['q4'][0], minmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() + minmax['heatmap'] = options['heatmap_xlim'] + + def update_widgets(options): for widget in options['widgets'].values(): - if widget['state'] != options['x_vals']: + if options['heatmap'] and (widget['state'] != 'heatmap'): + setattr(widget['w'], 'min', widget['heatmap_default']['min']) + setattr(widget['w'], 'max', widget['heatmap_default']['max']) + setattr(widget['w'], 'value', widget['heatmap_default']['value']) + setattr(widget['w'], 'step', widget['heatmap_default']['step']) + + widget['state'] = 'heatmap' + + elif not options['heatmap'] and (widget['state'] != options['x_vals']): for arg in widget[f'{options["x_vals"]}_default']: + + # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first + if arg == 'min': + if widget[f'{options["x_vals"]}_default']['min'] > getattr(widget['w'], 'max'): + setattr(widget['w'], 'max', widget[f'{options["x_vals"]}_default']['max']) + + elif arg == 'max': + if widget[f'{options["x_vals"]}_default']['max'] < getattr(widget['w'], 'min'): + setattr(widget['w'], 'min', widget[f'{options["x_vals"]}_default']['min']) + + setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) + widget['state'] = options['x_vals'] From c0449f2e183d17f684d0906791e9a2b9e82d52f7 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Apr 2022 13:52:27 +0200 Subject: [PATCH 105/355] Correct switch of ylim between diff and heatmap --- beamtime/xrd/plot.py | 199 +++++++++++++++++++++++++++++-------------- 1 file changed, 135 insertions(+), 64 deletions(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index df17e8d..c97dabe 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -176,9 +176,6 @@ def plot_diffractogram(data, options={}): if options['interactive_session_active']: update_widgets(options=options) - - - return data['diffractogram'], fig, ax @@ -282,38 +279,51 @@ def determine_grid_layout(options): def plot_diffractogram_interactive(data, options): - minmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None], 'heatmap': [None, None]} + xminmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None], 'heatmap': [None, None], 'start': [None, None, None, None]} + yminmax = {'diff': [None, None, None, None], 'heatmap': [None, None], 'start': [None, None, None, None]} - update_minmax(minmax=minmax, data=data, options=options) - - ymin, ymax = None, None - for index, diffractogram in enumerate(data['diffractogram']): - if not ymin or (ymin > (diffractogram['I'].min())): #+index*options['offset_y'])): - ymin = diffractogram['I'].min()#+index*options['offset_y'] - - if not ymax or (ymax < (diffractogram['I'].max())):#+index*options['offset_y'])): - ymax = diffractogram['I'].max()#+index*options['offset_y'] + update_xminmax(xminmax=xminmax, data=data, options=options) + update_yminmax(yminmax=yminmax, data=data, options=options) - ymin_start = ymin - 0.1*ymax - ymax_start = ymax+0.2*ymax - ymin = ymin - 5*ymax - ymax = ymax*5 + + # Get start values for ylim slider based on choice (FIXME This can be impleneted into update_yminmax). Can also make a 'start' item that stores the start values, instead of having 4 items in 'diff' as it is now. + if options['heatmap']: + ymin = yminmax['heatmap'][0] + ymax = yminmax['heatmap'][1] + ymin_start = yminmax['heatmap'][0] + ymax_start = yminmax['heatmap'][1] + + elif not options['heatmap']: + ymin = yminmax['diff'][0] + ymax = yminmax['diff'][1] + ymin_start = yminmax['diff'][2] + ymax_start = yminmax['diff'][3] + + + # FIXME The start values for xlim should probably also be decided by initial value of x_vals, and can likewise be implemented in update_xminmax() + options['widgets'] = { 'xlim': { - 'w': widgets.FloatRangeSlider(value=[minmax['2th'][0], minmax['2th'][1]], min=minmax['2th'][0], max=minmax['2th'][1], step=0.5, layout=widgets.Layout(width='95%')), - 'state': '2th', - '2th_default': {'min': minmax['2th'][0], 'max': minmax['2th'][1], 'value': [minmax['2th'][0], minmax['2th'][1]], 'step': 0.5}, - '2th_cuka_default': {'min': minmax['2th_cuka'][0], 'max': minmax['2th_cuka'][1], 'value': [minmax['2th_cuka'][0], minmax['2th_cuka'][1]], 'step': 0.5}, - '2th_moka_default': {'min': minmax['2th_moka'][0], 'max': minmax['2th_moka'][1], 'value': [minmax['2th_moka'][0], minmax['2th_moka'][1]], 'step': 0.5}, - 'd_default': {'min': minmax['d'][0], 'max': minmax['d'][1], 'value': [minmax['d'][0], minmax['d'][1]], 'step': 0.5}, - '1/d_default': {'min': minmax['1/d'][0], 'max': minmax['1/d'][1], 'value': [minmax['1/d'][0], minmax['1/d'][1]], 'step': 0.5}, - 'q_default': {'min': minmax['q'][0], 'max': minmax['q'][1], 'value': [minmax['q'][0], minmax['q'][1]], 'step': 0.5}, - 'q2_default': {'min': minmax['q2'][0], 'max': minmax['q2'][1], 'value': [minmax['q2'][0], minmax['q2'][1]], 'step': 0.5}, - 'q4_default': {'min': minmax['q4'][0], 'max': minmax['q4'][1], 'value': [minmax['q4'][0], minmax['q4'][1]], 'step': 0.5}, - 'heatmap_default': {'min': minmax['heatmap'][0], 'max': minmax['heatmap'][1], 'value': [minmax['heatmap'][0], minmax['heatmap'][1]], 'step': 10} + 'w': widgets.FloatRangeSlider(value=[xminmax['start'][2], xminmax['start'][3]], min=xminmax['start'][0], max=xminmax['start'][1], step=0.5, layout=widgets.Layout(width='95%')), + 'state': options['x_vals'], + '2th_default': {'min': xminmax['2th'][0], 'max': xminmax['2th'][1], 'value': [xminmax['2th'][0], xminmax['2th'][1]], 'step': 0.5}, + '2th_cuka_default': {'min': xminmax['2th_cuka'][0], 'max': xminmax['2th_cuka'][1], 'value': [xminmax['2th_cuka'][0], xminmax['2th_cuka'][1]], 'step': 0.5}, + '2th_moka_default': {'min': xminmax['2th_moka'][0], 'max': xminmax['2th_moka'][1], 'value': [xminmax['2th_moka'][0], xminmax['2th_moka'][1]], 'step': 0.5}, + 'd_default': {'min': xminmax['d'][0], 'max': xminmax['d'][1], 'value': [xminmax['d'][0], xminmax['d'][1]], 'step': 0.5}, + '1/d_default': {'min': xminmax['1/d'][0], 'max': xminmax['1/d'][1], 'value': [xminmax['1/d'][0], xminmax['1/d'][1]], 'step': 0.5}, + 'q_default': {'min': xminmax['q'][0], 'max': xminmax['q'][1], 'value': [xminmax['q'][0], xminmax['q'][1]], 'step': 0.5}, + 'q2_default': {'min': xminmax['q2'][0], 'max': xminmax['q2'][1], 'value': [xminmax['q2'][0], xminmax['q2'][1]], 'step': 0.5}, + 'q4_default': {'min': xminmax['q4'][0], 'max': xminmax['q4'][1], 'value': [xminmax['q4'][0], xminmax['q4'][1]], 'step': 0.5}, + 'heatmap_default': {'min': xminmax['heatmap'][0], 'max': xminmax['heatmap'][1], 'value': [xminmax['heatmap'][0], xminmax['heatmap'][1]], 'step': 10} + }, + 'ylim': { + 'w': widgets.FloatRangeSlider(value=[yminmax['start'][2], yminmax['start'][3]], min=yminmax['start'][0], max=yminmax['start'][1], step=0.5, layout=widgets.Layout(width='95%')), + 'state': 'heatmap' if options['heatmap'] else 'diff', + 'diff_default': {'min': yminmax['diff'][0], 'max': yminmax['diff'][1], 'value': [yminmax['diff'][2], yminmax['diff'][3]], 'step': 0.1}, + 'heatmap_default': {'min': yminmax['heatmap'][0], 'max': yminmax['heatmap'][1], 'value': [yminmax['heatmap'][0], yminmax['heatmap'][1]], 'step': 0.1} } } @@ -326,7 +336,7 @@ def plot_diffractogram_interactive(data, options): heatmap=widgets.ToggleButton(value=options['heatmap']), x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], - ylim=widgets.FloatRangeSlider(value=[ymin_start, ymax_start], min=ymin, max=ymax, step=0.5, layout=widgets.Layout(width='95%')), + ylim=options['widgets']['ylim']['w'], offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01), offset_x=widgets.BoundedFloatText(value=options['offset_x'], min=-1, max=1, step=0.01) ) @@ -341,61 +351,122 @@ def plot_diffractogram_interactive(data, options): display(w) -def update_minmax(minmax, data, options={}): +def update_xminmax(xminmax, data, options={}): ''' Finds minimum and maximum values of each column and updates the minmax dictionary to contain the correct values. Input: minmax (dict): contains ''' for index, diffractogram in enumerate(data['diffractogram']): - if not minmax['2th'][0] or diffractogram['2th'].min() < minmax['2th'][0]: - minmax['2th'][0] = diffractogram['2th'].min() + if not xminmax['2th'][0] or diffractogram['2th'].min() < xminmax['2th'][0]: + xminmax['2th'][0] = diffractogram['2th'].min() min_index = index - if not minmax['2th'][1] or diffractogram['2th'].max() > minmax['2th'][1]: - minmax['2th'][1] = diffractogram['2th'].max() + if not xminmax['2th'][1] or diffractogram['2th'].max() > xminmax['2th'][1]: + xminmax['2th'][1] = diffractogram['2th'].max() max_index = index - minmax['2th_cuka'][0], minmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() - minmax['2th_moka'][0], minmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() - minmax['d'][0], minmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() # swapped, intended - minmax['1/d'][0], minmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() - minmax['q'][0], minmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() - minmax['q2'][0], minmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() - minmax['q4'][0], minmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() - minmax['heatmap'] = options['heatmap_xlim'] + xminmax['2th_cuka'][0], xminmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() + xminmax['2th_moka'][0], xminmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() + xminmax['d'][0], xminmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() # swapped, intended + xminmax['1/d'][0], xminmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() + xminmax['q'][0], xminmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() + xminmax['q2'][0], xminmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() + xminmax['q4'][0], xminmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() + xminmax['heatmap'] = options['heatmap_xlim'] + + + xminmax['start'][0], xminmax['start'][1] = xminmax[options['x_vals']][0], xminmax[options['x_vals']][1] + xminmax['start'][2], xminmax['start'][3] = xminmax[options['x_vals']][0], xminmax[options['x_vals']][1] + + +def update_yminmax(yminmax: dict, data, options={}): + + for index, diffractogram in enumerate(data['diffractogram']): + if not yminmax['diff'][0] or (yminmax['diff'][0] > (diffractogram['I'].min())): + yminmax['diff'][0] = diffractogram['I'].min() + + if not yminmax['diff'][1] or (yminmax['diff'][1] < (diffractogram['I'].max())): + yminmax['diff'][1] = diffractogram['I'].max() + + + # Set start values of ymin and ymax to be slightly below lowest data points and slightly above highest data points to give some whitespace around the plot + yminmax['diff'][2] = yminmax['diff'][0] - 0.1*yminmax['diff'][1] + yminmax['diff'][3] = yminmax['diff'][1] + 0.2*yminmax['diff'][1] + + # Allow for adjustment up to five times ymax above and below data + yminmax['diff'][0] = yminmax['diff'][0] - 5*yminmax['diff'][1] + yminmax['diff'][1] = yminmax['diff'][1]*5 + + + yminmax['heatmap'][0] = 0 + yminmax['heatmap'][1] = data['heatmap'].shape[0] + + + if options['heatmap']: + yminmax['start'][0], yminmax['start'][1] = yminmax['heatmap'][0], yminmax['heatmap'][1] + yminmax['start'][2], yminmax['start'][3] = yminmax['heatmap'][0], yminmax['heatmap'][1] + + else: + # The third and fourth index are different here to not be zoomed completely out to begin with. + yminmax['start'][0], yminmax['start'][1] = yminmax['diff'][0], yminmax['diff'][1] + yminmax['start'][2], yminmax['start'][3] = yminmax['diff'][2], yminmax['diff'][3] + def update_widgets(options): - for widget in options['widgets'].values(): + for widget, attr in options['widgets'].items(): - if options['heatmap'] and (widget['state'] != 'heatmap'): - setattr(widget['w'], 'min', widget['heatmap_default']['min']) - setattr(widget['w'], 'max', widget['heatmap_default']['max']) - setattr(widget['w'], 'value', widget['heatmap_default']['value']) - setattr(widget['w'], 'step', widget['heatmap_default']['step']) + if widget == 'xlim': + + if options['heatmap'] and (attr['state'] != 'heatmap'): + setattr(attr['w'], 'min', attr['heatmap_default']['min']) + setattr(attr['w'], 'max', attr['heatmap_default']['max']) + setattr(attr['w'], 'value', attr['heatmap_default']['value']) + setattr(attr['w'], 'step', attr['heatmap_default']['step']) + + attr['state'] = 'heatmap' + + elif not options['heatmap'] and (attr['state'] != options['x_vals']): + for arg in attr[f'{options["x_vals"]}_default']: + + # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first + if arg == 'min': + if attr[f'{options["x_vals"]}_default']['min'] > getattr(attr['w'], 'max'): + setattr(attr['w'], 'max', attr[f'{options["x_vals"]}_default']['max']) + + elif arg == 'max': + if attr[f'{options["x_vals"]}_default']['max'] < getattr(attr['w'], 'min'): + setattr(attr['w'], 'min', attr[f'{options["x_vals"]}_default']['min']) - widget['state'] = 'heatmap' - - elif not options['heatmap'] and (widget['state'] != options['x_vals']): - for arg in widget[f'{options["x_vals"]}_default']: - # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first - if arg == 'min': - if widget[f'{options["x_vals"]}_default']['min'] > getattr(widget['w'], 'max'): - setattr(widget['w'], 'max', widget[f'{options["x_vals"]}_default']['max']) + setattr(attr['w'], arg, attr[f'{options["x_vals"]}_default'][arg]) - elif arg == 'max': - if widget[f'{options["x_vals"]}_default']['max'] < getattr(widget['w'], 'min'): - setattr(widget['w'], 'min', widget[f'{options["x_vals"]}_default']['min']) + + attr['state'] = options['x_vals'] + + elif widget == 'ylim': + state = 'heatmap' if options['heatmap'] else 'diff' + + if attr['state'] != state: + + for arg in attr[f'{state}_default']: + # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first + if arg == 'min': + if attr[f'{state}_default']['min'] > getattr(attr['w'], 'max'): + setattr(attr['w'], 'max', attr[f'{state}_default']['max']) + + elif arg == 'max': + if attr[f'{state}_default']['max'] < getattr(attr['w'], 'min'): + setattr(attr['w'], 'min', attr[f'{state}_default']['min']) + + + setattr(attr['w'], arg, attr[f'{state}_default'][arg]) + + attr['state'] = state - - setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) - - - widget['state'] = options['x_vals'] From 876e0f8d3d2759a762dbaf58c317e41fbf50c4bc Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Apr 2022 16:12:19 +0200 Subject: [PATCH 106/355] Allow rescaling ylim with interactive diff plot --- beamtime/xrd/io.py | 11 ++- beamtime/xrd/plot.py | 161 ++++++++++++++++++++++++++++++------------- 2 files changed, 122 insertions(+), 50 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 729b674..910811b 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -358,9 +358,18 @@ def read_data(data, options={}, index=0): def apply_offset(diffractogram, wavelength, index, options): - options['current_offset_y'] = options['offset_y'] + if 'current_offset_y' not in options.keys(): + options['current_offset_y'] = options['offset_y'] + else: + if options['current_offset_y'] != options['offset_y']: + options['offset_change'] = True + + options['current_offset_y'] = options['offset_y'] + options['current_offset_x'] = options['offset_x'] + + #Apply offset along y-axis diffractogram['I'] = diffractogram['I_org'] # Reset intensities diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index c97dabe..9012b4a 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -20,7 +20,7 @@ def plot_diffractogram(data, options={}): data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' # Update options - required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', + required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', 'offset_change', 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] default_options = { @@ -33,6 +33,7 @@ def plot_diffractogram(data, options={}): 'offset': True, 'offset_x': 0, 'offset_y': 1, + 'offset_change': False, 'line': True, # whether or not to plot diffractogram as a line plot 'scatter': False, # whether or not to plot individual data points 'reflections_plot': False, # whether to plot reflections as a plot @@ -53,6 +54,7 @@ def plot_diffractogram(data, options={}): default_options['offset_y'] = 0.05 options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options['current_offset_y'] = options['offset_y'] # Convert data['path'] to list to allow iteration over this to accommodate both single and multiple diffractograms if not isinstance(data['path'], list): @@ -174,7 +176,8 @@ def plot_diffractogram(data, options={}): if options['interactive_session_active']: - update_widgets(options=options) + options['current_y_offset'] = options['widget'].kwargs['offset_y'] + update_widgets(data=data, options=options) @@ -213,7 +216,7 @@ def generate_heatmap(data, options={}): ndatapoints = len(data['diffractogram'][0]['2th']) - xlims = [0, ndatapoints] + xlims = [0, ndatapoints, 0, ndatapoints] # 0: xmin, 1: xmax, 2: xmin_start, 3: xmax_start xticks = {} xticklabels = {} @@ -279,13 +282,18 @@ def determine_grid_layout(options): def plot_diffractogram_interactive(data, options): - xminmax = {'2th': [None, None], '2th_cuka': [None, None], '2th_moka': [None, None], 'd': [None, None], '1/d': [None, None], 'q': [None, None], 'q2': [None, None], 'q4': [None, None], 'heatmap': [None, None], 'start': [None, None, None, None]} - yminmax = {'diff': [None, None, None, None], 'heatmap': [None, None], 'start': [None, None, None, None]} + # Format here is xminmax[0]: xmin, xminmax[1]: xmax, xminmax[2]: xmin_start, xminmax[3]: xmax_start, where "_start" denotes starting value of the slider + xminmax = { '2th': [None, None, None, None], '2th_cuka': [None, None, None, None], '2th_moka': [None, None, None, None], + 'd': [None, None, None, None], '1/d': [None, None, None, None], + 'q': [None, None, None, None], 'q2': [None, None, None, None], 'q4': [None, None, None, None], + 'heatmap': [None, None, None, None], 'start': [None, None, None, None]} + + yminmax = { 'diff': [None, None, None, None], 'heatmap': [None, None, None, None], 'start': [None, None, None, None]} update_xminmax(xminmax=xminmax, data=data, options=options) update_yminmax(yminmax=yminmax, data=data, options=options) - + options['xminmax'], options['yminmax'] = xminmax, yminmax # Get start values for ylim slider based on choice (FIXME This can be impleneted into update_yminmax). Can also make a 'start' item that stores the start values, instead of having 4 items in 'diff' as it is now. if options['heatmap']: @@ -337,8 +345,8 @@ def plot_diffractogram_interactive(data, options): x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], ylim=options['widgets']['ylim']['w'], - offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01), - offset_x=widgets.BoundedFloatText(value=options['offset_x'], min=-1, max=1, step=0.01) + offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01, description='offset_y'), + offset_x=widgets.BoundedFloatText(value=options['offset_x'], min=-1, max=1, step=0.01, description='offset_x') ) else: @@ -348,6 +356,8 @@ def plot_diffractogram_interactive(data, options): xlim=options['widgets']['xlim']['w']) + options['widget'] = w + display(w) @@ -357,7 +367,9 @@ def update_xminmax(xminmax, data, options={}): Input: minmax (dict): contains ''' + xminmax['2th'] = [None, None, None, None] for index, diffractogram in enumerate(data['diffractogram']): + if not xminmax['2th'][0] or diffractogram['2th'].min() < xminmax['2th'][0]: xminmax['2th'][0] = diffractogram['2th'].min() min_index = index @@ -366,23 +378,43 @@ def update_xminmax(xminmax, data, options={}): xminmax['2th'][1] = diffractogram['2th'].max() max_index = index - xminmax['2th_cuka'][0], xminmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() - xminmax['2th_moka'][0], xminmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() - xminmax['d'][0], xminmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() # swapped, intended - xminmax['1/d'][0], xminmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() - xminmax['q'][0], xminmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() - xminmax['q2'][0], xminmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() - xminmax['q4'][0], xminmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() - xminmax['heatmap'] = options['heatmap_xlim'] + + xminmax['2th'][2], xminmax['2th'][3] = xminmax['2th'][0], xminmax['2th'][1] + + xminmax['2th_cuka'][0], xminmax['2th_cuka'][1] = data['diffractogram'][min_index]['2th_cuka'].min(), data['diffractogram'][max_index]['2th_cuka'].max() + xminmax['2th_cuka'][2], xminmax['2th_cuka'][3] = xminmax['2th_cuka'][0], xminmax['2th_cuka'][1] + + xminmax['2th_moka'][0], xminmax['2th_moka'][1] = data['diffractogram'][min_index]['2th_moka'].min(), data['diffractogram'][max_index]['2th_moka'].max() + xminmax['2th_moka'][2], xminmax['2th_moka'][3] = xminmax['2th_moka'][0], xminmax['2th_moka'][1] + + xminmax['d'][0], xminmax['d'][1] = data['diffractogram'][max_index]['d'].min(), data['diffractogram'][min_index]['d'].max() # swapped, intended + xminmax['d'][2], xminmax['d'][3] = xminmax['d'][0], xminmax['d'][1] + + xminmax['1/d'][0], xminmax['1/d'][1] = data['diffractogram'][min_index]['1/d'].min(), data['diffractogram'][max_index]['1/d'].max() + xminmax['1/d'][2], xminmax['1/d'][3] = xminmax['1/d'][0], xminmax['1/d'][1] + + xminmax['q'][0], xminmax['q'][1] = data['diffractogram'][min_index]['q'].min(), data['diffractogram'][max_index]['q'].max() + xminmax['q'][2], xminmax['q'][3] = xminmax['q'][0], xminmax['q'][1] + + xminmax['q2'][0], xminmax['q2'][1] = data['diffractogram'][min_index]['q2'].min(), data['diffractogram'][max_index]['q2'].max() + xminmax['q2'][2], xminmax['q2'][3] = xminmax['q2'][0], xminmax['q2'][1] + + xminmax['q4'][0], xminmax['q4'][1] = data['diffractogram'][min_index]['q4'].min(), data['diffractogram'][max_index]['q4'].max() + xminmax['q4'][2], xminmax['q4'][3] = xminmax['q4'][0], xminmax['q4'][1] + + + xminmax['heatmap'] = options['heatmap_xlim'] # This value is set in the generate_heatmap()-function xminmax['start'][0], xminmax['start'][1] = xminmax[options['x_vals']][0], xminmax[options['x_vals']][1] - xminmax['start'][2], xminmax['start'][3] = xminmax[options['x_vals']][0], xminmax[options['x_vals']][1] + xminmax['start'][2], xminmax['start'][3] = xminmax[options['x_vals']][2], xminmax[options['x_vals']][3] -def update_yminmax(yminmax: dict, data, options={}): +def update_yminmax(yminmax: dict, data: dict, options={}) -> None: - for index, diffractogram in enumerate(data['diffractogram']): + yminmax['diff'] = [None, None, None, None] + # Go through diffractograms and find the minimum and maximum intensity values + for diffractogram in data['diffractogram']: if not yminmax['diff'][0] or (yminmax['diff'][0] > (diffractogram['I'].min())): yminmax['diff'][0] = diffractogram['I'].min() @@ -399,8 +431,9 @@ def update_yminmax(yminmax: dict, data, options={}): yminmax['diff'][1] = yminmax['diff'][1]*5 - yminmax['heatmap'][0] = 0 - yminmax['heatmap'][1] = data['heatmap'].shape[0] + # Set start values to the edges of the dataset + yminmax['heatmap'][0], yminmax['heatmap'][1] = 0, data['heatmap'].shape[0] + yminmax['heatmap'][2], yminmax['heatmap'][3] = yminmax['heatmap'][0], yminmax['heatmap'][1] if options['heatmap']: @@ -413,59 +446,89 @@ def update_yminmax(yminmax: dict, data, options={}): yminmax['start'][2], yminmax['start'][3] = yminmax['diff'][2], yminmax['diff'][3] +def update_defaults(widget: dict, minmax: dict) -> None: + ''' Updates the default x- or y-limits of a given widget. Refer to plot_diffractogram_interactive() to see the form of the widget that is passed in. An update of the min/max-values is done just prior to calling this function. + Changes dictionaries in place. + + Input: + widget (dict): A dictionary containing the widget itself (widget['w']) and all its default-values (e.g. widget['2th_default']) + minmax (dict): A dictionary containing min and max values, as well as min_start and max_start values. (e.g. minmax['2th'] is a list with four elements: [xmin, xmax, xmin_start, xmax_start]) + + Output: + None.''' + + for name, attr in widget.items(): + if name.endswith('default'): + attr['min'] = minmax[name.replace('_default', '')][0] + attr['max'] = minmax[name.replace('_default', '')][1] + attr['value'] = [minmax[name.replace('_default', '')][2], minmax[name.replace('_default', '')][3]] -def update_widgets(options): +def update_widgets(data, options): - for widget, attr in options['widgets'].items(): - if widget == 'xlim': + for widget_name, widget in options['widgets'].items(): - if options['heatmap'] and (attr['state'] != 'heatmap'): - setattr(attr['w'], 'min', attr['heatmap_default']['min']) - setattr(attr['w'], 'max', attr['heatmap_default']['max']) - setattr(attr['w'], 'value', attr['heatmap_default']['value']) - setattr(attr['w'], 'step', attr['heatmap_default']['step']) + # Make changes to xlim-widget + if widget_name == 'xlim': + # First update the min and max values + update_xminmax(xminmax=options['xminmax'], data=data, options=options) + update_defaults(widget=widget, minmax=options['xminmax']) + - attr['state'] = 'heatmap' + if options['heatmap'] and (widget['state'] != 'heatmap'): + + + setattr(widget['w'], 'min', widget['heatmap_default']['min']) + setattr(widget['w'], 'max', widget['heatmap_default']['max']) + setattr(widget['w'], 'value', widget['heatmap_default']['value']) + setattr(widget['w'], 'step', widget['heatmap_default']['step']) + + widget['state'] = 'heatmap' - elif not options['heatmap'] and (attr['state'] != options['x_vals']): - for arg in attr[f'{options["x_vals"]}_default']: + elif not options['heatmap'] and (widget['state'] != options['x_vals']): + # Then loop through all attributes in the widget and change to current mode. + for arg in widget[f'{options["x_vals"]}_default']: # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first if arg == 'min': - if attr[f'{options["x_vals"]}_default']['min'] > getattr(attr['w'], 'max'): - setattr(attr['w'], 'max', attr[f'{options["x_vals"]}_default']['max']) + if widget[f'{options["x_vals"]}_default']['min'] > getattr(widget['w'], 'max'): + setattr(widget['w'], 'max', widget[f'{options["x_vals"]}_default']['max']) elif arg == 'max': - if attr[f'{options["x_vals"]}_default']['max'] < getattr(attr['w'], 'min'): - setattr(attr['w'], 'min', attr[f'{options["x_vals"]}_default']['min']) + if widget[f'{options["x_vals"]}_default']['max'] < getattr(widget['w'], 'min'): + setattr(widget['w'], 'min', widget[f'{options["x_vals"]}_default']['min']) - setattr(attr['w'], arg, attr[f'{options["x_vals"]}_default'][arg]) + setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) - attr['state'] = options['x_vals'] + widget['state'] = options['x_vals'] - elif widget == 'ylim': - state = 'heatmap' if options['heatmap'] else 'diff' + # Make changes to ylim-widget + elif widget_name == 'ylim': + update_yminmax(yminmax=options['yminmax'], data=data, options=options) + update_defaults(widget=widget, minmax=options['yminmax']) + + state = 'heatmap' if options['heatmap'] else 'diff' - if attr['state'] != state: + if widget['state'] != state or options['offset_change']: - for arg in attr[f'{state}_default']: + for arg in widget[f'{state}_default']: # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first if arg == 'min': - if attr[f'{state}_default']['min'] > getattr(attr['w'], 'max'): - setattr(attr['w'], 'max', attr[f'{state}_default']['max']) + if widget[f'{state}_default']['min'] > getattr(widget['w'], 'max'): + setattr(widget['w'], 'max', widget[f'{state}_default']['max']) elif arg == 'max': - if attr[f'{state}_default']['max'] < getattr(attr['w'], 'min'): - setattr(attr['w'], 'min', attr[f'{state}_default']['min']) + if widget[f'{state}_default']['max'] < getattr(widget['w'], 'min'): + setattr(widget['w'], 'min', widget[f'{state}_default']['min']) - setattr(attr['w'], arg, attr[f'{state}_default'][arg]) - - attr['state'] = state + setattr(widget['w'], arg, widget[f'{state}_default'][arg]) + + options['offset_change'] = False + widget['state'] = state From dd7f2d9dea4f91ea3758ab776d0ef5c01037fce3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 12:44:33 +0200 Subject: [PATCH 107/355] Fix bug making last commit not work as intended --- beamtime/xrd/plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index 9012b4a..db60b1a 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -54,7 +54,7 @@ def plot_diffractogram(data, options={}): default_options['offset_y'] = 0.05 options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - options['current_offset_y'] = options['offset_y'] + #options['current_offset_y'] = options['offset_y'] # Convert data['path'] to list to allow iteration over this to accommodate both single and multiple diffractograms if not isinstance(data['path'], list): From 63726033249475c45a897d45fb311f08feb7ce99 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 13:42:34 +0200 Subject: [PATCH 108/355] Translate relfections to heatmap x-coords --- beamtime/xrd/io.py | 25 ++++++++++++++++---- beamtime/xrd/plot.py | 54 ++++++++++++++++++++++++-------------------- 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index 910811b..c2447d8 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -400,7 +400,7 @@ def revert_offset(diffractogram,which=None): return diffractogram -def load_reflection_table(data, options={}): +def load_reflection_table(data: dict, reflections_params: dict, options={}): required_options = ['ref_wavelength', 'to_wavelength'] @@ -413,12 +413,12 @@ def load_reflection_table(data, options={}): # VESTA outputs the file with a header that has a space between the parameter and units - so there is some extra code to rectify the issue # that ensues from this formatting - reflections = pd.read_csv(data['path'], delim_whitespace=True) + reflections = pd.read_csv(reflections_params['path'], delim_whitespace=True) # Remove the extra column that appears from the headers issue reflections.drop(reflections.columns[-1], axis=1, inplace=True) - with open(data['path'], 'r') as f: + with open(reflections_params['path'], 'r') as f: line = f.readline() headers = line.split() @@ -434,13 +434,28 @@ def load_reflection_table(data, options={}): reflections = translate_wavelengths(data=reflections, wavelength=options['ref_wavelength'], to_wavelength=options['to_wavelength']) - #print(reflections) + if 'heatmap' in data.keys(): + + start_2th, stop_2th = data['diffractogram'][0]['2th'].min(), data['diffractogram'][0]['2th'].max() + len_2th = stop_2th - start_2th + #print(start_2th, stop_2th, len_2th) + + start_heatmap, stop_heatmap = 0, data['heatmap'].shape[1] + len_heatmap = stop_heatmap - start_heatmap + #print(start_heatmap, stop_heatmap, len_heatmap) + + scale = len_heatmap/len_2th + + #print(scale) + #print(stop_2th * scale) + + reflections['heatmap'] = (reflections['2th']-start_2th) * scale return reflections -def translate_wavelengths(data, wavelength, to_wavelength=None): +def translate_wavelengths(data: pd.DataFrame, wavelength: float, to_wavelength=None) -> pd.DataFrame: # FIXME Somewhere here there is an invalid arcsin-argument. Not sure where. pd.options.mode.chained_assignment = None diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index db60b1a..cc6baa3 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -80,6 +80,8 @@ def plot_diffractogram(data, options={}): # Generate heatmap data data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'] = generate_heatmap(data=data, options=options) + options['heatmap_loaded'] = True + if options['heatmap']: options['xlim'] = options['heatmap_xlim'] @@ -163,16 +165,16 @@ def plot_diffractogram(data, options={}): options['xlim'] = ax.get_xlim() options['to_wavelength'] = data['wavelength'][0] - for reference, axis in zip(options['reflections_data'], ref_axes): - plot_reflection_table(data=reference, ax=axis, options=options) + for reflections_params, axis in zip(options['reflections_data'], ref_axes): + plot_reflection_table(data=data, reflections_params=reflections_params, ax=axis, options=options) # Print the reflection indices. By default, the wavelength of the first diffractogram will be used for this. if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() options['to_wavelength'] = data['wavelength'][0] - for reference in options['reflections_data']: - plot_reflection_indices(data=reference, ax=indices_ax, options=options) + for reflections_params in options['reflections_data']: + plot_reflection_indices(data=data, reflections_params=reflections_params, ax=indices_ax, options=options) if options['interactive_session_active']: @@ -533,7 +535,7 @@ def update_widgets(data, options): -def plot_reflection_indices(data, ax, options={}): +def plot_reflection_indices(data, reflections_params, ax, options={}): ''' Print reflection indices from output generated by VESTA. Required contents of data: @@ -547,20 +549,21 @@ def plot_reflection_indices(data, ax, options={}): 'hide_indices': False } - data = aux.update_options(options=data, required_options=required_options, default_options=default_options) + reflections_params = aux.update_options(options=reflections_params, required_options=required_options, default_options=default_options) - if not data['hide_indices']: - reflection_table = xrd.io.load_reflection_table(data=data, options=options) + if not reflections_params['hide_indices']: + reflection_table = xrd.io.load_reflection_table(data=data, reflections_params=reflections_params, options=options) - if data['reflection_indices'] > 0: + if reflections_params['reflection_indices'] > 0: # Get the data['reflection_indices'] number of highest reflections within the subrange options['xlim'] - reflection_indices = reflection_table.loc[(reflection_table[options['x_vals']] > options['xlim'][0]) & (reflection_table[options['x_vals']] < options['xlim'][1])].nlargest(options['reflection_indices'], 'I') + x_vals = 'heatmap' if options['heatmap'] else options['x_vals'] + reflection_indices = reflection_table.loc[(reflection_table[x_vals] > options['xlim'][0]) & (reflection_table[x_vals] < options['xlim'][1])].nlargest(options['reflection_indices'], 'I') # Plot the indices - for i in range(data['reflection_indices']): + for i in range(reflections_params['reflection_indices']): if reflection_indices.shape[0] > i: - ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices[options['x_vals']].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=data['text_colour']) + ax.text(s=f'({reflection_indices["h"].iloc[i]} {reflection_indices["k"].iloc[i]} {reflection_indices["l"].iloc[i]})', x=reflection_indices[x_vals].iloc[i], y=0, fontsize=2.5, rotation=90, va='bottom', ha='center', c=reflections_params['text_colour']) if options['xlim']: @@ -571,7 +574,7 @@ def plot_reflection_indices(data, ax, options={}): return -def plot_reflection_table(data, ax=None, options={}): +def plot_reflection_table(data, reflections_params, ax=None, options={}): ''' Plots a reflection table from output generated by VESTA. Required contents of data: @@ -590,15 +593,15 @@ def plot_reflection_table(data, ax=None, options={}): } if 'colour' in data.keys(): - options['reflections_colour'] = data['colour'] - if 'min_alpha' in data.keys(): - options['min_alpha'] = data['min_alpha'] - if 'reflection_indices' in data.keys(): - options['reflection_indices'] = data['reflection_indices'] - if 'label' in data.keys(): - options['label'] = data['label'] - if 'wavelength' in data.keys(): - options['wavelength'] = data['wavelength'] + options['reflections_colour'] = reflections_params['colour'] + if 'min_alpha' in reflections_params.keys(): + options['min_alpha'] = reflections_params['min_alpha'] + if 'reflection_indices' in reflections_params.keys(): + options['reflection_indices'] = reflections_params['reflection_indices'] + if 'label' in reflections_params.keys(): + options['label'] = reflections_params['label'] + if 'wavelength' in reflections_params.keys(): + options['wavelength'] = reflections_params['wavelength'] options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -606,9 +609,10 @@ def plot_reflection_table(data, ax=None, options={}): if not ax: _, ax = btp.prepare_plot(options) - reflection_table = xrd.io.load_reflection_table(data=data, options=options) + x_vals = 'heatmap' if options['heatmap'] else options['x_vals'] - reflections, intensities = reflection_table[options['x_vals']], reflection_table['I'] + reflection_table = xrd.io.load_reflection_table(data=data, reflections_params=reflections_params, options=options) + reflections, intensities = reflection_table[x_vals], reflection_table['I'] @@ -638,7 +642,7 @@ def plot_reflection_table(data, ax=None, options={}): xlim_range = ax.get_xlim()[1] - ax.get_xlim()[0] ylim_avg = (ax.get_ylim()[0]+ax.get_ylim()[1])/2 - ax.text(s=data['label'], x=(ax.get_xlim()[0]-0.01*xlim_range), y=ylim_avg, ha = 'right', va = 'center') + ax.text(s=reflections_params['label'], x=(ax.get_xlim()[0]-0.01*xlim_range), y=ylim_avg, ha = 'right', va = 'center') From bdfc31901341309a725e5450615fc2823a2554ba Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 14:43:07 +0200 Subject: [PATCH 109/355] Initial commit of test --- beamtime/test/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test/__init__.py diff --git a/beamtime/test/__init__.py b/beamtime/test/__init__.py new file mode 100644 index 0000000..e69de29 From 1146c04a383086fd41f4362c36de0e9a08a34298 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 15:02:38 +0200 Subject: [PATCH 110/355] Add first tests --- beamtime/test/pytest.ini | 6 ++++++ beamtime/test/test_auxillary.py | 29 +++++++++++++++++++++++++++++ beamtime/test/test_plotting.py | 8 ++++++++ beamtime/test/xrd/test_io.py | 0 beamtime/test/xrd/test_plot.py | 0 5 files changed, 43 insertions(+) create mode 100644 beamtime/test/pytest.ini create mode 100644 beamtime/test/test_auxillary.py create mode 100644 beamtime/test/test_plotting.py create mode 100644 beamtime/test/xrd/test_io.py create mode 100644 beamtime/test/xrd/test_plot.py diff --git a/beamtime/test/pytest.ini b/beamtime/test/pytest.ini new file mode 100644 index 0000000..8108e29 --- /dev/null +++ b/beamtime/test/pytest.ini @@ -0,0 +1,6 @@ +# pytest.ini + +[pytest] +minversion = 6.0 +testpaths = + . diff --git a/beamtime/test/test_auxillary.py b/beamtime/test/test_auxillary.py new file mode 100644 index 0000000..5a9e85e --- /dev/null +++ b/beamtime/test/test_auxillary.py @@ -0,0 +1,29 @@ +import beamtime.auxillary as aux + +def test_swap_values(): + + + dict = {'test1': 1, 'test2': 2} + key1 = 'test1' + key2 = 'test2' + + oldval1 = dict[key1] + oldval2 = dict[key2] + + new_dict = aux.swap_values(dict=dict, key1=key1, key2=key2) + + assert (dict[key1] == oldval2) and (dict[key2] == oldval1) + + +def test_ceil() -> None: + + assert aux.ceil(1.05, 0.5) == 1.5 + assert aux.ceil(1.05, 1) == 2.0 + assert aux.ceil(1.1, 0.2) == 1.2 + + +def test_floor() -> None: + + assert aux.floor(2.02, 1) == 2.0 + assert aux.floor(2.02, 0.01) == 2.02 + assert aux.floor(2.013, 0.01) == 2.01 \ No newline at end of file diff --git a/beamtime/test/test_plotting.py b/beamtime/test/test_plotting.py new file mode 100644 index 0000000..1374778 --- /dev/null +++ b/beamtime/test/test_plotting.py @@ -0,0 +1,8 @@ +import beamtime.plotting as btp +from cycler import cycler +import itertools + + +def test_generate_colours() -> None: + + assert type(btp.generate_colours('black', kind='single')) == itertools.cycle \ No newline at end of file diff --git a/beamtime/test/xrd/test_io.py b/beamtime/test/xrd/test_io.py new file mode 100644 index 0000000..e69de29 diff --git a/beamtime/test/xrd/test_plot.py b/beamtime/test/xrd/test_plot.py new file mode 100644 index 0000000..e69de29 From e07acdb4bb839580b541777ebc8937a624aecec6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 15:06:03 +0200 Subject: [PATCH 111/355] Remove packages not used --- beamtime/plotting.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index 500f532..946a4e4 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -1,13 +1,11 @@ import beamtime.auxillary as aux import matplotlib.pyplot as plt -from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) -from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition, mark_inset) +from matplotlib.ticker import (MultipleLocator) import importlib import matplotlib.patches as mpatches from matplotlib.lines import Line2D import matplotlib.lines as mlines -from cycler import cycler import itertools From 657276eb9177669dc4e4bda468759ea64c93ee20 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 15:57:30 +0200 Subject: [PATCH 112/355] Add tests for auxillary.py --- beamtime/test/test_auxillary.py | 51 ++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/beamtime/test/test_auxillary.py b/beamtime/test/test_auxillary.py index 5a9e85e..668b792 100644 --- a/beamtime/test/test_auxillary.py +++ b/beamtime/test/test_auxillary.py @@ -1,4 +1,5 @@ import beamtime.auxillary as aux +import os def test_swap_values(): @@ -26,4 +27,52 @@ def test_floor() -> None: assert aux.floor(2.02, 1) == 2.0 assert aux.floor(2.02, 0.01) == 2.02 - assert aux.floor(2.013, 0.01) == 2.01 \ No newline at end of file + assert aux.floor(2.013, 0.01) == 2.01 + + + +def test_options() -> None: + + + options = {} + required_options = ['test1', 'test2', 'test3', 'test4'] + default_options = { + 'test1': 1, + 'test2': 2, + 'test3': 3, + 'test4': 4, + 'test5': 5, + } + + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + assert options['test1'] == default_options['test1'] + assert len(options.items()) == len(required_options) + assert 'test5' not in options.keys() + + +def test_save_options() -> None: + + options = {'test1': 1, 'test2': 2} + path = 'tmp.dat' + + aux.save_options(options, path) + + assert os.path.isfile(path) + + os.remove(path) + + +def test_load_options() -> None: + + options = {'test1': 1, 'test2': 2} + path = 'tmp.dat' + + aux.save_options(options, path) + + loaded_options = aux.load_options(path) + + assert (loaded_options['test1'] == 1) and (loaded_options['test2'] == 2) + + os.remove(path) \ No newline at end of file From 4587322a9b199016f443e03b459e5fa1a447ac4b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 17:25:46 +0200 Subject: [PATCH 113/355] Add tests for plotting.py --- beamtime/test/test_plotting.py | 126 ++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 1 deletion(-) diff --git a/beamtime/test/test_plotting.py b/beamtime/test/test_plotting.py index 1374778..3b9ccda 100644 --- a/beamtime/test/test_plotting.py +++ b/beamtime/test/test_plotting.py @@ -1,8 +1,132 @@ import beamtime.plotting as btp from cycler import cycler import itertools +import numpy as np + +import matplotlib.pyplot as plt def test_generate_colours() -> None: - assert type(btp.generate_colours('black', kind='single')) == itertools.cycle \ No newline at end of file + assert type(btp.generate_colours('black', kind='single')) == itertools.cycle + + palettes = [('qualitative', 'Dark2_8')] + colour_cycle = btp.generate_colours(palettes) + + assert type(colour_cycle) == itertools.cycle + + + # Test that it actually loaded 8 colours when given a set of 8 colours to + + same_colour = None + for i in range(10): + colour = next(colour_cycle) + if i == 0: + first_colour = colour + + if colour == first_colour: + repeat_colour_index = i + + + assert repeat_colour_index == 8 + + + + +def test_update_rc_params() -> None: + + rc_params = { + 'lines.linewidth': 100 + } + + prev_params = plt.rcParams['lines.linewidth'] + + # Update run commands if any is passed (will pass an empty dictionary if not passed) + btp.update_rc_params(rc_params) + + new_params = plt.rcParams['lines.linewidth'] + + assert new_params == 100 + assert prev_params != new_params + + + # Reset run commands + plt.rcdefaults() + + + +def test_scale_figure() -> None: + + width, height = 1, 1 + + format_params = { + 'upscaling_factor': 2, + 'compress_width': 1, + 'compress_height': 1 + } + + width1, height1 = btp.scale_figure(format_params=format_params, width=width, height=height) + + assert width1 == 2 and height1 == 2 + + format_params = { + 'upscaling_factor': 1, + 'compress_width': 0.5, + 'compress_height': 1 + } + + width2, height2 = btp.scale_figure(format_params=format_params, width=width, height=height) + + assert width2 == 0.5 and height2 == 1 + + format_params = { + 'upscaling_factor': 2, + 'compress_width': 0.5, + 'compress_height': 0.2 + } + + width2, height2 = btp.scale_figure(format_params=format_params, width=width, height=height) + + assert width2 == 1 and height2 == 0.4 + + +def test_determine_width() -> None: + + conversion_cm_inch = 0.3937008 # cm to inch + + format_params = { + 'column_type': 'single', + 'single_column_width': 5, + 'double_column_width': 10, + 'width_ratio': '1:1' + } + + assert np.round(btp.determine_width(format_params),6) == np.round(5*conversion_cm_inch,6) + + format_params['column_type'] = 'double' + + assert np.round(btp.determine_width(format_params), 6) == np.round(10*conversion_cm_inch, 6) + + + format_params['column_type'] = 'single' + format_params['width_ratio'] = '1:2' + + assert np.round(btp.determine_width(format_params), 6) == np.round(2.5*conversion_cm_inch, 6) + +def test_determine_height() -> None: + + + width = 1 + + format_params = { + 'aspect_ratio': '1:1' + } + + assert btp.determine_height(format_params=format_params, width=width) == 1 + + format_params['aspect_ratio'] = '3:1' + + assert (btp.determine_height(format_params=format_params, width=width) - 0.333333333333333) < 10e-7 + + assert True + From d702875ab6ff31aed76a2c4e5c75a3e32894ed8c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 17:28:04 +0200 Subject: [PATCH 114/355] Ignore DeprecationWarning --- beamtime/test/pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beamtime/test/pytest.ini b/beamtime/test/pytest.ini index 8108e29..c317621 100644 --- a/beamtime/test/pytest.ini +++ b/beamtime/test/pytest.ini @@ -4,3 +4,6 @@ minversion = 6.0 testpaths = . + +filterwarnings = + ignore::DeprecationWarning From e3b0e2bc14fee5733f87803254aa4fe2e614b9f6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Apr 2022 17:29:45 +0200 Subject: [PATCH 115/355] Move some files to root folder --- beamtime/README.md => README.md | 0 beamtime/feature_list.txt => feature_list.txt | 0 beamtime/reqirements2.txt => reqirements2.txt | 0 beamtime/requirements.txt => requirements.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename beamtime/README.md => README.md (100%) rename beamtime/feature_list.txt => feature_list.txt (100%) rename beamtime/reqirements2.txt => reqirements2.txt (100%) rename beamtime/requirements.txt => requirements.txt (100%) diff --git a/beamtime/README.md b/README.md similarity index 100% rename from beamtime/README.md rename to README.md diff --git a/beamtime/feature_list.txt b/feature_list.txt similarity index 100% rename from beamtime/feature_list.txt rename to feature_list.txt diff --git a/beamtime/reqirements2.txt b/reqirements2.txt similarity index 100% rename from beamtime/reqirements2.txt rename to reqirements2.txt diff --git a/beamtime/requirements.txt b/requirements.txt similarity index 100% rename from beamtime/requirements.txt rename to requirements.txt From 872ae759b29a45ece7cf070a6deac869a3c6fe3a Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 6 Apr 2022 21:10:40 +0200 Subject: [PATCH 116/355] Optimizing code and splitting into smaller functions --- beamtime/xanes/calib.py | 125 +++++++++++++++++++++------------------- beamtime/xanes/io.py | 2 +- 2 files changed, 68 insertions(+), 59 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 3c54dd1..8cdbdf7 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -3,7 +3,8 @@ import numpy as np import os import matplotlib.pyplot as plt import beamtime.auxillary as aux - +import beamtime.xanes as xas +import beamtime.xanes.io as io def rbkerbest(): print("ROSENBORG!<3") @@ -14,99 +15,108 @@ def rbkerbest(): ##Better to make a new function that loops through the files, and performing the split_xanes_scan on +#Tryiung to make a function that can decide which edge it is based on the first ZapEnergy-value +def finding_edge(df): + if 5.9 < df["ZapEnergy"][0] < 6.5: + edge='Mn' + return(edge) + if 8.0 < df["ZapEnergy"][0] < 8.6: + edge='Ni' + return(edge) -def pre_edge_subtraction(df,filenames, options={}): +#def pre_edge_subtraction(df,filenames, options={}): +def test(innmat): + df_test= xas.io.put_in_dataframe(innmat) + print(df_test) - required_options = ['edge', 'print'] +def pre_edge_subtraction(path, options={}): + required_options = ['print','troubleshoot'] default_options = { - 'edge' : 'Mn', - 'print': False + 'print': False, + 'troubleshoot': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge - if str(options['edge']) == 'Mn': + filenames = xas.io.get_filenames(path) + df= xas.io.put_in_dataframe(path) + edge=finding_edge(df) + + #Defining the end of the region used to define the background, thus start of the edge + #implement widget + if edge == 'Mn': edge_start = 6.45 - if str(options['edge']) == 'Ni': + if edge == 'Ni': edge_start = 8.3 - - #making a function to check the difference between values in the list and the defined start of the edge (where background regression will stop): - absolute_difference_function = lambda list_value : abs(list_value - edge_start) - - #finding the energy data point value that is closest to what I defined as the end of the background - edge_start_value = min(df["ZapEnergy"], key=absolute_difference_function) - - #Finding what the index of the edge shift end point is - start_index=df[df["ZapEnergy"]==edge_start_value].index.values[0] - - #Defining x-range for linear background fit, ending at the edge start index - df_start=df[0:start_index] - - #Making a new dataframe, with only the ZapEnergies as the first column - df_background = pd.DataFrame(df["ZapEnergy"]) + #making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) + df_start=df.loc[df["ZapEnergy"] < edge_start] + + #Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data + df_bkgd = pd.DataFrame(df["ZapEnergy"]) for files in filenames: - #Fitting linear function to the pre-edge + #Fitting linear function to the background d = np.polyfit(df_start["ZapEnergy"],df_start[files],1) - function_pre = np.poly1d(d) + function_bkgd = np.poly1d(d) #making a list, y_pre,so the background will be applied to all ZapEnergy-values - y_pre=function_pre(df["ZapEnergy"]) + y_bkgd=function_bkgd(df["ZapEnergy"]) #adding a new column in df_background with the y-values of the background - df_background.insert(1,files,y_pre) + df_bkgd.insert(1,files,y_bkgd) - #Plotting the calculated pre-edge background with the region used for the regression - - ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILES IS CORRUPT - #ax = df.plot(x = "ZapEnergy",y=files) + if options['troubleshoot'] == True: + ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILE IS CORRUPT + ax = df.plot(x = "ZapEnergy",y=files) + #Plotting the calculated pre-edge background with the region used for the regression if options['print'] == True: #Plotting an example of the edge_start region and the fitted background that will later be subtracted - ax = df.plot(x = "ZapEnergy",y=filenames[0]) #defining x and y - plt.axvline(x = edge_start_value) - fig = plt.figure(figsize=(15,15)) - df_background.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) - + fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) + df.plot(x = "ZapEnergy",y=filenames[0],ax=ax1) #defining x and y + plt.axvline(x = max(df_start["ZapEnergy"])) + #fig = plt.figure(figsize=(15,15)) + df_bkgd.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax1) + ax1.set_title('Data and fitted background') ###################### Subtracting the pre edge from xmap_roi00 ################ #making a new dataframe to insert the background subtracted intensities - df_new = pd.DataFrame(df["ZapEnergy"]) + df_bkgd_sub = pd.DataFrame(df["ZapEnergy"]) #inserting the pre_edge-background subtracted original xmap_roi00 data for files in filenames: - newintensity_calc=df[files]-df_background[files] - df_new.insert(1,files,newintensity_calc) + newintensity_calc=df[files]-df_bkgd[files] + df_bkgd_sub.insert(1,files,newintensity_calc) if options['print'] == True: - #Plotting original data (black) and background subtracted data (red) - ax = df.plot(x = "ZapEnergy",y=filenames[0], color="Black") - plt.axvline(x = edge_start_value) - fig = plt.figure(figsize=(15,15)) - df_new.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax) - return df_new + df.plot(x = "ZapEnergy",y=filenames[0], color="Black", ax=ax2, legend=False) + plt.axvline(x = max(df_start["ZapEnergy"])) + df_bkgd_sub.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax2, legend=False) + ax2.set_title('Data and background-subtracted data') -def post_edge_normalization(df,df_new,filenames, options={}): + return df_bkgd_sub - required_options = ['edge', 'print'] +def post_edge_normalization(df,df_backg_sub,filenames, options={}): + + required_options = ['print'] default_options = { - 'edge' : 'Mn', 'print': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - + + edge=finding_edge(df) #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge - if str(options['edge']) == 'Mn': + #Implement widget + if edge == 'Mn': edge_stop = 6.565 - if str(options['edge']) == 'Ni': + if edge == 'Ni': edge_stop = 8.361 absolute_difference_function = lambda list_value : abs(list_value - edge_stop) - edge_stop_value = min(df_new["ZapEnergy"], key=absolute_difference_function) - end_index=df_new[df_new["ZapEnergy"]==edge_stop_value].index.values[0] + edge_stop_value = min(df_backg_sub["ZapEnergy"], key=absolute_difference_function) + end_index=df_backg_sub[df_backg_sub["ZapEnergy"]==edge_stop_value].index.values[0] #Defining x-range for linear fit - df_fix=df_new + df_fix=df_backg_sub df_fix.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit df_end=df_fix[end_index:] #The region of interest for the post edge #print(df_end) @@ -125,14 +135,13 @@ def post_edge_normalization(df,df_new,filenames, options={}): #print(df_postedge) #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line if options['print'] == True: - ax = df_new.plot(x = "ZapEnergy",y=filenames) #defining x and y + ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y plt.axvline(x = edge_stop_value) fig = plt.figure(figsize=(15,15)) df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) #print(function_post_list) #print(function_post) - ax = df_new.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y + ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) plt.axvline(x = edge_stop_value) - - \ No newline at end of file + \ No newline at end of file diff --git a/beamtime/xanes/io.py b/beamtime/xanes/io.py index 2ed6d29..527f300 100644 --- a/beamtime/xanes/io.py +++ b/beamtime/xanes/io.py @@ -1,7 +1,7 @@ import pandas as pd import matplotlib.pyplot as plt import os - +import numpy as np def split_xanes_scan(root, destination=None, replace=False): #root is the path to the beamtime-folder From 135d577b4b78c3e6985bc99c765cf1e89a272456 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Apr 2022 11:34:44 +0200 Subject: [PATCH 117/355] Adjusting post edge processing --- beamtime/xanes/calib.py | 68 ++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 8cdbdf7..9e3cbdb 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -44,7 +44,7 @@ def pre_edge_subtraction(path, options={}): #Defining the end of the region used to define the background, thus start of the edge #implement widget if edge == 'Mn': - edge_start = 6.45 + edge_start = 6.42 if edge == 'Ni': edge_start = 8.3 @@ -73,30 +73,44 @@ def pre_edge_subtraction(path, options={}): #Plotting the calculated pre-edge background with the region used for the regression if options['print'] == True: #Plotting an example of the edge_start region and the fitted background that will later be subtracted - fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) - df.plot(x = "ZapEnergy",y=filenames[0],ax=ax1) #defining x and y + fig, (ax1,ax2,ax3) = plt.subplots(1,3,figsize=(15,5)) + df.plot(x="ZapEnergy", y=filenames,color="Black",ax=ax1) + df_bkgd.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax1) plt.axvline(x = max(df_start["ZapEnergy"])) #fig = plt.figure(figsize=(15,15)) - df_bkgd.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax1) + df_bkgd.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax2) ax1.set_title('Data and fitted background') + #Zooming into bacground region to confirm fit and limits looks reasonable + df.plot(x = "ZapEnergy",y=filenames,ax=ax2) #defining x and y) + ax2.set_xlim([min(df_start["ZapEnergy"]),max(df_start["ZapEnergy"])+0.01]) + #finding maximum and minimum values in the backgrounds + min_values=[] + max_values=[] + for file in filenames: + min_values.append(min(df_start[file])) + max_values.append(max(df_start[file])) + ax2.set_ylim([min(min_values),max(max_values)]) + plt.axvline(x = max(df_start["ZapEnergy"])) + #ax2.set_xlim([25, 50]) ###################### Subtracting the pre edge from xmap_roi00 ################ + #making a new dataframe to insert the background subtracted intensities df_bkgd_sub = pd.DataFrame(df["ZapEnergy"]) - #inserting the pre_edge-background subtracted original xmap_roi00 data + #inserting the background subtracted original xmap_roi00 data for files in filenames: newintensity_calc=df[files]-df_bkgd[files] df_bkgd_sub.insert(1,files,newintensity_calc) if options['print'] == True: - df.plot(x = "ZapEnergy",y=filenames[0], color="Black", ax=ax2, legend=False) - plt.axvline(x = max(df_start["ZapEnergy"])) - df_bkgd_sub.plot(x="ZapEnergy", y=filenames[0],color="Red",ax=ax2, legend=False) - ax2.set_title('Data and background-subtracted data') + df.plot(x = "ZapEnergy",y=filenames, color="Black", ax=ax3, legend=False) + #plt.axvline(x = max(df_start["ZapEnergy"])) + df_bkgd_sub.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax3, legend=False) + ax3.set_title('Data and background-subtracted data') - return df_bkgd_sub + return df_bkgd_sub,filenames,edge -def post_edge_normalization(df,df_backg_sub,filenames, options={}): +def post_edge_normalization(path, options={}): required_options = ['print'] default_options = { @@ -104,7 +118,7 @@ def post_edge_normalization(df,df_backg_sub,filenames, options={}): } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - edge=finding_edge(df) + df_bkgd_sub,filenames,edge = pre_edge_subtraction(path) #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge #Implement widget if edge == 'Mn': @@ -112,22 +126,26 @@ def post_edge_normalization(df,df_backg_sub,filenames, options={}): if edge == 'Ni': edge_stop = 8.361 - absolute_difference_function = lambda list_value : abs(list_value - edge_stop) - edge_stop_value = min(df_backg_sub["ZapEnergy"], key=absolute_difference_function) - end_index=df_backg_sub[df_backg_sub["ZapEnergy"]==edge_stop_value].index.values[0] +#============================================================= + #absolute_difference_function = lambda list_value : abs(list_value - edge_stop) + #edge_stop_value = min(df_bkgd_sub["ZapEnergy"], key=absolute_difference_function) + #end_index=df_bkgd_sub[df_bkgd_sub["ZapEnergy"]==edge_stop_value].index.values[0] #Defining x-range for linear fit - df_fix=df_backg_sub - df_fix.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit - df_end=df_fix[end_index:] #The region of interest for the post edge - #print(df_end) + #df_fix=df_bkgd_sub + #df_fix.dropna(inplace=True) + #df_end=df_fix[end_index:] #The region of interest for the post edge + #Fitting linear function to the pre-edge using the background corrected intensities to make the post edge fit - df_postedge = pd.DataFrame(df["ZapEnergy"]) + #=============================================================== + df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge -> to be fitted + df_end.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + df_postedge = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) #making a new dataframe function_post_list=[] for files in filenames: d = np.polyfit(df_end["ZapEnergy"],df_end[files],1) function_post = np.poly1d(d) - y_post=function_post(df["ZapEnergy"]) + y_post=function_post(df_bkgd_sub["ZapEnergy"]) function_post_list.append(function_post) df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge @@ -135,13 +153,13 @@ def post_edge_normalization(df,df_backg_sub,filenames, options={}): #print(df_postedge) #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line if options['print'] == True: - ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y - plt.axvline(x = edge_stop_value) + ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y + plt.axvline(x = min(df_end["ZapEnergy"])) fig = plt.figure(figsize=(15,15)) df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) #print(function_post_list) #print(function_post) - ax = df_backg_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y + ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) - plt.axvline(x = edge_stop_value) + plt.axvline(x = min(df_end["ZapEnergy"])) \ No newline at end of file From 1db489c21ddd0926b95ae617f56bb4f167201dbe Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 12:02:13 +0200 Subject: [PATCH 118/355] First commit on new repo --- beamtime/test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..2bf4342 --- /dev/null +++ b/beamtime/test.txt @@ -0,0 +1 @@ +Testing new repository \ No newline at end of file From a84bd065b2bc1b6bd62277af160088982bc398fa Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 12:20:08 +0200 Subject: [PATCH 119/355] Testing push on new repo --- beamtime/test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt deleted file mode 100644 index 2bf4342..0000000 --- a/beamtime/test.txt +++ /dev/null @@ -1 +0,0 @@ -Testing new repository \ No newline at end of file From e8ae6ba1224450ef587eac25ea5289a53c36a768 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 12:21:43 +0200 Subject: [PATCH 120/355] Testing push on new repo again --- beamtime/test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test.txt diff --git a/beamtime/test.txt b/beamtime/test.txt new file mode 100644 index 0000000..e69de29 From aafcc5a1ed5b432e1ab6e4b8ed7f0eeda71cb90d Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 12:25:31 +0200 Subject: [PATCH 121/355] Test of push in VS Code --- beamtime/test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/beamtime/test.txt b/beamtime/test.txt index e69de29..85476a8 100644 --- a/beamtime/test.txt +++ b/beamtime/test.txt @@ -0,0 +1 @@ +sdfsdfsdfsdf \ No newline at end of file From ab6bf231001a82deb1e29048d43eedf6033d9443 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Apr 2022 12:30:32 +0200 Subject: [PATCH 122/355] testing from terminal --- beamtime/test2.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 beamtime/test2.txt diff --git a/beamtime/test2.txt b/beamtime/test2.txt new file mode 100644 index 0000000..e69de29 From baa253ab3e9b369960576040668b69a0792e2386 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Apr 2022 12:34:49 +0200 Subject: [PATCH 123/355] tester fra vscode --- test.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..35d92ec --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +hei på dej \ No newline at end of file From bf689886659bb64be611d1d6441269a8db96222b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 14:05:40 +0200 Subject: [PATCH 124/355] Add more tests for plotting.py and made more general --- beamtime/plotting.py | 78 ++++++---------------------------- beamtime/test/test_plotting.py | 49 +++++++++++++++++++++ 2 files changed, 62 insertions(+), 65 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index 946a4e4..780a022 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -20,8 +20,15 @@ def prepare_plot(options={}): format_params will determine the size, aspect ratio, resolution etc. of the figure. Should be modified to conform with any requirements from a journal.''' - rc_params = options['rc_params'] - format_params = options['format_params'] + if 'rc_params' in options.keys(): + rc_params = options['rc_params'] + else: + rc_params = {} + + if 'format_params' in options.keys(): + format_params = options['format_params'] + else: + format_params = {} required_format_params = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'width', 'height', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi', @@ -80,53 +87,14 @@ def prepare_plot(options={}): return fig, axes -def prepare_plots(options={}): - - rc_params = options['rc_params'] - format_params = options['format_params'] - - required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] - - default_options = { - 'single_column_width': 8.3, - 'double_column_width': 17.1, - 'column_type': 'single', - 'width_ratio': '1:1', - 'aspect_ratio': '1:1', - 'compress_width': 1, - 'compress_height': 1, - 'upscaling_factor': 1.0, - 'dpi': 600, - } - - format_params = aux.update_options(format_params, required_options, default_options) - - - # Reset run commands - plt.rcdefaults() - - # Update run commands if any is passed (will pass an empty dictionary if not passed) - update_rc_params(rc_params) - - width = determine_width(format_params) - height = determine_height(format_params, width) - width, height = scale_figure(options=format_params, width=width, height=height) - - - if options['plot_kind'] == 'relative': - fig, axes = plt.subplots(nrows=1, ncols=options['number_of_frames'], figsize=(width,height), facecolor='w', dpi=format_params['dpi']) - - elif options['plot_kind'] == 'absolute': - fig, axes = plt.subplots(nrows=2, ncols=options['number_of_frames'], figsize=(width,height), gridspec_kw={'height_ratios': [1,5]}, facecolor='w', dpi=format_params['dpi']) - - return fig, axes - def adjust_plot(fig, ax, options): ''' A general function to adjust plot according to contents of the options-dictionary ''' required_options = [ 'plot_kind', + 'xlabel', 'ylabel', + 'xunit', 'yunit', 'hide_x_labels', 'hide_y_labels', 'hide_x_ticklabels', 'hide_y_ticklabels', 'hide_x_ticks', 'hide_y_ticks', @@ -141,6 +109,8 @@ def adjust_plot(fig, ax, options): default_options = { 'plot_kind': None, # defaults to None, but should be utilised when requiring special formatting for a particular plot + 'xlabel': None, 'ylabel': None, + 'xunit': None, 'yunit': None, 'hide_x_labels': False, 'hide_y_labels': False, # Whether the main labels on the x- and/or y-axes should be hidden 'hide_x_ticklabels': False, 'hide_y_ticklabels': False, # Whether ticklabels on the x- and/or y-axes should be hidden 'hide_x_ticks': False, 'hide_y_ticks': False, # Whether the ticks on the x- and/or y-axes should be hidden @@ -305,28 +275,6 @@ def ipywidgets_update(func, data, options={}, **kwargs): func(data=data, options=options) -def update_widgets(options): - - for widget in options['widgets'].values(): - - if widget['state'] != options['x_vals']: - for arg in widget[f'{options["x_vals"]}_default']: - - # If new min value is larger than previous max, or new max value is smaller than previous min, set the opposite first - if arg == 'min': - if widget[f'{options["x_vals"]}_default']['min'] > getattr(widget['w'], 'max'): - setattr(widget['w'], 'max', widget[f'{options["x_vals"]}_default']['max']) - - elif arg == 'max': - if widget[f'{options["x_vals"]}_default']['max'] < getattr(widget['w'], 'min'): - setattr(widget['w'], 'min', widget[f'{options["x_vals"]}_default']['min']) - - - setattr(widget['w'], arg, widget[f'{options["x_vals"]}_default'][arg]) - - widget['state'] = options['x_vals'] - - def determine_width(format_params): ''' ''' diff --git a/beamtime/test/test_plotting.py b/beamtime/test/test_plotting.py index 3b9ccda..8979bc1 100644 --- a/beamtime/test/test_plotting.py +++ b/beamtime/test/test_plotting.py @@ -4,6 +4,7 @@ import itertools import numpy as np import matplotlib.pyplot as plt +import matplotlib as mpl def test_generate_colours() -> None: @@ -130,3 +131,51 @@ def test_determine_height() -> None: assert True + + +def test_prepare_plot() -> None: + + fig, ax = btp.prepare_plot() + + assert type(fig) == plt.Figure + assert fig.get_dpi() == 600 + assert ax.get_xlim() == (0.0, 1.0) + + + +def test_adjust_plot() -> None: + + fig, ax = btp.prepare_plot() + + options = { + 'xlim': (0.0, 2.0), + 'title': 'Test' + } + + fig, ax = btp.adjust_plot(fig, ax, options) + + + assert ax.get_xlim() == (0.0, 2.0) + assert ax.get_title() == 'Test' + + + +def test_ipywidgets_update() -> None: + + + + def test_func(data, options): + test1 = options['test1'] + test2 = options['test2'] + + assert type(data) == dict + assert test1 == 1 + assert test2 == 2 + + + + data = {} + options = {} + + btp.ipywidgets_update(func=test_func, data=data, options=options, test1=1, test2=2) + From 239ea9f61e5fd8d1db1e8fc2e8a4f15429bddd23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:10:25 +0200 Subject: [PATCH 125/355] Create automated testing --- .github/workflows/python-app.yml | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml new file mode 100644 index 0000000..2e8690d --- /dev/null +++ b/.github/workflows/python-app.yml @@ -0,0 +1,39 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Python application + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +permissions: + contents: read + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + pytest From 5c9c93fbb60b6a6b28ba155d34206215d95dbb1c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 14:14:38 +0200 Subject: [PATCH 126/355] Exporting new requirements and environment --- environment.yml | 154 +++++++++++++++++++++++++++++ reqirements2.txt | 141 -------------------------- requirements.txt | 253 +++++++++++++++++++++++++++-------------------- 3 files changed, 302 insertions(+), 246 deletions(-) create mode 100644 environment.yml delete mode 100644 reqirements2.txt diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..b80378b --- /dev/null +++ b/environment.yml @@ -0,0 +1,154 @@ +name: beamtime +channels: + - conda-forge + - diffpy + - defaults +dependencies: + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py39h2bbff1b_0 + - atomicwrites=1.4.0=py_0 + - attrs=21.4.0=pyhd3eb1b0_0 + - backcall=0.2.0=pyhd3eb1b0_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - bottleneck=1.3.2=py39h7cc1a96_1 + - ca-certificates=2022.3.29=haa95532_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - certifi=2021.10.8=py39haa95532_2 + - cffi=1.15.0=py39h2bbff1b_1 + - colorama=0.4.4=pyhd3eb1b0_0 + - cycler=0.10.0=py_2 + - debugpy=1.4.1=py39hd77b12b_0 + - decorator=5.1.0=pyhd3eb1b0_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - entrypoints=0.3=py39haa95532_0 + - fabio=0.12.0=py39h5d4886f_0 + - freetype=2.10.4=h546665d_1 + - glymur=0.9.4=pyhd8ed1ab_0 + - h5py=3.2.1=nompi_py39hf27771d_100 + - hdf5=1.10.6=nompi_h5268f04_1114 + - hdf5plugin=3.1.1=py39h71586dd_0 + - icc_rt=2019.0.0=h0cc432a_1 + - icu=68.1=h6c2663c_0 + - importlib-metadata=4.8.2=py39haa95532_0 + - importlib_metadata=4.8.2=hd3eb1b0_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intel-openmp=2021.3.0=haa95532_3372 + - ipykernel=6.4.1=py39haa95532_1 + - ipython=7.27.0=py39hd4e2768_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_1 + - jbig=2.1=h8d14728_2003 + - jedi=0.18.0=py39haa95532_1 + - jinja2=3.0.2=pyhd3eb1b0_0 + - jpeg=9d=h2bbff1b_0 + - jsonschema=3.2.0=pyhd3eb1b0_2 + - jupyter_client=7.0.1=pyhd3eb1b0_0 + - jupyter_core=4.8.1=py39haa95532_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 + - kiwisolver=1.3.2=py39h2e07f2f_0 + - krb5=1.19.2=hbae68bd_2 + - lerc=3.0=h0e60522_0 + - libclang=11.1.0=default_h5c34c98_1 + - libcurl=7.79.1=h789b8ee_1 + - libdeflate=1.8=h2bbff1b_5 + - libiconv=1.16=he774522_0 + - libpng=1.6.37=h1d00b33_2 + - libssh2=1.10.0=h680486a_2 + - libtiff=4.3.0=hd413186_2 + - libwebp=1.2.0=h2bbff1b_0 + - libxml2=2.9.12=h0ad7f3c_0 + - libxslt=1.1.34=he774522_0 + - libzlib=1.2.11=h8ffe710_1013 + - lxml=4.6.3=py39h4fd7cdf_0 + - lz4-c=1.9.3=h8ffe710_1 + - mako=1.1.5=pyhd8ed1ab_0 + - markupsafe=2.0.1=py39h2bbff1b_0 + - matplotlib=3.4.3=py39hcbf5309_1 + - matplotlib-base=3.4.3=py39h581301d_1 + - matplotlib-inline=0.1.2=pyhd3eb1b0_2 + - mistune=0.8.4=py39h2bbff1b_1000 + - mkl=2021.3.0=haa95532_524 + - mkl-service=2.4.0=py39h2bbff1b_0 + - mkl_fft=1.3.1=py39h277e83a_0 + - mkl_random=1.2.2=py39hf11a4ad_0 + - mpmath=1.2.1=py39haa95532_0 + - nbclient=0.5.11=pyhd3eb1b0_0 + - nbconvert=6.1.0=py39haa95532_0 + - nbformat=5.1.3=pyhd3eb1b0_0 + - nest-asyncio=1.5.1=pyhd3eb1b0_0 + - notebook=6.4.8=py39haa95532_0 + - numexpr=2.7.3=py39hb80d3ca_1 + - numpy=1.21.2=py39hfca59bb_0 + - numpy-base=1.21.2=py39h0829f74_0 + - olefile=0.46=pyh9f0ad1d_1 + - openjpeg=2.4.0=hb211442_1 + - openssl=1.1.1n=h2bbff1b_0 + - packaging=21.3=pyhd3eb1b0_0 + - palettable=3.3.0=pyhd3eb1b0_0 + - pandas=1.3.3=py39h6214cd6_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - parso=0.8.2=pyhd3eb1b0_0 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pillow=8.4.0=py39hd45dc43_0 + - pip=21.2.4=py39haa95532_0 + - pluggy=1.0.0=py39haa95532_1 + - prometheus_client=0.13.1=pyhd3eb1b0_0 + - prompt-toolkit=3.0.20=pyhd3eb1b0_0 + - py=1.11.0=pyhd3eb1b0_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyfai=0.20.0=hd8ed1ab_0 + - pyfai-base=0.20.0=py39h2e25243_0 + - pygments=2.10.0=pyhd3eb1b0_0 + - pyparsing=2.4.7=pyhd3eb1b0_0 + - pyqt=5.12.3=py39hcbf5309_7 + - pyqt-impl=5.12.3=py39h415ef7b_7 + - pyqt5-sip=4.19.18=py39h415ef7b_7 + - pyqtchart=5.12=py39h415ef7b_7 + - pyqtwebengine=5.12.1=py39h415ef7b_7 + - pyreadline=2.1=py39hcbf5309_1004 + - pyrsistent=0.18.0=py39h196d8e1_0 + - pytest=7.1.1=py39haa95532_0 + - python=3.9.7=h6244533_1 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python_abi=3.9=2_cp39 + - pytz=2021.3=pyhd3eb1b0_0 + - pywin32=228=py39hbaba5e8_1 + - pywinpty=2.0.2=py39h5da7b33_0 + - pyzmq=22.2.1=py39hd77b12b_1 + - qt=5.12.9=h5909a2a_4 + - qtconsole=5.1.1=pyhd3eb1b0_0 + - qtpy=1.11.2=pyhd8ed1ab_0 + - scipy=1.7.1=py39hbe87c03_2 + - seaborn=0.11.2=pyhd3eb1b0_0 + - send2trash=1.8.0=pyhd3eb1b0_1 + - setuptools=58.0.4=py39haa95532_0 + - silx=0.15.2=hd8ed1ab_0 + - silx-base=0.15.2=py39h2e25243_0 + - six=1.16.0=pyhd3eb1b0_0 + - sqlite=3.36.0=h2bbff1b_0 + - sympy=1.9=py39haa95532_0 + - terminado=0.13.1=py39haa95532_0 + - testpath=0.5.0=pyhd3eb1b0_0 + - tk=8.6.11=h8ffe710_1 + - tomli=1.2.2=pyhd3eb1b0_0 + - tornado=6.1=py39h2bbff1b_0 + - traitlets=5.1.0=pyhd3eb1b0_0 + - typing-extensions=3.10.0.2=hd3eb1b0_0 + - typing_extensions=3.10.0.2=pyh06a4308_0 + - tzdata=2021a=h5d7bf9c_0 + - vc=14.2=h21ff451_1 + - vs2015_runtime=14.27.29016=h5e58377_2 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py39haa95532_1 + - wheel=0.37.0=pyhd3eb1b0_1 + - widgetsnbextension=3.5.2=py39haa95532_0 + - wincertstore=0.2=py39haa95532_2 + - winpty=0.4.3=4 + - xz=5.2.5=h62dcd97_1 + - zipp=3.7.0=pyhd3eb1b0_0 + - zlib=1.2.11=h8ffe710_1013 + - zstd=1.5.0=h6255e5f_0 +prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime diff --git a/reqirements2.txt b/reqirements2.txt deleted file mode 100644 index 5cef354..0000000 --- a/reqirements2.txt +++ /dev/null @@ -1,141 +0,0 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: win-64 -argon2-cffi=21.3.0=pyhd3eb1b0_0 -argon2-cffi-bindings=21.2.0=py39h2bbff1b_0 -attrs=21.4.0=pyhd3eb1b0_0 -backcall=0.2.0=pyhd3eb1b0_0 -beamtime=0.1=pypi_0 -blas=1.0=mkl -bleach=4.1.0=pyhd3eb1b0_0 -bottleneck=1.3.2=py39h7cc1a96_1 -ca-certificates=2022.2.1=haa95532_0 -cached-property=1.5.2=hd8ed1ab_1 -cached_property=1.5.2=pyha770c72_1 -certifi=2021.10.8=py39haa95532_2 -cffi=1.15.0=py39h2bbff1b_1 -colorama=0.4.4=pyhd3eb1b0_0 -cycler=0.10.0=py_2 -debugpy=1.4.1=py39hd77b12b_0 -decorator=5.1.0=pyhd3eb1b0_0 -defusedxml=0.7.1=pyhd3eb1b0_0 -entrypoints=0.3=py39haa95532_0 -fabio=0.12.0=py39h5d4886f_0 -freetype=2.10.4=h546665d_1 -glymur=0.9.4=pyhd8ed1ab_0 -h5py=3.2.1=nompi_py39hf27771d_100 -hdf5=1.10.6=nompi_h5268f04_1114 -hdf5plugin=3.1.1=py39h71586dd_0 -icc_rt=2019.0.0=h0cc432a_1 -icu=68.1=h6c2663c_0 -importlib-metadata=4.8.2=py39haa95532_0 -importlib_metadata=4.8.2=hd3eb1b0_0 -intel-openmp=2021.3.0=haa95532_3372 -ipykernel=6.4.1=py39haa95532_1 -ipython=7.27.0=py39hd4e2768_0 -ipython_genutils=0.2.0=pyhd3eb1b0_1 -ipywidgets=7.6.5=pyhd3eb1b0_1 -jbig=2.1=h8d14728_2003 -jedi=0.18.0=py39haa95532_1 -jinja2=3.0.2=pyhd3eb1b0_0 -jpeg=9d=h2bbff1b_0 -jsonschema=3.2.0=pyhd3eb1b0_2 -jupyter_client=7.0.1=pyhd3eb1b0_0 -jupyter_core=4.8.1=py39haa95532_0 -jupyterlab_pygments=0.1.2=py_0 -jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 -kiwisolver=1.3.2=py39h2e07f2f_0 -krb5=1.19.2=hbae68bd_2 -lerc=3.0=h0e60522_0 -libclang=11.1.0=default_h5c34c98_1 -libcurl=7.79.1=h789b8ee_1 -libdeflate=1.8=h2bbff1b_5 -libiconv=1.16=he774522_0 -libpng=1.6.37=h1d00b33_2 -libssh2=1.10.0=h680486a_2 -libtiff=4.3.0=hd413186_2 -libwebp=1.2.0=h2bbff1b_0 -libxml2=2.9.12=h0ad7f3c_0 -libxslt=1.1.34=he774522_0 -libzlib=1.2.11=h8ffe710_1013 -lxml=4.6.3=py39h4fd7cdf_0 -lz4-c=1.9.3=h8ffe710_1 -mako=1.1.5=pyhd8ed1ab_0 -markupsafe=2.0.1=py39h2bbff1b_0 -matplotlib=3.4.3=py39hcbf5309_1 -matplotlib-base=3.4.3=py39h581301d_1 -matplotlib-inline=0.1.2=pyhd3eb1b0_2 -mistune=0.8.4=py39h2bbff1b_1000 -mkl=2021.3.0=haa95532_524 -mkl-service=2.4.0=py39h2bbff1b_0 -mkl_fft=1.3.1=py39h277e83a_0 -mkl_random=1.2.2=py39hf11a4ad_0 -nbclient=0.5.11=pyhd3eb1b0_0 -nbconvert=6.1.0=py39haa95532_0 -nbformat=5.1.3=pyhd3eb1b0_0 -nest-asyncio=1.5.1=pyhd3eb1b0_0 -notebook=6.4.8=py39haa95532_0 -numexpr=2.7.3=py39hb80d3ca_1 -numpy=1.21.2=py39hfca59bb_0 -numpy-base=1.21.2=py39h0829f74_0 -olefile=0.46=pyh9f0ad1d_1 -openjpeg=2.4.0=hb211442_1 -openssl=1.1.1m=h2bbff1b_0 -packaging=21.3=pyhd3eb1b0_0 -pandas=1.3.3=py39h6214cd6_0 -pandocfilters=1.5.0=pyhd3eb1b0_0 -parso=0.8.2=pyhd3eb1b0_0 -pickleshare=0.7.5=pyhd3eb1b0_1003 -pillow=8.4.0=py39hd45dc43_0 -pip=21.2.4=py39haa95532_0 -prometheus_client=0.13.1=pyhd3eb1b0_0 -prompt-toolkit=3.0.20=pyhd3eb1b0_0 -pycparser=2.21=pyhd3eb1b0_0 -pyfai=0.20.0=hd8ed1ab_0 -pyfai-base=0.20.0=py39h2e25243_0 -pygments=2.10.0=pyhd3eb1b0_0 -pyparsing=2.4.7=pyhd3eb1b0_0 -pyqt=5.12.3=py39hcbf5309_7 -pyqt-impl=5.12.3=py39h415ef7b_7 -pyqt5-sip=4.19.18=py39h415ef7b_7 -pyqtchart=5.12=py39h415ef7b_7 -pyqtwebengine=5.12.1=py39h415ef7b_7 -pyreadline=2.1=py39hcbf5309_1004 -pyrsistent=0.18.0=py39h196d8e1_0 -python=3.9.7=h6244533_1 -python-dateutil=2.8.2=pyhd3eb1b0_0 -python_abi=3.9=2_cp39 -pytz=2021.3=pyhd3eb1b0_0 -pywin32=228=py39hbaba5e8_1 -pywinpty=2.0.2=py39h5da7b33_0 -pyzmq=22.2.1=py39hd77b12b_1 -qt=5.12.9=h5909a2a_4 -qtconsole=5.1.1=pyhd3eb1b0_0 -qtpy=1.11.2=pyhd8ed1ab_0 -scipy=1.7.1=py39hbe87c03_2 -send2trash=1.8.0=pyhd3eb1b0_1 -setuptools=58.0.4=py39haa95532_0 -silx=0.15.2=hd8ed1ab_0 -silx-base=0.15.2=py39h2e25243_0 -six=1.16.0=pyhd3eb1b0_0 -sqlite=3.36.0=h2bbff1b_0 -terminado=0.13.1=py39haa95532_0 -testpath=0.5.0=pyhd3eb1b0_0 -tk=8.6.11=h8ffe710_1 -tornado=6.1=py39h2bbff1b_0 -traitlets=5.1.0=pyhd3eb1b0_0 -typing-extensions=3.10.0.2=hd3eb1b0_0 -typing_extensions=3.10.0.2=pyh06a4308_0 -tzdata=2021a=h5d7bf9c_0 -vc=14.2=h21ff451_1 -vs2015_runtime=14.27.29016=h5e58377_2 -wcwidth=0.2.5=pyhd3eb1b0_0 -webencodings=0.5.1=py39haa95532_1 -wheel=0.37.0=pyhd3eb1b0_1 -widgetsnbextension=3.5.2=py39haa95532_0 -wincertstore=0.2=py39haa95532_2 -winpty=0.4.3=4 -xz=5.2.5=h62dcd97_1 -zipp=3.7.0=pyhd3eb1b0_0 -zlib=1.2.11=h8ffe710_1013 -zstd=1.5.0=h6255e5f_0 diff --git a/requirements.txt b/requirements.txt index d8eeb75..8b428c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,108 +1,151 @@ # This file may be used to create an environment using: # $ conda create --name --file # platform: win-64 -@EXPLICIT -https://repo.anaconda.com/pkgs/main/win-64/blas-1.0-mkl.conda -https://conda.anaconda.org/conda-forge/win-64/ca-certificates-2021.10.8-h5b45459_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/icc_rt-2019.0.0-h0cc432a_1.conda -https://repo.anaconda.com/pkgs/main/win-64/intel-openmp-2021.3.0-haa95532_3372.conda -https://repo.anaconda.com/pkgs/main/noarch/tzdata-2021a-h5d7bf9c_0.conda -https://repo.anaconda.com/pkgs/main/win-64/vs2015_runtime-14.27.29016-h5e58377_2.conda -https://repo.anaconda.com/pkgs/main/win-64/mkl-2021.3.0-haa95532_524.conda -https://repo.anaconda.com/pkgs/main/win-64/vc-14.2-h21ff451_1.conda -https://repo.anaconda.com/pkgs/main/win-64/icu-68.1-h6c2663c_0.conda -https://conda.anaconda.org/conda-forge/win-64/jbig-2.1-h8d14728_2003.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/jpeg-9d-h2bbff1b_0.conda -https://conda.anaconda.org/conda-forge/win-64/lerc-3.0-h0e60522_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libclang-11.1.0-default_h5c34c98_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/libdeflate-1.8-h2bbff1b_5.conda -https://conda.anaconda.org/conda-forge/win-64/libiconv-1.16-he774522_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/libwebp-1.2.0-h2bbff1b_0.conda -https://conda.anaconda.org/conda-forge/win-64/libzlib-1.2.11-h8ffe710_1013.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/lz4-c-1.9.3-h8ffe710_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/openssl-1.1.1l-h8ffe710_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/sqlite-3.36.0-h2bbff1b_0.conda -https://conda.anaconda.org/conda-forge/win-64/tk-8.6.11-h8ffe710_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/xz-5.2.5-h62dcd97_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/krb5-1.19.2-hbae68bd_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/python-3.9.7-h6244533_1.conda -https://conda.anaconda.org/conda-forge/win-64/zlib-1.2.11-h8ffe710_1013.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/backcall-0.2.0-pyhd3eb1b0_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cached_property-1.5.2-pyha770c72_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/certifi-2021.10.8-py39haa95532_0.conda -https://repo.anaconda.com/pkgs/main/noarch/colorama-0.4.4-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/win-64/debugpy-1.4.1-py39hd77b12b_0.conda -https://repo.anaconda.com/pkgs/main/noarch/decorator-5.1.0-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/win-64/entrypoints-0.3-py39haa95532_0.conda -https://repo.anaconda.com/pkgs/main/noarch/ipython_genutils-0.2.0-pyhd3eb1b0_1.conda -https://conda.anaconda.org/conda-forge/win-64/libpng-1.6.37-h1d00b33_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libssh2-1.10.0-h680486a_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/libxml2-2.9.12-h0ad7f3c_0.conda -https://repo.anaconda.com/pkgs/main/win-64/markupsafe-2.0.1-py39h2bbff1b_0.conda -https://repo.anaconda.com/pkgs/main/noarch/nest-asyncio-1.5.1-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/noarch/olefile-0.46-pyh9f0ad1d_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/parso-0.8.2-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/noarch/pickleshare-0.7.5-pyhd3eb1b0_1003.conda -https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.7-pyhd3eb1b0_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/python_abi-3.9-2_cp39.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pytz-2021.3-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pywin32-228-py39hbaba5e8_1.conda -https://repo.anaconda.com/pkgs/main/win-64/pyzmq-22.2.1-py39hd77b12b_1.conda -https://conda.anaconda.org/conda-forge/noarch/qtpy-1.11.2-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/six-1.16.0-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/win-64/tornado-6.1-py39h2bbff1b_0.conda -https://repo.anaconda.com/pkgs/main/noarch/traitlets-5.1.0-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.2.5-pyhd3eb1b0_0.conda -https://repo.anaconda.com/pkgs/main/noarch/wheel-0.37.0-pyhd3eb1b0_1.conda -https://repo.anaconda.com/pkgs/main/win-64/wincertstore-0.2-py39haa95532_2.conda -https://conda.anaconda.org/conda-forge/win-64/zstd-1.5.0-h6255e5f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cached-property-1.5.2-hd8ed1ab_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cycler-0.10.0-py_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/freetype-2.10.4-h546665d_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/jedi-0.18.0-py39haa95532_1.conda -https://repo.anaconda.com/pkgs/main/win-64/jupyter_core-4.8.1-py39haa95532_0.conda -https://conda.anaconda.org/conda-forge/win-64/kiwisolver-1.3.2-py39h2e07f2f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libcurl-7.79.1-h789b8ee_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libtiff-4.3.0-hd413186_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/libxslt-1.1.34-he774522_0.conda -https://conda.anaconda.org/conda-forge/noarch/mako-1.1.5-pyhd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/matplotlib-inline-0.1.2-pyhd3eb1b0_2.conda -https://repo.anaconda.com/pkgs/main/win-64/mkl-service-2.4.0-py39h2bbff1b_0.conda -https://repo.anaconda.com/pkgs/main/noarch/prompt-toolkit-3.0.20-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/win-64/pyqt5-sip-4.19.18-py39h415ef7b_7.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/pyreadline-2.1-py39hcbf5309_1004.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/python-dateutil-2.8.2-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/win-64/qt-5.12.9-h5909a2a_4.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/setuptools-58.0.4-py39haa95532_0.conda -https://conda.anaconda.org/conda-forge/win-64/hdf5-1.10.6-nompi_h5268f04_1114.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jupyter_client-7.0.1-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/win-64/lxml-4.6.3-py39h4fd7cdf_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/numpy-base-1.21.2-py39h0829f74_0.conda -https://conda.anaconda.org/conda-forge/win-64/openjpeg-2.4.0-hb211442_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pillow-8.4.0-py39hd45dc43_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pip-21.2.4-py39haa95532_0.conda -https://repo.anaconda.com/pkgs/main/noarch/pygments-2.10.0-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/win-64/pyqt-impl-5.12.3-py39h415ef7b_7.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/ipython-7.27.0-py39hd4e2768_0.conda -https://conda.anaconda.org/conda-forge/win-64/pyqtchart-5.12-py39h415ef7b_7.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/pyqtwebengine-5.12.1-py39h415ef7b_7.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/ipykernel-6.4.1-py39haa95532_1.conda -https://conda.anaconda.org/conda-forge/win-64/pyqt-5.12.3-py39hcbf5309_7.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/qtconsole-5.1.1-pyhd3eb1b0_0.conda -https://conda.anaconda.org/conda-forge/noarch/glymur-0.9.4-pyhd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/fabio-0.12.0-py39h5d4886f_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/h5py-3.2.1-nompi_py39hf27771d_100.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/hdf5plugin-3.1.1-py39h71586dd_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/matplotlib-3.4.3-py39hcbf5309_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/matplotlib-base-3.4.3-py39h581301d_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/silx-base-0.15.2-py39h2e25243_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/silx-0.15.2-hd8ed1ab_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/pyfai-0.20.0-hd8ed1ab_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/bottleneck-1.3.2-py39h7cc1a96_1.conda -https://repo.anaconda.com/pkgs/main/win-64/mkl_fft-1.3.1-py39h277e83a_0.conda -https://repo.anaconda.com/pkgs/main/win-64/mkl_random-1.2.2-py39hf11a4ad_0.conda -https://repo.anaconda.com/pkgs/main/win-64/numpy-1.21.2-py39hfca59bb_0.conda -https://repo.anaconda.com/pkgs/main/win-64/numexpr-2.7.3-py39hb80d3ca_1.conda -https://repo.anaconda.com/pkgs/main/win-64/scipy-1.7.1-py39hbe87c03_2.conda -https://repo.anaconda.com/pkgs/main/win-64/pandas-1.3.3-py39h6214cd6_0.conda -https://conda.anaconda.org/conda-forge/win-64/pyfai-base-0.20.0-py39h2e25243_0.tar.bz2 +argon2-cffi=21.3.0=pyhd3eb1b0_0 +argon2-cffi-bindings=21.2.0=py39h2bbff1b_0 +atomicwrites=1.4.0=py_0 +attrs=21.4.0=pyhd3eb1b0_0 +backcall=0.2.0=pyhd3eb1b0_0 +beamtime=0.1=dev_0 +blas=1.0=mkl +bleach=4.1.0=pyhd3eb1b0_0 +bottleneck=1.3.2=py39h7cc1a96_1 +ca-certificates=2022.3.29=haa95532_0 +cached-property=1.5.2=hd8ed1ab_1 +cached_property=1.5.2=pyha770c72_1 +certifi=2021.10.8=py39haa95532_2 +cffi=1.15.0=py39h2bbff1b_1 +colorama=0.4.4=pyhd3eb1b0_0 +cycler=0.10.0=py_2 +debugpy=1.4.1=py39hd77b12b_0 +decorator=5.1.0=pyhd3eb1b0_0 +defusedxml=0.7.1=pyhd3eb1b0_0 +entrypoints=0.3=py39haa95532_0 +fabio=0.12.0=py39h5d4886f_0 +freetype=2.10.4=h546665d_1 +glymur=0.9.4=pyhd8ed1ab_0 +h5py=3.2.1=nompi_py39hf27771d_100 +hdf5=1.10.6=nompi_h5268f04_1114 +hdf5plugin=3.1.1=py39h71586dd_0 +icc_rt=2019.0.0=h0cc432a_1 +icu=68.1=h6c2663c_0 +importlib-metadata=4.8.2=py39haa95532_0 +importlib_metadata=4.8.2=hd3eb1b0_0 +iniconfig=1.1.1=pyhd3eb1b0_0 +intel-openmp=2021.3.0=haa95532_3372 +ipykernel=6.4.1=py39haa95532_1 +ipython=7.27.0=py39hd4e2768_0 +ipython_genutils=0.2.0=pyhd3eb1b0_1 +ipywidgets=7.6.5=pyhd3eb1b0_1 +jbig=2.1=h8d14728_2003 +jedi=0.18.0=py39haa95532_1 +jinja2=3.0.2=pyhd3eb1b0_0 +jpeg=9d=h2bbff1b_0 +jsonschema=3.2.0=pyhd3eb1b0_2 +jupyter_client=7.0.1=pyhd3eb1b0_0 +jupyter_core=4.8.1=py39haa95532_0 +jupyterlab_pygments=0.1.2=py_0 +jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 +kiwisolver=1.3.2=py39h2e07f2f_0 +krb5=1.19.2=hbae68bd_2 +lerc=3.0=h0e60522_0 +libclang=11.1.0=default_h5c34c98_1 +libcurl=7.79.1=h789b8ee_1 +libdeflate=1.8=h2bbff1b_5 +libiconv=1.16=he774522_0 +libpng=1.6.37=h1d00b33_2 +libssh2=1.10.0=h680486a_2 +libtiff=4.3.0=hd413186_2 +libwebp=1.2.0=h2bbff1b_0 +libxml2=2.9.12=h0ad7f3c_0 +libxslt=1.1.34=he774522_0 +libzlib=1.2.11=h8ffe710_1013 +lxml=4.6.3=py39h4fd7cdf_0 +lz4-c=1.9.3=h8ffe710_1 +mako=1.1.5=pyhd8ed1ab_0 +markupsafe=2.0.1=py39h2bbff1b_0 +matplotlib=3.4.3=py39hcbf5309_1 +matplotlib-base=3.4.3=py39h581301d_1 +matplotlib-inline=0.1.2=pyhd3eb1b0_2 +mistune=0.8.4=py39h2bbff1b_1000 +mkl=2021.3.0=haa95532_524 +mkl-service=2.4.0=py39h2bbff1b_0 +mkl_fft=1.3.1=py39h277e83a_0 +mkl_random=1.2.2=py39hf11a4ad_0 +mpmath=1.2.1=py39haa95532_0 +nbclient=0.5.11=pyhd3eb1b0_0 +nbconvert=6.1.0=py39haa95532_0 +nbformat=5.1.3=pyhd3eb1b0_0 +nest-asyncio=1.5.1=pyhd3eb1b0_0 +notebook=6.4.8=py39haa95532_0 +numexpr=2.7.3=py39hb80d3ca_1 +numpy=1.21.2=py39hfca59bb_0 +numpy-base=1.21.2=py39h0829f74_0 +olefile=0.46=pyh9f0ad1d_1 +openjpeg=2.4.0=hb211442_1 +openssl=1.1.1n=h2bbff1b_0 +packaging=21.3=pyhd3eb1b0_0 +palettable=3.3.0=pyhd3eb1b0_0 +pandas=1.3.3=py39h6214cd6_0 +pandocfilters=1.5.0=pyhd3eb1b0_0 +parso=0.8.2=pyhd3eb1b0_0 +pickleshare=0.7.5=pyhd3eb1b0_1003 +pillow=8.4.0=py39hd45dc43_0 +pip=21.2.4=py39haa95532_0 +pluggy=1.0.0=py39haa95532_1 +prometheus_client=0.13.1=pyhd3eb1b0_0 +prompt-toolkit=3.0.20=pyhd3eb1b0_0 +py=1.11.0=pyhd3eb1b0_0 +pycparser=2.21=pyhd3eb1b0_0 +pyfai=0.20.0=hd8ed1ab_0 +pyfai-base=0.20.0=py39h2e25243_0 +pygments=2.10.0=pyhd3eb1b0_0 +pyparsing=2.4.7=pyhd3eb1b0_0 +pyqt=5.12.3=py39hcbf5309_7 +pyqt-impl=5.12.3=py39h415ef7b_7 +pyqt5-sip=4.19.18=py39h415ef7b_7 +pyqtchart=5.12=py39h415ef7b_7 +pyqtwebengine=5.12.1=py39h415ef7b_7 +pyreadline=2.1=py39hcbf5309_1004 +pyrsistent=0.18.0=py39h196d8e1_0 +pytest=7.1.1=py39haa95532_0 +python=3.9.7=h6244533_1 +python-dateutil=2.8.2=pyhd3eb1b0_0 +python_abi=3.9=2_cp39 +pytz=2021.3=pyhd3eb1b0_0 +pywin32=228=py39hbaba5e8_1 +pywinpty=2.0.2=py39h5da7b33_0 +pyzmq=22.2.1=py39hd77b12b_1 +qt=5.12.9=h5909a2a_4 +qtconsole=5.1.1=pyhd3eb1b0_0 +qtpy=1.11.2=pyhd8ed1ab_0 +scipy=1.7.1=py39hbe87c03_2 +seaborn=0.11.2=pyhd3eb1b0_0 +send2trash=1.8.0=pyhd3eb1b0_1 +setuptools=58.0.4=py39haa95532_0 +silx=0.15.2=hd8ed1ab_0 +silx-base=0.15.2=py39h2e25243_0 +six=1.16.0=pyhd3eb1b0_0 +sqlite=3.36.0=h2bbff1b_0 +sympy=1.9=py39haa95532_0 +terminado=0.13.1=py39haa95532_0 +testpath=0.5.0=pyhd3eb1b0_0 +tk=8.6.11=h8ffe710_1 +tomli=1.2.2=pyhd3eb1b0_0 +tornado=6.1=py39h2bbff1b_0 +traitlets=5.1.0=pyhd3eb1b0_0 +typing-extensions=3.10.0.2=hd3eb1b0_0 +typing_extensions=3.10.0.2=pyh06a4308_0 +tzdata=2021a=h5d7bf9c_0 +vc=14.2=h21ff451_1 +vs2015_runtime=14.27.29016=h5e58377_2 +wcwidth=0.2.5=pyhd3eb1b0_0 +webencodings=0.5.1=py39haa95532_1 +wheel=0.37.0=pyhd3eb1b0_1 +widgetsnbextension=3.5.2=py39haa95532_0 +wincertstore=0.2=py39haa95532_2 +winpty=0.4.3=4 +xz=5.2.5=h62dcd97_1 +zipp=3.7.0=pyhd3eb1b0_0 +zlib=1.2.11=h8ffe710_1013 +zstd=1.5.0=h6255e5f_0 From b5f4f98070dd30292f98b06f056d34a76526454d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:22:28 +0200 Subject: [PATCH 127/355] Delete first attempt at automated testing --- .github/workflows/python-app.yml | 39 -------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 .github/workflows/python-app.yml diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml deleted file mode 100644 index 2e8690d..0000000 --- a/.github/workflows/python-app.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python application - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -permissions: - contents: read - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest From 739b197e9aca699af70dd5cb984417d155b1071f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:23:03 +0200 Subject: [PATCH 128/355] Set up automated testing with conda --- .github/workflows/python-package-conda.yml | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml new file mode 100644 index 0000000..57940bd --- /dev/null +++ b/.github/workflows/python-package-conda.yml @@ -0,0 +1,34 @@ +name: Python Package using Conda + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: 3.10 + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + - name: Lint with flake8 + run: | + conda install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + conda install pytest + pytest From c9885976f6d9a71b8ca6ae79e7142e0197a0ac01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:24:57 +0200 Subject: [PATCH 129/355] Change python version in automated testing --- .github/workflows/python-package-conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 57940bd..6261945 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.9 uses: actions/setup-python@v3 with: - python-version: 3.10 + python-version: 3.9.7 - name: Add conda to system path run: | # $CONDA is an environment variable pointing to the root of the miniconda directory From c10317c1d32df92724ffd802b6cdc7e50f9e3143 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 14:35:41 +0200 Subject: [PATCH 130/355] Update requirements with --no-build tag --- requirements.txt | 305 ++++++++++++++++++++++++----------------------- 1 file changed, 154 insertions(+), 151 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8b428c5..e7bc821 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,151 +1,154 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: win-64 -argon2-cffi=21.3.0=pyhd3eb1b0_0 -argon2-cffi-bindings=21.2.0=py39h2bbff1b_0 -atomicwrites=1.4.0=py_0 -attrs=21.4.0=pyhd3eb1b0_0 -backcall=0.2.0=pyhd3eb1b0_0 -beamtime=0.1=dev_0 -blas=1.0=mkl -bleach=4.1.0=pyhd3eb1b0_0 -bottleneck=1.3.2=py39h7cc1a96_1 -ca-certificates=2022.3.29=haa95532_0 -cached-property=1.5.2=hd8ed1ab_1 -cached_property=1.5.2=pyha770c72_1 -certifi=2021.10.8=py39haa95532_2 -cffi=1.15.0=py39h2bbff1b_1 -colorama=0.4.4=pyhd3eb1b0_0 -cycler=0.10.0=py_2 -debugpy=1.4.1=py39hd77b12b_0 -decorator=5.1.0=pyhd3eb1b0_0 -defusedxml=0.7.1=pyhd3eb1b0_0 -entrypoints=0.3=py39haa95532_0 -fabio=0.12.0=py39h5d4886f_0 -freetype=2.10.4=h546665d_1 -glymur=0.9.4=pyhd8ed1ab_0 -h5py=3.2.1=nompi_py39hf27771d_100 -hdf5=1.10.6=nompi_h5268f04_1114 -hdf5plugin=3.1.1=py39h71586dd_0 -icc_rt=2019.0.0=h0cc432a_1 -icu=68.1=h6c2663c_0 -importlib-metadata=4.8.2=py39haa95532_0 -importlib_metadata=4.8.2=hd3eb1b0_0 -iniconfig=1.1.1=pyhd3eb1b0_0 -intel-openmp=2021.3.0=haa95532_3372 -ipykernel=6.4.1=py39haa95532_1 -ipython=7.27.0=py39hd4e2768_0 -ipython_genutils=0.2.0=pyhd3eb1b0_1 -ipywidgets=7.6.5=pyhd3eb1b0_1 -jbig=2.1=h8d14728_2003 -jedi=0.18.0=py39haa95532_1 -jinja2=3.0.2=pyhd3eb1b0_0 -jpeg=9d=h2bbff1b_0 -jsonschema=3.2.0=pyhd3eb1b0_2 -jupyter_client=7.0.1=pyhd3eb1b0_0 -jupyter_core=4.8.1=py39haa95532_0 -jupyterlab_pygments=0.1.2=py_0 -jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 -kiwisolver=1.3.2=py39h2e07f2f_0 -krb5=1.19.2=hbae68bd_2 -lerc=3.0=h0e60522_0 -libclang=11.1.0=default_h5c34c98_1 -libcurl=7.79.1=h789b8ee_1 -libdeflate=1.8=h2bbff1b_5 -libiconv=1.16=he774522_0 -libpng=1.6.37=h1d00b33_2 -libssh2=1.10.0=h680486a_2 -libtiff=4.3.0=hd413186_2 -libwebp=1.2.0=h2bbff1b_0 -libxml2=2.9.12=h0ad7f3c_0 -libxslt=1.1.34=he774522_0 -libzlib=1.2.11=h8ffe710_1013 -lxml=4.6.3=py39h4fd7cdf_0 -lz4-c=1.9.3=h8ffe710_1 -mako=1.1.5=pyhd8ed1ab_0 -markupsafe=2.0.1=py39h2bbff1b_0 -matplotlib=3.4.3=py39hcbf5309_1 -matplotlib-base=3.4.3=py39h581301d_1 -matplotlib-inline=0.1.2=pyhd3eb1b0_2 -mistune=0.8.4=py39h2bbff1b_1000 -mkl=2021.3.0=haa95532_524 -mkl-service=2.4.0=py39h2bbff1b_0 -mkl_fft=1.3.1=py39h277e83a_0 -mkl_random=1.2.2=py39hf11a4ad_0 -mpmath=1.2.1=py39haa95532_0 -nbclient=0.5.11=pyhd3eb1b0_0 -nbconvert=6.1.0=py39haa95532_0 -nbformat=5.1.3=pyhd3eb1b0_0 -nest-asyncio=1.5.1=pyhd3eb1b0_0 -notebook=6.4.8=py39haa95532_0 -numexpr=2.7.3=py39hb80d3ca_1 -numpy=1.21.2=py39hfca59bb_0 -numpy-base=1.21.2=py39h0829f74_0 -olefile=0.46=pyh9f0ad1d_1 -openjpeg=2.4.0=hb211442_1 -openssl=1.1.1n=h2bbff1b_0 -packaging=21.3=pyhd3eb1b0_0 -palettable=3.3.0=pyhd3eb1b0_0 -pandas=1.3.3=py39h6214cd6_0 -pandocfilters=1.5.0=pyhd3eb1b0_0 -parso=0.8.2=pyhd3eb1b0_0 -pickleshare=0.7.5=pyhd3eb1b0_1003 -pillow=8.4.0=py39hd45dc43_0 -pip=21.2.4=py39haa95532_0 -pluggy=1.0.0=py39haa95532_1 -prometheus_client=0.13.1=pyhd3eb1b0_0 -prompt-toolkit=3.0.20=pyhd3eb1b0_0 -py=1.11.0=pyhd3eb1b0_0 -pycparser=2.21=pyhd3eb1b0_0 -pyfai=0.20.0=hd8ed1ab_0 -pyfai-base=0.20.0=py39h2e25243_0 -pygments=2.10.0=pyhd3eb1b0_0 -pyparsing=2.4.7=pyhd3eb1b0_0 -pyqt=5.12.3=py39hcbf5309_7 -pyqt-impl=5.12.3=py39h415ef7b_7 -pyqt5-sip=4.19.18=py39h415ef7b_7 -pyqtchart=5.12=py39h415ef7b_7 -pyqtwebengine=5.12.1=py39h415ef7b_7 -pyreadline=2.1=py39hcbf5309_1004 -pyrsistent=0.18.0=py39h196d8e1_0 -pytest=7.1.1=py39haa95532_0 -python=3.9.7=h6244533_1 -python-dateutil=2.8.2=pyhd3eb1b0_0 -python_abi=3.9=2_cp39 -pytz=2021.3=pyhd3eb1b0_0 -pywin32=228=py39hbaba5e8_1 -pywinpty=2.0.2=py39h5da7b33_0 -pyzmq=22.2.1=py39hd77b12b_1 -qt=5.12.9=h5909a2a_4 -qtconsole=5.1.1=pyhd3eb1b0_0 -qtpy=1.11.2=pyhd8ed1ab_0 -scipy=1.7.1=py39hbe87c03_2 -seaborn=0.11.2=pyhd3eb1b0_0 -send2trash=1.8.0=pyhd3eb1b0_1 -setuptools=58.0.4=py39haa95532_0 -silx=0.15.2=hd8ed1ab_0 -silx-base=0.15.2=py39h2e25243_0 -six=1.16.0=pyhd3eb1b0_0 -sqlite=3.36.0=h2bbff1b_0 -sympy=1.9=py39haa95532_0 -terminado=0.13.1=py39haa95532_0 -testpath=0.5.0=pyhd3eb1b0_0 -tk=8.6.11=h8ffe710_1 -tomli=1.2.2=pyhd3eb1b0_0 -tornado=6.1=py39h2bbff1b_0 -traitlets=5.1.0=pyhd3eb1b0_0 -typing-extensions=3.10.0.2=hd3eb1b0_0 -typing_extensions=3.10.0.2=pyh06a4308_0 -tzdata=2021a=h5d7bf9c_0 -vc=14.2=h21ff451_1 -vs2015_runtime=14.27.29016=h5e58377_2 -wcwidth=0.2.5=pyhd3eb1b0_0 -webencodings=0.5.1=py39haa95532_1 -wheel=0.37.0=pyhd3eb1b0_1 -widgetsnbextension=3.5.2=py39haa95532_0 -wincertstore=0.2=py39haa95532_2 -winpty=0.4.3=4 -xz=5.2.5=h62dcd97_1 -zipp=3.7.0=pyhd3eb1b0_0 -zlib=1.2.11=h8ffe710_1013 -zstd=1.5.0=h6255e5f_0 +name: beamtime +channels: + - conda-forge + - diffpy + - defaults +dependencies: + - argon2-cffi=21.3.0 + - argon2-cffi-bindings=21.2.0 + - atomicwrites=1.4.0 + - attrs=21.4.0 + - backcall=0.2.0 + - blas=1.0 + - bleach=4.1.0 + - bottleneck=1.3.2 + - ca-certificates=2022.3.29 + - cached-property=1.5.2 + - cached_property=1.5.2 + - certifi=2021.10.8 + - cffi=1.15.0 + - colorama=0.4.4 + - cycler=0.10.0 + - debugpy=1.4.1 + - decorator=5.1.0 + - defusedxml=0.7.1 + - entrypoints=0.3 + - fabio=0.12.0 + - freetype=2.10.4 + - glymur=0.9.4 + - h5py=3.2.1 + - hdf5=1.10.6 + - hdf5plugin=3.1.1 + - icc_rt=2019.0.0 + - icu=68.1 + - importlib-metadata=4.8.2 + - importlib_metadata=4.8.2 + - iniconfig=1.1.1 + - intel-openmp=2021.3.0 + - ipykernel=6.4.1 + - ipython=7.27.0 + - ipython_genutils=0.2.0 + - ipywidgets=7.6.5 + - jbig=2.1 + - jedi=0.18.0 + - jinja2=3.0.2 + - jpeg=9d + - jsonschema=3.2.0 + - jupyter_client=7.0.1 + - jupyter_core=4.8.1 + - jupyterlab_pygments=0.1.2 + - jupyterlab_widgets=1.0.0 + - kiwisolver=1.3.2 + - krb5=1.19.2 + - lerc=3.0 + - libclang=11.1.0 + - libcurl=7.79.1 + - libdeflate=1.8 + - libiconv=1.16 + - libpng=1.6.37 + - libssh2=1.10.0 + - libtiff=4.3.0 + - libwebp=1.2.0 + - libxml2=2.9.12 + - libxslt=1.1.34 + - libzlib=1.2.11 + - lxml=4.6.3 + - lz4-c=1.9.3 + - mako=1.1.5 + - markupsafe=2.0.1 + - matplotlib=3.4.3 + - matplotlib-base=3.4.3 + - matplotlib-inline=0.1.2 + - mistune=0.8.4 + - mkl=2021.3.0 + - mkl-service=2.4.0 + - mkl_fft=1.3.1 + - mkl_random=1.2.2 + - mpmath=1.2.1 + - nbclient=0.5.11 + - nbconvert=6.1.0 + - nbformat=5.1.3 + - nest-asyncio=1.5.1 + - notebook=6.4.8 + - numexpr=2.7.3 + - numpy=1.21.2 + - numpy-base=1.21.2 + - olefile=0.46 + - openjpeg=2.4.0 + - openssl=1.1.1n + - packaging=21.3 + - palettable=3.3.0 + - pandas=1.3.3 + - pandocfilters=1.5.0 + - parso=0.8.2 + - pickleshare=0.7.5 + - pillow=8.4.0 + - pip=21.2.4 + - pluggy=1.0.0 + - prometheus_client=0.13.1 + - prompt-toolkit=3.0.20 + - py=1.11.0 + - pycparser=2.21 + - pyfai=0.20.0 + - pyfai-base=0.20.0 + - pygments=2.10.0 + - pyparsing=2.4.7 + - pyqt=5.12.3 + - pyqt-impl=5.12.3 + - pyqt5-sip=4.19.18 + - pyqtchart=5.12 + - pyqtwebengine=5.12.1 + - pyreadline=2.1 + - pyrsistent=0.18.0 + - pytest=7.1.1 + - python=3.9.7 + - python-dateutil=2.8.2 + - python_abi=3.9 + - pytz=2021.3 + - pywin32=228 + - pywinpty=2.0.2 + - pyzmq=22.2.1 + - qt=5.12.9 + - qtconsole=5.1.1 + - qtpy=1.11.2 + - scipy=1.7.1 + - seaborn=0.11.2 + - send2trash=1.8.0 + - setuptools=58.0.4 + - silx=0.15.2 + - silx-base=0.15.2 + - six=1.16.0 + - sqlite=3.36.0 + - sympy=1.9 + - terminado=0.13.1 + - testpath=0.5.0 + - tk=8.6.11 + - tomli=1.2.2 + - tornado=6.1 + - traitlets=5.1.0 + - typing-extensions=3.10.0.2 + - typing_extensions=3.10.0.2 + - tzdata=2021a + - vc=14.2 + - vs2015_runtime=14.27.29016 + - wcwidth=0.2.5 + - webencodings=0.5.1 + - wheel=0.37.0 + - widgetsnbextension=3.5.2 + - wincertstore=0.2 + - winpty=0.4.3 + - xz=5.2.5 + - zipp=3.7.0 + - zlib=1.2.11 + - zstd=1.5.0 +prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime From e8bc4d4bc4f9615ee4f27cd7e83ee0ce41226031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:39:03 +0200 Subject: [PATCH 131/355] Delete second attempt at setting up automated testing --- .github/workflows/python-package-conda.yml | 34 ---------------------- 1 file changed, 34 deletions(-) delete mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml deleted file mode 100644 index 6261945..0000000 --- a/.github/workflows/python-package-conda.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Python Package using Conda - -on: [push] - -jobs: - build-linux: - runs-on: ubuntu-latest - strategy: - max-parallel: 5 - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 - with: - python-version: 3.9.7 - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - - name: Install dependencies - run: | - conda env update --file environment.yml --name base - - name: Lint with flake8 - run: | - conda install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - conda install pytest - pytest From c8f1f64af86d8c3060616b2e136921c955b562e6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 14:40:38 +0200 Subject: [PATCH 132/355] Update environment.yml without build --- environment.yml | 294 +++++++++++++++++++++++------------------------ requirements.txt | 154 ------------------------- 2 files changed, 147 insertions(+), 301 deletions(-) delete mode 100644 requirements.txt diff --git a/environment.yml b/environment.yml index b80378b..e7bc821 100644 --- a/environment.yml +++ b/environment.yml @@ -4,151 +4,151 @@ channels: - diffpy - defaults dependencies: - - argon2-cffi=21.3.0=pyhd3eb1b0_0 - - argon2-cffi-bindings=21.2.0=py39h2bbff1b_0 - - atomicwrites=1.4.0=py_0 - - attrs=21.4.0=pyhd3eb1b0_0 - - backcall=0.2.0=pyhd3eb1b0_0 - - blas=1.0=mkl - - bleach=4.1.0=pyhd3eb1b0_0 - - bottleneck=1.3.2=py39h7cc1a96_1 - - ca-certificates=2022.3.29=haa95532_0 - - cached-property=1.5.2=hd8ed1ab_1 - - cached_property=1.5.2=pyha770c72_1 - - certifi=2021.10.8=py39haa95532_2 - - cffi=1.15.0=py39h2bbff1b_1 - - colorama=0.4.4=pyhd3eb1b0_0 - - cycler=0.10.0=py_2 - - debugpy=1.4.1=py39hd77b12b_0 - - decorator=5.1.0=pyhd3eb1b0_0 - - defusedxml=0.7.1=pyhd3eb1b0_0 - - entrypoints=0.3=py39haa95532_0 - - fabio=0.12.0=py39h5d4886f_0 - - freetype=2.10.4=h546665d_1 - - glymur=0.9.4=pyhd8ed1ab_0 - - h5py=3.2.1=nompi_py39hf27771d_100 - - hdf5=1.10.6=nompi_h5268f04_1114 - - hdf5plugin=3.1.1=py39h71586dd_0 - - icc_rt=2019.0.0=h0cc432a_1 - - icu=68.1=h6c2663c_0 - - importlib-metadata=4.8.2=py39haa95532_0 - - importlib_metadata=4.8.2=hd3eb1b0_0 - - iniconfig=1.1.1=pyhd3eb1b0_0 - - intel-openmp=2021.3.0=haa95532_3372 - - ipykernel=6.4.1=py39haa95532_1 - - ipython=7.27.0=py39hd4e2768_0 - - ipython_genutils=0.2.0=pyhd3eb1b0_1 - - ipywidgets=7.6.5=pyhd3eb1b0_1 - - jbig=2.1=h8d14728_2003 - - jedi=0.18.0=py39haa95532_1 - - jinja2=3.0.2=pyhd3eb1b0_0 - - jpeg=9d=h2bbff1b_0 - - jsonschema=3.2.0=pyhd3eb1b0_2 - - jupyter_client=7.0.1=pyhd3eb1b0_0 - - jupyter_core=4.8.1=py39haa95532_0 - - jupyterlab_pygments=0.1.2=py_0 - - jupyterlab_widgets=1.0.0=pyhd3eb1b0_1 - - kiwisolver=1.3.2=py39h2e07f2f_0 - - krb5=1.19.2=hbae68bd_2 - - lerc=3.0=h0e60522_0 - - libclang=11.1.0=default_h5c34c98_1 - - libcurl=7.79.1=h789b8ee_1 - - libdeflate=1.8=h2bbff1b_5 - - libiconv=1.16=he774522_0 - - libpng=1.6.37=h1d00b33_2 - - libssh2=1.10.0=h680486a_2 - - libtiff=4.3.0=hd413186_2 - - libwebp=1.2.0=h2bbff1b_0 - - libxml2=2.9.12=h0ad7f3c_0 - - libxslt=1.1.34=he774522_0 - - libzlib=1.2.11=h8ffe710_1013 - - lxml=4.6.3=py39h4fd7cdf_0 - - lz4-c=1.9.3=h8ffe710_1 - - mako=1.1.5=pyhd8ed1ab_0 - - markupsafe=2.0.1=py39h2bbff1b_0 - - matplotlib=3.4.3=py39hcbf5309_1 - - matplotlib-base=3.4.3=py39h581301d_1 - - matplotlib-inline=0.1.2=pyhd3eb1b0_2 - - mistune=0.8.4=py39h2bbff1b_1000 - - mkl=2021.3.0=haa95532_524 - - mkl-service=2.4.0=py39h2bbff1b_0 - - mkl_fft=1.3.1=py39h277e83a_0 - - mkl_random=1.2.2=py39hf11a4ad_0 - - mpmath=1.2.1=py39haa95532_0 - - nbclient=0.5.11=pyhd3eb1b0_0 - - nbconvert=6.1.0=py39haa95532_0 - - nbformat=5.1.3=pyhd3eb1b0_0 - - nest-asyncio=1.5.1=pyhd3eb1b0_0 - - notebook=6.4.8=py39haa95532_0 - - numexpr=2.7.3=py39hb80d3ca_1 - - numpy=1.21.2=py39hfca59bb_0 - - numpy-base=1.21.2=py39h0829f74_0 - - olefile=0.46=pyh9f0ad1d_1 - - openjpeg=2.4.0=hb211442_1 - - openssl=1.1.1n=h2bbff1b_0 - - packaging=21.3=pyhd3eb1b0_0 - - palettable=3.3.0=pyhd3eb1b0_0 - - pandas=1.3.3=py39h6214cd6_0 - - pandocfilters=1.5.0=pyhd3eb1b0_0 - - parso=0.8.2=pyhd3eb1b0_0 - - pickleshare=0.7.5=pyhd3eb1b0_1003 - - pillow=8.4.0=py39hd45dc43_0 - - pip=21.2.4=py39haa95532_0 - - pluggy=1.0.0=py39haa95532_1 - - prometheus_client=0.13.1=pyhd3eb1b0_0 - - prompt-toolkit=3.0.20=pyhd3eb1b0_0 - - py=1.11.0=pyhd3eb1b0_0 - - pycparser=2.21=pyhd3eb1b0_0 - - pyfai=0.20.0=hd8ed1ab_0 - - pyfai-base=0.20.0=py39h2e25243_0 - - pygments=2.10.0=pyhd3eb1b0_0 - - pyparsing=2.4.7=pyhd3eb1b0_0 - - pyqt=5.12.3=py39hcbf5309_7 - - pyqt-impl=5.12.3=py39h415ef7b_7 - - pyqt5-sip=4.19.18=py39h415ef7b_7 - - pyqtchart=5.12=py39h415ef7b_7 - - pyqtwebengine=5.12.1=py39h415ef7b_7 - - pyreadline=2.1=py39hcbf5309_1004 - - pyrsistent=0.18.0=py39h196d8e1_0 - - pytest=7.1.1=py39haa95532_0 - - python=3.9.7=h6244533_1 - - python-dateutil=2.8.2=pyhd3eb1b0_0 - - python_abi=3.9=2_cp39 - - pytz=2021.3=pyhd3eb1b0_0 - - pywin32=228=py39hbaba5e8_1 - - pywinpty=2.0.2=py39h5da7b33_0 - - pyzmq=22.2.1=py39hd77b12b_1 - - qt=5.12.9=h5909a2a_4 - - qtconsole=5.1.1=pyhd3eb1b0_0 - - qtpy=1.11.2=pyhd8ed1ab_0 - - scipy=1.7.1=py39hbe87c03_2 - - seaborn=0.11.2=pyhd3eb1b0_0 - - send2trash=1.8.0=pyhd3eb1b0_1 - - setuptools=58.0.4=py39haa95532_0 - - silx=0.15.2=hd8ed1ab_0 - - silx-base=0.15.2=py39h2e25243_0 - - six=1.16.0=pyhd3eb1b0_0 - - sqlite=3.36.0=h2bbff1b_0 - - sympy=1.9=py39haa95532_0 - - terminado=0.13.1=py39haa95532_0 - - testpath=0.5.0=pyhd3eb1b0_0 - - tk=8.6.11=h8ffe710_1 - - tomli=1.2.2=pyhd3eb1b0_0 - - tornado=6.1=py39h2bbff1b_0 - - traitlets=5.1.0=pyhd3eb1b0_0 - - typing-extensions=3.10.0.2=hd3eb1b0_0 - - typing_extensions=3.10.0.2=pyh06a4308_0 - - tzdata=2021a=h5d7bf9c_0 - - vc=14.2=h21ff451_1 - - vs2015_runtime=14.27.29016=h5e58377_2 - - wcwidth=0.2.5=pyhd3eb1b0_0 - - webencodings=0.5.1=py39haa95532_1 - - wheel=0.37.0=pyhd3eb1b0_1 - - widgetsnbextension=3.5.2=py39haa95532_0 - - wincertstore=0.2=py39haa95532_2 - - winpty=0.4.3=4 - - xz=5.2.5=h62dcd97_1 - - zipp=3.7.0=pyhd3eb1b0_0 - - zlib=1.2.11=h8ffe710_1013 - - zstd=1.5.0=h6255e5f_0 + - argon2-cffi=21.3.0 + - argon2-cffi-bindings=21.2.0 + - atomicwrites=1.4.0 + - attrs=21.4.0 + - backcall=0.2.0 + - blas=1.0 + - bleach=4.1.0 + - bottleneck=1.3.2 + - ca-certificates=2022.3.29 + - cached-property=1.5.2 + - cached_property=1.5.2 + - certifi=2021.10.8 + - cffi=1.15.0 + - colorama=0.4.4 + - cycler=0.10.0 + - debugpy=1.4.1 + - decorator=5.1.0 + - defusedxml=0.7.1 + - entrypoints=0.3 + - fabio=0.12.0 + - freetype=2.10.4 + - glymur=0.9.4 + - h5py=3.2.1 + - hdf5=1.10.6 + - hdf5plugin=3.1.1 + - icc_rt=2019.0.0 + - icu=68.1 + - importlib-metadata=4.8.2 + - importlib_metadata=4.8.2 + - iniconfig=1.1.1 + - intel-openmp=2021.3.0 + - ipykernel=6.4.1 + - ipython=7.27.0 + - ipython_genutils=0.2.0 + - ipywidgets=7.6.5 + - jbig=2.1 + - jedi=0.18.0 + - jinja2=3.0.2 + - jpeg=9d + - jsonschema=3.2.0 + - jupyter_client=7.0.1 + - jupyter_core=4.8.1 + - jupyterlab_pygments=0.1.2 + - jupyterlab_widgets=1.0.0 + - kiwisolver=1.3.2 + - krb5=1.19.2 + - lerc=3.0 + - libclang=11.1.0 + - libcurl=7.79.1 + - libdeflate=1.8 + - libiconv=1.16 + - libpng=1.6.37 + - libssh2=1.10.0 + - libtiff=4.3.0 + - libwebp=1.2.0 + - libxml2=2.9.12 + - libxslt=1.1.34 + - libzlib=1.2.11 + - lxml=4.6.3 + - lz4-c=1.9.3 + - mako=1.1.5 + - markupsafe=2.0.1 + - matplotlib=3.4.3 + - matplotlib-base=3.4.3 + - matplotlib-inline=0.1.2 + - mistune=0.8.4 + - mkl=2021.3.0 + - mkl-service=2.4.0 + - mkl_fft=1.3.1 + - mkl_random=1.2.2 + - mpmath=1.2.1 + - nbclient=0.5.11 + - nbconvert=6.1.0 + - nbformat=5.1.3 + - nest-asyncio=1.5.1 + - notebook=6.4.8 + - numexpr=2.7.3 + - numpy=1.21.2 + - numpy-base=1.21.2 + - olefile=0.46 + - openjpeg=2.4.0 + - openssl=1.1.1n + - packaging=21.3 + - palettable=3.3.0 + - pandas=1.3.3 + - pandocfilters=1.5.0 + - parso=0.8.2 + - pickleshare=0.7.5 + - pillow=8.4.0 + - pip=21.2.4 + - pluggy=1.0.0 + - prometheus_client=0.13.1 + - prompt-toolkit=3.0.20 + - py=1.11.0 + - pycparser=2.21 + - pyfai=0.20.0 + - pyfai-base=0.20.0 + - pygments=2.10.0 + - pyparsing=2.4.7 + - pyqt=5.12.3 + - pyqt-impl=5.12.3 + - pyqt5-sip=4.19.18 + - pyqtchart=5.12 + - pyqtwebengine=5.12.1 + - pyreadline=2.1 + - pyrsistent=0.18.0 + - pytest=7.1.1 + - python=3.9.7 + - python-dateutil=2.8.2 + - python_abi=3.9 + - pytz=2021.3 + - pywin32=228 + - pywinpty=2.0.2 + - pyzmq=22.2.1 + - qt=5.12.9 + - qtconsole=5.1.1 + - qtpy=1.11.2 + - scipy=1.7.1 + - seaborn=0.11.2 + - send2trash=1.8.0 + - setuptools=58.0.4 + - silx=0.15.2 + - silx-base=0.15.2 + - six=1.16.0 + - sqlite=3.36.0 + - sympy=1.9 + - terminado=0.13.1 + - testpath=0.5.0 + - tk=8.6.11 + - tomli=1.2.2 + - tornado=6.1 + - traitlets=5.1.0 + - typing-extensions=3.10.0.2 + - typing_extensions=3.10.0.2 + - tzdata=2021a + - vc=14.2 + - vs2015_runtime=14.27.29016 + - wcwidth=0.2.5 + - webencodings=0.5.1 + - wheel=0.37.0 + - widgetsnbextension=3.5.2 + - wincertstore=0.2 + - winpty=0.4.3 + - xz=5.2.5 + - zipp=3.7.0 + - zlib=1.2.11 + - zstd=1.5.0 prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index e7bc821..0000000 --- a/requirements.txt +++ /dev/null @@ -1,154 +0,0 @@ -name: beamtime -channels: - - conda-forge - - diffpy - - defaults -dependencies: - - argon2-cffi=21.3.0 - - argon2-cffi-bindings=21.2.0 - - atomicwrites=1.4.0 - - attrs=21.4.0 - - backcall=0.2.0 - - blas=1.0 - - bleach=4.1.0 - - bottleneck=1.3.2 - - ca-certificates=2022.3.29 - - cached-property=1.5.2 - - cached_property=1.5.2 - - certifi=2021.10.8 - - cffi=1.15.0 - - colorama=0.4.4 - - cycler=0.10.0 - - debugpy=1.4.1 - - decorator=5.1.0 - - defusedxml=0.7.1 - - entrypoints=0.3 - - fabio=0.12.0 - - freetype=2.10.4 - - glymur=0.9.4 - - h5py=3.2.1 - - hdf5=1.10.6 - - hdf5plugin=3.1.1 - - icc_rt=2019.0.0 - - icu=68.1 - - importlib-metadata=4.8.2 - - importlib_metadata=4.8.2 - - iniconfig=1.1.1 - - intel-openmp=2021.3.0 - - ipykernel=6.4.1 - - ipython=7.27.0 - - ipython_genutils=0.2.0 - - ipywidgets=7.6.5 - - jbig=2.1 - - jedi=0.18.0 - - jinja2=3.0.2 - - jpeg=9d - - jsonschema=3.2.0 - - jupyter_client=7.0.1 - - jupyter_core=4.8.1 - - jupyterlab_pygments=0.1.2 - - jupyterlab_widgets=1.0.0 - - kiwisolver=1.3.2 - - krb5=1.19.2 - - lerc=3.0 - - libclang=11.1.0 - - libcurl=7.79.1 - - libdeflate=1.8 - - libiconv=1.16 - - libpng=1.6.37 - - libssh2=1.10.0 - - libtiff=4.3.0 - - libwebp=1.2.0 - - libxml2=2.9.12 - - libxslt=1.1.34 - - libzlib=1.2.11 - - lxml=4.6.3 - - lz4-c=1.9.3 - - mako=1.1.5 - - markupsafe=2.0.1 - - matplotlib=3.4.3 - - matplotlib-base=3.4.3 - - matplotlib-inline=0.1.2 - - mistune=0.8.4 - - mkl=2021.3.0 - - mkl-service=2.4.0 - - mkl_fft=1.3.1 - - mkl_random=1.2.2 - - mpmath=1.2.1 - - nbclient=0.5.11 - - nbconvert=6.1.0 - - nbformat=5.1.3 - - nest-asyncio=1.5.1 - - notebook=6.4.8 - - numexpr=2.7.3 - - numpy=1.21.2 - - numpy-base=1.21.2 - - olefile=0.46 - - openjpeg=2.4.0 - - openssl=1.1.1n - - packaging=21.3 - - palettable=3.3.0 - - pandas=1.3.3 - - pandocfilters=1.5.0 - - parso=0.8.2 - - pickleshare=0.7.5 - - pillow=8.4.0 - - pip=21.2.4 - - pluggy=1.0.0 - - prometheus_client=0.13.1 - - prompt-toolkit=3.0.20 - - py=1.11.0 - - pycparser=2.21 - - pyfai=0.20.0 - - pyfai-base=0.20.0 - - pygments=2.10.0 - - pyparsing=2.4.7 - - pyqt=5.12.3 - - pyqt-impl=5.12.3 - - pyqt5-sip=4.19.18 - - pyqtchart=5.12 - - pyqtwebengine=5.12.1 - - pyreadline=2.1 - - pyrsistent=0.18.0 - - pytest=7.1.1 - - python=3.9.7 - - python-dateutil=2.8.2 - - python_abi=3.9 - - pytz=2021.3 - - pywin32=228 - - pywinpty=2.0.2 - - pyzmq=22.2.1 - - qt=5.12.9 - - qtconsole=5.1.1 - - qtpy=1.11.2 - - scipy=1.7.1 - - seaborn=0.11.2 - - send2trash=1.8.0 - - setuptools=58.0.4 - - silx=0.15.2 - - silx-base=0.15.2 - - six=1.16.0 - - sqlite=3.36.0 - - sympy=1.9 - - terminado=0.13.1 - - testpath=0.5.0 - - tk=8.6.11 - - tomli=1.2.2 - - tornado=6.1 - - traitlets=5.1.0 - - typing-extensions=3.10.0.2 - - typing_extensions=3.10.0.2 - - tzdata=2021a - - vc=14.2 - - vs2015_runtime=14.27.29016 - - wcwidth=0.2.5 - - webencodings=0.5.1 - - wheel=0.37.0 - - widgetsnbextension=3.5.2 - - wincertstore=0.2 - - winpty=0.4.3 - - xz=5.2.5 - - zipp=3.7.0 - - zlib=1.2.11 - - zstd=1.5.0 -prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime From bae40fcabdf6acae4f9a9c2a9550082e74479518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 14:41:29 +0200 Subject: [PATCH 133/355] Set up third attempt at autotesting --- .github/workflows/python-package-conda.yml | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml new file mode 100644 index 0000000..6261945 --- /dev/null +++ b/.github/workflows/python-package-conda.yml @@ -0,0 +1,34 @@ +name: Python Package using Conda + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.9 + uses: actions/setup-python@v3 + with: + python-version: 3.9.7 + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + - name: Lint with flake8 + run: | + conda install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + conda install pytest + pytest From 89ea45a9bf3aa5f8d7f137764df0ee3e34cef966 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 14:50:09 +0200 Subject: [PATCH 134/355] Make manual environment file --- environment.yml | 155 +++--------------------------------------------- 1 file changed, 8 insertions(+), 147 deletions(-) diff --git a/environment.yml b/environment.yml index e7bc821..9500050 100644 --- a/environment.yml +++ b/environment.yml @@ -4,151 +4,12 @@ channels: - diffpy - defaults dependencies: - - argon2-cffi=21.3.0 - - argon2-cffi-bindings=21.2.0 - - atomicwrites=1.4.0 - - attrs=21.4.0 - - backcall=0.2.0 - - blas=1.0 - - bleach=4.1.0 - - bottleneck=1.3.2 - - ca-certificates=2022.3.29 - - cached-property=1.5.2 - - cached_property=1.5.2 - - certifi=2021.10.8 - - cffi=1.15.0 - - colorama=0.4.4 - - cycler=0.10.0 - - debugpy=1.4.1 - - decorator=5.1.0 - - defusedxml=0.7.1 - - entrypoints=0.3 - - fabio=0.12.0 - - freetype=2.10.4 - - glymur=0.9.4 - - h5py=3.2.1 - - hdf5=1.10.6 - - hdf5plugin=3.1.1 - - icc_rt=2019.0.0 - - icu=68.1 - - importlib-metadata=4.8.2 - - importlib_metadata=4.8.2 - - iniconfig=1.1.1 - - intel-openmp=2021.3.0 - - ipykernel=6.4.1 - - ipython=7.27.0 - - ipython_genutils=0.2.0 - - ipywidgets=7.6.5 - - jbig=2.1 - - jedi=0.18.0 - - jinja2=3.0.2 - - jpeg=9d - - jsonschema=3.2.0 - - jupyter_client=7.0.1 - - jupyter_core=4.8.1 - - jupyterlab_pygments=0.1.2 - - jupyterlab_widgets=1.0.0 - - kiwisolver=1.3.2 - - krb5=1.19.2 - - lerc=3.0 - - libclang=11.1.0 - - libcurl=7.79.1 - - libdeflate=1.8 - - libiconv=1.16 - - libpng=1.6.37 - - libssh2=1.10.0 - - libtiff=4.3.0 - - libwebp=1.2.0 - - libxml2=2.9.12 - - libxslt=1.1.34 - - libzlib=1.2.11 - - lxml=4.6.3 - - lz4-c=1.9.3 - - mako=1.1.5 - - markupsafe=2.0.1 - - matplotlib=3.4.3 - - matplotlib-base=3.4.3 - - matplotlib-inline=0.1.2 - - mistune=0.8.4 - - mkl=2021.3.0 - - mkl-service=2.4.0 - - mkl_fft=1.3.1 - - mkl_random=1.2.2 - - mpmath=1.2.1 - - nbclient=0.5.11 - - nbconvert=6.1.0 - - nbformat=5.1.3 - - nest-asyncio=1.5.1 - - notebook=6.4.8 - - numexpr=2.7.3 - - numpy=1.21.2 - - numpy-base=1.21.2 - - olefile=0.46 - - openjpeg=2.4.0 - - openssl=1.1.1n - - packaging=21.3 - - palettable=3.3.0 - - pandas=1.3.3 - - pandocfilters=1.5.0 - - parso=0.8.2 - - pickleshare=0.7.5 - - pillow=8.4.0 - - pip=21.2.4 - - pluggy=1.0.0 - - prometheus_client=0.13.1 - - prompt-toolkit=3.0.20 - - py=1.11.0 - - pycparser=2.21 - - pyfai=0.20.0 - - pyfai-base=0.20.0 - - pygments=2.10.0 - - pyparsing=2.4.7 - - pyqt=5.12.3 - - pyqt-impl=5.12.3 - - pyqt5-sip=4.19.18 - - pyqtchart=5.12 - - pyqtwebengine=5.12.1 - - pyreadline=2.1 - - pyrsistent=0.18.0 - - pytest=7.1.1 - - python=3.9.7 - - python-dateutil=2.8.2 - - python_abi=3.9 - - pytz=2021.3 - - pywin32=228 - - pywinpty=2.0.2 - - pyzmq=22.2.1 - - qt=5.12.9 - - qtconsole=5.1.1 - - qtpy=1.11.2 - - scipy=1.7.1 - - seaborn=0.11.2 - - send2trash=1.8.0 - - setuptools=58.0.4 - - silx=0.15.2 - - silx-base=0.15.2 - - six=1.16.0 - - sqlite=3.36.0 - - sympy=1.9 - - terminado=0.13.1 - - testpath=0.5.0 - - tk=8.6.11 - - tomli=1.2.2 - - tornado=6.1 - - traitlets=5.1.0 - - typing-extensions=3.10.0.2 - - typing_extensions=3.10.0.2 - - tzdata=2021a - - vc=14.2 - - vs2015_runtime=14.27.29016 - - wcwidth=0.2.5 - - webencodings=0.5.1 - - wheel=0.37.0 - - widgetsnbextension=3.5.2 - - wincertstore=0.2 - - winpty=0.4.3 - - xz=5.2.5 - - zipp=3.7.0 - - zlib=1.2.11 - - zstd=1.5.0 + - pandas + - numpy + - matplotlib + - ipywidgets + - palettable + - sympy + - seaborn + - pytest prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime From ceb9d1ab462ecc4574cb2cde55fe7df1478fb344 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 15:15:12 +0200 Subject: [PATCH 135/355] Upload new environment --- environment.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/environment.yml b/environment.yml index 9500050..24ccf4d 100644 --- a/environment.yml +++ b/environment.yml @@ -1,15 +1,15 @@ -name: beamtime +name: nafuma channels: - - conda-forge - diffpy - defaults dependencies: - - pandas - - numpy - - matplotlib - ipywidgets - - palettable - - sympy - seaborn + - sympy + - matplotlib - pytest -prefix: C:\Users\rasmusvt\Anaconda3\envs\beamtime + - numpy + - pandas + - palettable + - pyfai +prefix: C:\Users\rasmusvt\Anaconda3\envs\nafuma From 19918b3207f791563814fff54600e0b68390b8da Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 15:17:39 +0200 Subject: [PATCH 136/355] Remove old workflow --- .github/workflows/python-package-conda.yml | 34 ---------------------- 1 file changed, 34 deletions(-) delete mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml deleted file mode 100644 index 6261945..0000000 --- a/.github/workflows/python-package-conda.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Python Package using Conda - -on: [push] - -jobs: - build-linux: - runs-on: ubuntu-latest - strategy: - max-parallel: 5 - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.9 - uses: actions/setup-python@v3 - with: - python-version: 3.9.7 - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH - - name: Install dependencies - run: | - conda env update --file environment.yml --name base - - name: Lint with flake8 - run: | - conda install flake8 - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - conda install pytest - pytest From 2c697be9da6abd969b2846618eacaa246b864796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Thu, 7 Apr 2022 15:18:17 +0200 Subject: [PATCH 137/355] New attempt at setting up workflow --- .github/workflows/python-package-conda.yml | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/python-package-conda.yml diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml new file mode 100644 index 0000000..2c3dce4 --- /dev/null +++ b/.github/workflows/python-package-conda.yml @@ -0,0 +1,34 @@ +name: Python Package using Conda + +on: [push] + +jobs: + build-linux: + runs-on: ubuntu-latest + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: 3.10.4 + - name: Add conda to system path + run: | + # $CONDA is an environment variable pointing to the root of the miniconda directory + echo $CONDA/bin >> $GITHUB_PATH + - name: Install dependencies + run: | + conda env update --file environment.yml --name base + - name: Lint with flake8 + run: | + conda install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + conda install pytest + pytest From 601a5b061911c6be8effcbe0f70dbf769bcda490 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 15:19:57 +0200 Subject: [PATCH 138/355] Add conda-forge to channels --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 24ccf4d..04c88d5 100644 --- a/environment.yml +++ b/environment.yml @@ -2,6 +2,7 @@ name: nafuma channels: - diffpy - defaults + - conda-forge dependencies: - ipywidgets - seaborn From de8c0ab8d5baada16e01adef6b08f21c864ff109 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 15:28:49 +0200 Subject: [PATCH 139/355] Add install of package to workflow --- .github/workflows/python-package-conda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 2c3dce4..ebf95a8 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -21,6 +21,7 @@ jobs: - name: Install dependencies run: | conda env update --file environment.yml --name base + pip install . - name: Lint with flake8 run: | conda install flake8 From 3d99af9a7a8e7d64721ba63026cb4bfb08b5c34c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 15:39:07 +0200 Subject: [PATCH 140/355] Fix linting issues --- beamtime/plotting.py | 6 ++--- beamtime/xanes/calib.py | 2 +- beamtime/xrd/plot.py | 49 +---------------------------------------- 3 files changed, 5 insertions(+), 52 deletions(-) diff --git a/beamtime/plotting.py b/beamtime/plotting.py index 780a022..e8c5457 100644 --- a/beamtime/plotting.py +++ b/beamtime/plotting.py @@ -151,9 +151,9 @@ def adjust_plot(fig, ax, options): # FIXME THIS NEEDS REWORK FOR IT TO FUNCTION PROPERLY! - if options['xticks']: - ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) - ax.set_xticklabels(options['xticks']) + #if options['xticks']: + # ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) + # ax.set_xticklabels(options['xticks']) # else: # ax.set_xticks(np.arange(plot_data['start'], plot_data['end']+1)) # ax.set_xticklabels([x/2 for x in np.arange(plot_data['start'], plot_data['end']+1)]) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 2ef6c4d..6a4c8f6 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -13,7 +13,7 @@ def rbkerbest(): ##Better to make a new function that loops through the files, and performing the split_xanes_scan on -def split_xanes_scan(root, destination=None, replace=False): +def split_xanes_scan(filename, destination=None, replace=False): #root is the path to the beamtime-folder #destination should be the path to the processed data diff --git a/beamtime/xrd/plot.py b/beamtime/xrd/plot.py index cc6baa3..950a27f 100644 --- a/beamtime/xrd/plot.py +++ b/beamtime/xrd/plot.py @@ -7,6 +7,7 @@ import numpy as np import math import ipywidgets as widgets +from IPython.display import display import beamtime.xrd as xrd import beamtime.auxillary as aux @@ -657,54 +658,6 @@ def prettify_labels(label): return labels_dict[label] -def plot_diffractograms(paths, kind, options=None): - - - fig, ax = prepare_diffractogram_plot(options=options) - - diffractograms = [] - - for path in paths: - diffractogram = xrd.io.read_data(path=path, kind=kind, options=options) - diffractograms.append(diffractogram) - - - required_options = ['type', 'xvals', 'yvals', 'x_offset', 'y_offset', 'normalise', 'normalise_around', 'reverse_order'] - default_options = { - 'type': 'stacked', - 'xvals': '2th', - 'yvals': 'I', - 'x_offset': 0, - 'y_offset': 0.2, - 'normalise': True, - 'normalise_around': None, - 'reverse_order': False - } - - - # If reverse_order is enabled, reverse the order - if options['reverse_order']: - diffractograms = reverse_diffractograms(diffractograms) - - - # If normalise is enbaled, normalise all the diffractograms - if options['normalise']: - if not options['normalise_around']: - for diffractogram in diffractograms: - diffractogram["I"] = diffractogram["I"]/diffractogram["I"].max() - else: - diffractogram["I"] = diffractogram["I"]/diffractogram["I"].loc[(diffractogram['2th'] > options['normalise_around'][0]) & (diffractogram['2th'] < options['normalise_around'][1])].max() - - - if options['type'] == 'stacked': - for diffractogram in diffractograms: - diffractogram.plot(x=options['xvals'], y=options['yvals'], ax=ax) - - - fig, ax = prettify_diffractogram_plot(fig=fig, ax=ax, options=options) - - - return diffractogram, fig, ax def reverse_diffractograms(diffractograms): From 049f30d96b492573a5e64e8e091b9b5d7c16fa16 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 15:47:26 +0200 Subject: [PATCH 141/355] Fix last remaining liniting issue --- beamtime/xrd/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beamtime/xrd/io.py b/beamtime/xrd/io.py index c2447d8..fc5c292 100644 --- a/beamtime/xrd/io.py +++ b/beamtime/xrd/io.py @@ -149,7 +149,7 @@ def average_images(images): image_arrays = [] for image in images: - image_array = xrd.io.get_image_array(image) + image_array = get_image_array(image) image_arrays.append(image_array) From 27c911cf5454ed48aab8cb90f8ca3691bb7d68a4 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 17:05:52 +0200 Subject: [PATCH 142/355] Rename package to nafuma from beamtime --- feature_list.txt | 2 -- {beamtime => nafuma}/__init__.py | 0 {beamtime => nafuma}/auxillary.py | 0 {beamtime => nafuma}/electrochemistry/__init__.py | 0 {beamtime => nafuma}/electrochemistry/io.py | 0 {beamtime => nafuma}/electrochemistry/plot.py | 0 {beamtime => nafuma}/electrochemistry/unit_tables.py | 0 {beamtime => nafuma}/pdf/__init__.py | 0 {beamtime => nafuma}/plotting.py | 0 {beamtime => nafuma}/test.txt | 0 {beamtime => nafuma}/test/__init__.py | 0 {beamtime => nafuma}/test/pytest.ini | 0 {beamtime => nafuma}/test/test_auxillary.py | 0 {beamtime => nafuma}/test/test_plotting.py | 0 {beamtime => nafuma}/test/xrd/test_io.py | 0 {beamtime => nafuma}/test/xrd/test_plot.py | 0 {beamtime => nafuma}/test2.txt | 0 {beamtime => nafuma}/xanes/__init__.py | 0 {beamtime => nafuma}/xanes/calib.py | 0 {beamtime => nafuma}/xanes/io.py | 0 {beamtime => nafuma}/xrd/__init__.py | 0 {beamtime => nafuma}/xrd/io.py | 0 {beamtime => nafuma}/xrd/plot.py | 0 {beamtime => nafuma}/xrd/test.txt | 0 setup.py | 10 +++++----- 25 files changed, 5 insertions(+), 7 deletions(-) delete mode 100644 feature_list.txt rename {beamtime => nafuma}/__init__.py (100%) rename {beamtime => nafuma}/auxillary.py (100%) rename {beamtime => nafuma}/electrochemistry/__init__.py (100%) rename {beamtime => nafuma}/electrochemistry/io.py (100%) rename {beamtime => nafuma}/electrochemistry/plot.py (100%) rename {beamtime => nafuma}/electrochemistry/unit_tables.py (100%) rename {beamtime => nafuma}/pdf/__init__.py (100%) rename {beamtime => nafuma}/plotting.py (100%) rename {beamtime => nafuma}/test.txt (100%) rename {beamtime => nafuma}/test/__init__.py (100%) rename {beamtime => nafuma}/test/pytest.ini (100%) rename {beamtime => nafuma}/test/test_auxillary.py (100%) rename {beamtime => nafuma}/test/test_plotting.py (100%) rename {beamtime => nafuma}/test/xrd/test_io.py (100%) rename {beamtime => nafuma}/test/xrd/test_plot.py (100%) rename {beamtime => nafuma}/test2.txt (100%) rename {beamtime => nafuma}/xanes/__init__.py (100%) rename {beamtime => nafuma}/xanes/calib.py (100%) rename {beamtime => nafuma}/xanes/io.py (100%) rename {beamtime => nafuma}/xrd/__init__.py (100%) rename {beamtime => nafuma}/xrd/io.py (100%) rename {beamtime => nafuma}/xrd/plot.py (100%) rename {beamtime => nafuma}/xrd/test.txt (100%) diff --git a/feature_list.txt b/feature_list.txt deleted file mode 100644 index f4a0efd..0000000 --- a/feature_list.txt +++ /dev/null @@ -1,2 +0,0 @@ -- Must allow for automatic normalisation between different diffractograms, must only happen upon reading data -- diff --git a/beamtime/__init__.py b/nafuma/__init__.py similarity index 100% rename from beamtime/__init__.py rename to nafuma/__init__.py diff --git a/beamtime/auxillary.py b/nafuma/auxillary.py similarity index 100% rename from beamtime/auxillary.py rename to nafuma/auxillary.py diff --git a/beamtime/electrochemistry/__init__.py b/nafuma/electrochemistry/__init__.py similarity index 100% rename from beamtime/electrochemistry/__init__.py rename to nafuma/electrochemistry/__init__.py diff --git a/beamtime/electrochemistry/io.py b/nafuma/electrochemistry/io.py similarity index 100% rename from beamtime/electrochemistry/io.py rename to nafuma/electrochemistry/io.py diff --git a/beamtime/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py similarity index 100% rename from beamtime/electrochemistry/plot.py rename to nafuma/electrochemistry/plot.py diff --git a/beamtime/electrochemistry/unit_tables.py b/nafuma/electrochemistry/unit_tables.py similarity index 100% rename from beamtime/electrochemistry/unit_tables.py rename to nafuma/electrochemistry/unit_tables.py diff --git a/beamtime/pdf/__init__.py b/nafuma/pdf/__init__.py similarity index 100% rename from beamtime/pdf/__init__.py rename to nafuma/pdf/__init__.py diff --git a/beamtime/plotting.py b/nafuma/plotting.py similarity index 100% rename from beamtime/plotting.py rename to nafuma/plotting.py diff --git a/beamtime/test.txt b/nafuma/test.txt similarity index 100% rename from beamtime/test.txt rename to nafuma/test.txt diff --git a/beamtime/test/__init__.py b/nafuma/test/__init__.py similarity index 100% rename from beamtime/test/__init__.py rename to nafuma/test/__init__.py diff --git a/beamtime/test/pytest.ini b/nafuma/test/pytest.ini similarity index 100% rename from beamtime/test/pytest.ini rename to nafuma/test/pytest.ini diff --git a/beamtime/test/test_auxillary.py b/nafuma/test/test_auxillary.py similarity index 100% rename from beamtime/test/test_auxillary.py rename to nafuma/test/test_auxillary.py diff --git a/beamtime/test/test_plotting.py b/nafuma/test/test_plotting.py similarity index 100% rename from beamtime/test/test_plotting.py rename to nafuma/test/test_plotting.py diff --git a/beamtime/test/xrd/test_io.py b/nafuma/test/xrd/test_io.py similarity index 100% rename from beamtime/test/xrd/test_io.py rename to nafuma/test/xrd/test_io.py diff --git a/beamtime/test/xrd/test_plot.py b/nafuma/test/xrd/test_plot.py similarity index 100% rename from beamtime/test/xrd/test_plot.py rename to nafuma/test/xrd/test_plot.py diff --git a/beamtime/test2.txt b/nafuma/test2.txt similarity index 100% rename from beamtime/test2.txt rename to nafuma/test2.txt diff --git a/beamtime/xanes/__init__.py b/nafuma/xanes/__init__.py similarity index 100% rename from beamtime/xanes/__init__.py rename to nafuma/xanes/__init__.py diff --git a/beamtime/xanes/calib.py b/nafuma/xanes/calib.py similarity index 100% rename from beamtime/xanes/calib.py rename to nafuma/xanes/calib.py diff --git a/beamtime/xanes/io.py b/nafuma/xanes/io.py similarity index 100% rename from beamtime/xanes/io.py rename to nafuma/xanes/io.py diff --git a/beamtime/xrd/__init__.py b/nafuma/xrd/__init__.py similarity index 100% rename from beamtime/xrd/__init__.py rename to nafuma/xrd/__init__.py diff --git a/beamtime/xrd/io.py b/nafuma/xrd/io.py similarity index 100% rename from beamtime/xrd/io.py rename to nafuma/xrd/io.py diff --git a/beamtime/xrd/plot.py b/nafuma/xrd/plot.py similarity index 100% rename from beamtime/xrd/plot.py rename to nafuma/xrd/plot.py diff --git a/beamtime/xrd/test.txt b/nafuma/xrd/test.txt similarity index 100% rename from beamtime/xrd/test.txt rename to nafuma/xrd/test.txt diff --git a/setup.py b/setup.py index 4c93b78..b3f9f34 100644 --- a/setup.py +++ b/setup.py @@ -1,11 +1,11 @@ from setuptools import setup, find_packages -setup(name='beamtime', - version='0.1', - description='Package for process and analysis of beamtime data from SNBL', - url='http://github.uio.no/rasmusvt/beamtime', +setup(name='nafuma', + version='0.2', + description='Analysis tools for inorganic materials chemistry at the NAFUMA-group at the University of Oslo', + url='https://github.com/rasmusthog/nafuma', author='Rasmus Vester Thøgersen, Halvor Høen Hval', - author_email='rasmusvt@smn.uio.no', + author_email='code@rasmusthog.me', license='MIT', packages=find_packages(), zip_safe=False) From 08276301f251e25eb66cce51c46edee6f19f9955 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 17:11:14 +0200 Subject: [PATCH 143/355] Update imports to match new package name --- nafuma/electrochemistry/plot.py | 2 +- nafuma/plotting.py | 2 +- nafuma/test/test_auxillary.py | 2 +- nafuma/test/test_plotting.py | 2 +- nafuma/xrd/io.py | 2 +- nafuma/xrd/plot.py | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 64b7115..df977a5 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -5,7 +5,7 @@ import pandas as pd import numpy as np import math -import beamtime.electrochemistry as ec +import nafuma.electrochemistry as ec def plot_gc(path, kind, options=None): diff --git a/nafuma/plotting.py b/nafuma/plotting.py index e8c5457..2233ab7 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -1,4 +1,4 @@ -import beamtime.auxillary as aux +import nafuma.auxillary as aux import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator) diff --git a/nafuma/test/test_auxillary.py b/nafuma/test/test_auxillary.py index 668b792..4cb9cd1 100644 --- a/nafuma/test/test_auxillary.py +++ b/nafuma/test/test_auxillary.py @@ -1,4 +1,4 @@ -import beamtime.auxillary as aux +import nafuma.auxillary as aux import os def test_swap_values(): diff --git a/nafuma/test/test_plotting.py b/nafuma/test/test_plotting.py index 8979bc1..e1af0b8 100644 --- a/nafuma/test/test_plotting.py +++ b/nafuma/test/test_plotting.py @@ -1,4 +1,4 @@ -import beamtime.plotting as btp +import nafuma.plotting as btp from cycler import cycler import itertools import numpy as np diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index fc5c292..b3f3951 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -9,7 +9,7 @@ import zipfile import xml.etree.ElementTree as ET -import beamtime.auxillary as aux +import nafuma.auxillary as aux def get_image_array(path): diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 950a27f..8dd6a27 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -9,9 +9,9 @@ import math import ipywidgets as widgets from IPython.display import display -import beamtime.xrd as xrd -import beamtime.auxillary as aux -import beamtime.plotting as btp +import nafuma.xrd as xrd +import nafuma.auxillary as aux +import nafuma.plotting as btp def plot_diffractogram(data, options={}): From f53527fe0b2f8bbc350d850154e1c9a6d811c3b3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 17:17:28 +0200 Subject: [PATCH 144/355] Clean up test files --- nafuma/test.txt | 1 - nafuma/test2.txt | 0 nafuma/xrd/test.txt | 0 3 files changed, 1 deletion(-) delete mode 100644 nafuma/test.txt delete mode 100644 nafuma/test2.txt delete mode 100644 nafuma/xrd/test.txt diff --git a/nafuma/test.txt b/nafuma/test.txt deleted file mode 100644 index 85476a8..0000000 --- a/nafuma/test.txt +++ /dev/null @@ -1 +0,0 @@ -sdfsdfsdfsdf \ No newline at end of file diff --git a/nafuma/test2.txt b/nafuma/test2.txt deleted file mode 100644 index e69de29..0000000 diff --git a/nafuma/xrd/test.txt b/nafuma/xrd/test.txt deleted file mode 100644 index e69de29..0000000 From de2616067daf2ebc333a592f2729a2425b9b1e25 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Apr 2022 17:19:26 +0200 Subject: [PATCH 145/355] Rename automated test workflow --- .../{python-package-conda.yml => automated-testing.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{python-package-conda.yml => automated-testing.yml} (96%) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/automated-testing.yml similarity index 96% rename from .github/workflows/python-package-conda.yml rename to .github/workflows/automated-testing.yml index ebf95a8..5b553a3 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/automated-testing.yml @@ -1,4 +1,4 @@ -name: Python Package using Conda +name: Automated testing on: [push] From 7962b3fc5cd67c1e47e04bb2063bac60cd4b3522 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 8 Apr 2022 13:28:47 +0200 Subject: [PATCH 146/355] Fixing functions --- beamtime/xanes/calib.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/beamtime/xanes/calib.py b/beamtime/xanes/calib.py index 9e3cbdb..7a0ecb5 100644 --- a/beamtime/xanes/calib.py +++ b/beamtime/xanes/calib.py @@ -126,18 +126,7 @@ def post_edge_normalization(path, options={}): if edge == 'Ni': edge_stop = 8.361 -#============================================================= - #absolute_difference_function = lambda list_value : abs(list_value - edge_stop) - #edge_stop_value = min(df_bkgd_sub["ZapEnergy"], key=absolute_difference_function) - #end_index=df_bkgd_sub[df_bkgd_sub["ZapEnergy"]==edge_stop_value].index.values[0] - #Defining x-range for linear fit - #df_fix=df_bkgd_sub - #df_fix.dropna(inplace=True) - #df_end=df_fix[end_index:] #The region of interest for the post edge - - #Fitting linear function to the pre-edge using the background corrected intensities to make the post edge fit - #=============================================================== - df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge -> to be fitted + df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge, where a regression line will be calculated in the for-loop below df_end.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit df_postedge = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) #making a new dataframe @@ -149,17 +138,14 @@ def post_edge_normalization(path, options={}): function_post_list.append(function_post) df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge - #print(filenames[0]) - #print(df_postedge) #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line if options['print'] == True: ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y plt.axvline(x = min(df_end["ZapEnergy"])) fig = plt.figure(figsize=(15,15)) df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) - #print(function_post_list) - #print(function_post) ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) plt.axvline(x = min(df_end["ZapEnergy"])) - \ No newline at end of file + + return df_bkgd_sub, df_postedge \ No newline at end of file From 67ba03dc0c70d194348135cb8b62a17783b2f3f8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Apr 2022 14:30:05 +0200 Subject: [PATCH 147/355] Fix lint problem in xanes.io.py --- nafuma/xanes/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 527f300..0633177 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -3,7 +3,7 @@ import matplotlib.pyplot as plt import os import numpy as np -def split_xanes_scan(root, destination=None, replace=False): +def split_xanes_scan(filename, destination=None, replace=False): #root is the path to the beamtime-folder #destination should be the path to the processed data From 4b74ea45922e28c3bfe98b04efec3ba309b3fab0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Apr 2022 14:47:35 +0200 Subject: [PATCH 148/355] Fixing another lint problem --- nafuma/xanes/calib.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index ad3443f..ec5c33c 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -2,9 +2,9 @@ import pandas as pd import numpy as np import os import matplotlib.pyplot as plt -import beamtime.auxillary as aux -import beamtime.xanes as xas -import beamtime.xanes.io as io +import nafuma.auxillary as aux +import nafuma.xanes as xas +import nafuma.xanes.io as io def rbkerbest(): print("ROSENBORG!<3") @@ -24,7 +24,6 @@ def finding_edge(df): edge='Ni' return(edge) -<<<<<<< HEAD:beamtime/xanes/calib.py #def pre_edge_subtraction(df,filenames, options={}): def test(innmat): df_test= xas.io.put_in_dataframe(innmat) @@ -41,11 +40,6 @@ def pre_edge_subtraction(path, options={}): filenames = xas.io.get_filenames(path) df= xas.io.put_in_dataframe(path) edge=finding_edge(df) -======= -def split_xanes_scan(filename, destination=None, replace=False): - #root is the path to the beamtime-folder - #destination should be the path to the processed data ->>>>>>> master:nafuma/xanes/calib.py #Defining the end of the region used to define the background, thus start of the edge #implement widget From 80cc8f67798c1f393afc1bea8ef8e3c2d1fc6af5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Apr 2022 15:31:40 +0200 Subject: [PATCH 149/355] Clean up files --- test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index 35d92ec..0000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -hei på dej \ No newline at end of file From f52f00a0b72a45b994830476a1a99eee8a1388a7 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Apr 2022 15:32:55 +0200 Subject: [PATCH 150/355] Clean up files --- test.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index 35d92ec..0000000 --- a/test.txt +++ /dev/null @@ -1 +0,0 @@ -hei på dej \ No newline at end of file From cd5cbe5dd4b37761c1c2e029411a1a4a84ace18c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Apr 2022 18:16:57 +0200 Subject: [PATCH 151/355] Add first draft of docs built with sphinx --- docs/Makefile | 20 +++++ docs/about.md | 9 +++ docs/conf.py | 57 ++++++++++++++ docs/index.rst | 22 ++++++ docs/installation.md | 25 ++++++ docs/make.bat | 35 +++++++++ docs/modules/electrochemistry.md | 3 + docs/modules/modules.rst | 12 +++ docs/modules/xanes.md | 1 + docs/modules/xrd.md | 130 +++++++++++++++++++++++++++++++ 10 files changed, 314 insertions(+) create mode 100644 docs/Makefile create mode 100644 docs/about.md create mode 100644 docs/conf.py create mode 100644 docs/index.rst create mode 100644 docs/installation.md create mode 100644 docs/make.bat create mode 100644 docs/modules/electrochemistry.md create mode 100644 docs/modules/modules.rst create mode 100644 docs/modules/xanes.md create mode 100644 docs/modules/xrd.md diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000..755a797 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,9 @@ +# About + +This package contains data processing, analysis and viewing tools written in Python for several different activities related to inorganic materials chemistry conducted in the NAFUMA-group at the University of Oslo. It is written with the intention of creating a reproducible workflow for documentation purposes, with a focus on interactivity in the data exploration process. + +As of now (08-04-22), the intention is to include tools for XRD-, XANES- and electrochemistry-analysis, however other modules might be added as well. + + + + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..1be3af8 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,57 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'NAFUMA' +copyright = '2022, Rasmus Vester Thøgersen & Halvor Høen Hval' +author = 'Rasmus Vester Thøgersen & Halvor Høen Hval' + +# The full version, including alpha/beta/rc tags +release = '0.2' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['myst_parser'] +source_suffix = ['.rst', '.md'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']} diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..c775e23 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,22 @@ +.. NAFUMA documentation master file, created by + sphinx-quickstart on Fri Apr 8 15:32:14 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to NAFUMA's documentation! +================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + about + installation + modules/modules + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..252dc5b --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,25 @@ +# Installation + +This package is not available on any package repositories, but can be installed by cloning the repository from GitHub and installing via ```pip install``` from the root folder: + +``` +$ git clone git@github.com:rasmusthog/nafuma.git +$ cd nafuma +$ pip install . +``` +If you are planning on making changes to the code base, you might want to consider installing it in develop-mode in order for changes to take effect without reinstalling by including the ```-e``` flag: + +``` +pip install -e . +``` + +As of now (v0.2, 08-04-22), the installer will not install any dependencies. It is recommended that you use `conda` to create an environment from `environment.yml` in the root folder: + +``` +$ conda env create --name --file environment.yml +$ conda activate +``` + +(remember to also get rid of <> when substituting your environment name). + +This should get you up and running! diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..8084272 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/modules/electrochemistry.md b/docs/modules/electrochemistry.md new file mode 100644 index 0000000..b962bd1 --- /dev/null +++ b/docs/modules/electrochemistry.md @@ -0,0 +1,3 @@ +# Electrochemistry + +This is a placeholder diff --git a/docs/modules/modules.rst b/docs/modules/modules.rst new file mode 100644 index 0000000..1f659da --- /dev/null +++ b/docs/modules/modules.rst @@ -0,0 +1,12 @@ +Modules +================================== + +.. toctree:: + :maxdepth: 1 + :caption: Contents + + xrd.md + xanes.md + electrochemistry.md + + diff --git a/docs/modules/xanes.md b/docs/modules/xanes.md new file mode 100644 index 0000000..c18eec6 --- /dev/null +++ b/docs/modules/xanes.md @@ -0,0 +1 @@ +# XANES diff --git a/docs/modules/xrd.md b/docs/modules/xrd.md new file mode 100644 index 0000000..58e8c3c --- /dev/null +++ b/docs/modules/xrd.md @@ -0,0 +1,130 @@ +# XRD + +This module contains functions to view diffractogram data from several different sources. The Some features include: + +- Allows the user to plot the data in wavelength independent parameters (d, 1/d, q, q{math}`^2`, q{math}`^4`), or translated to CuK{math}`\alpha` or MoK{math}`\alpha` allowing comparison between diffractograms obtained with different wavelengths +- Plotting in interactive mode within Jupyter Notebook using the `ipywidgets`-package allowing real-time change of (certain) parameters +- Plotting reflection ticks and/or reflection indices from multiple simulated reflection tables (generated by VESTA) for comparison +- Plotting series of diffractograms in stacked mode (including ability to rotate the view for a 3D-view) or as a heatmap + + + +## 1 Compatible file formats + +The module is partially built as a wrapper around [pyFAI](https://github.com/silx-kit/pyFAI) (Fast Azimuthal Integrator) developed at the ESRF for integrating 2D diffractograms from the detectors they have. Given a suitable calibration file (`.poni`), the XRD-module will automatically integrate any file pyFAI can integrate. Upon running in interactive mode, the integration is only done once, but it is advised to perform integration of many diffractograms in a separate processing step and saving the results as `.xy`-files, as the integration will run again each time the function is called. + +In addition to this, it can also read the `.brml`-files produced by Bruker-instruments in the RECX-lab at the University of Oslo. + +## 2 Basic usage + +Plotting diffractograms is done by calling the `xrd.plot.plot_diffractogram()`-function, which takes two dictionaries as arguments: `data`, containing all data specific information and `options` which allows customisation of a range of different parameters. The `options`-argument is optional, and the function will contains a bunch of default values to make an as good plot as possible to begin with. + +**Example #1: Single diffractogram** + +```py +import nafuma.xrd as xrd + +data = { + 'path': 'path/to/data/diffractogram.brml' +} + +options = { + 'reflections_data': [ + {'path': 'reflections_phase_1.txt', 'min_alpha': 0.1, 'reflection_indices': 4, 'label': 'Phase 1', 'text_colour': 'black'}, + {'path': 'reflections_phase_2.txt', 'min_alpha': 0.1, 'reflections_indices': 4, 'label': 'Phase 2', 'text_colour': 'red'} + ], + 'hide_y_ticklabels': True, + 'hide_y_ticks': True +} + + +diff, fig, ax = xrd.plot.plot_diffractogram(data=data, options=options) +``` + +The return value `diff` is a list containing one `pandas.DataFrame` per diffractogram passed, in the above example only one. `fig` and `ax` are `matplotlib.pyplot.Figure`- and `matplotlib.pyplot.Axes`-objects, respectively. + +**Example #2: 2D diffractogram from ESRF requiring integration** + +```py +import nafuma.xrd as xrd + +data = { + 'path': 'path/to/data/2d_diffractogram.edf', + 'calibrant': 'path/to/calibrant/calibrant.poni', + 'nbins': 3000 +} + +diff, _ = xrd.plot.plot_diffractogram(data=data, options=options) +``` + +In this case we did not specify any options and will thus only use default values, and we stored both `fig` and `ax` in the variable `_` as we do not intend to use these. + +**Example #3: Plotting with interactive mode** + +This will can be done within a Jupyter Notebook, and will allow the user to tweak certain parameters real-time instead of having to recall the function every time. + +```py +import nafuma.xrd as xrd + +data = { + 'path': 'path/to/data/diffractogram.brml' +} + +options = { + 'interactive': True +} + + +diff, _ = xrd.plot.plot_diffractogram(data=data, options=options) +``` + +**Example #4: Plotting multiple diffractograms as stacked plots** + +Instead of passing just a string, you can pass a lsit of filenames. This will be plotted sequentially, with offsets, if desired (`offset_x` and `offset_y`). Default values of `offset_y` is 1 if less than 10 diffractograms have been passed, and 0.1 if more than 10 diffractograms are passed. When plotting series data (e.g. from *in situ* or *operando* measurements), a smaller offset is suitable. Keep in mind that these values only makes sense when the diffractograms are normalised (`'normalise': True`) - if not, the default offsets will be way too small to be noticeable. + +```py +import nafuma.xrd as xrd + +data = { + 'path': ['path/to/data/diffractogram_1.brml', 'path/to/data/diffractogram_2.brml'] +} + + +options = { + 'offset_y': 0.1, + 'offset_x': 0.05, +} + + +diff, _ = xrd.plot.plot_diffractogram(data=data, options=options) +``` + + +**Example #5: Plotting series data as heatmap** + +This differs very little from above, except that heatmaps are probably nonesense if not used on series data, and that you don't want offset in heatmaps. + +```py +import nafuma.xrd as xrd + +list_of_data = ['data_1.brml', 'data_2.brml'. ...., 'data_n.brml'] + +data = { + 'path': lists_of_data +} + + +options = { + 'heatmap': True +} + + +diff, _ = xrd.plot.plot_diffractogram(data=data, options=options) +``` + + + + + + + From 95e411ac21772ee0df3bc452f61862e684cf8412 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 22 Apr 2022 13:03:31 +0200 Subject: [PATCH 152/355] Fix bugs giving errors with plot_gc() --- nafuma/electrochemistry/io.py | 4 ++-- nafuma/electrochemistry/plot.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 1963253..e3dd668 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -76,13 +76,13 @@ def read_biologic(path): Output: df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' - with open(path, 'r') as f: + with open(path, 'rb') as f: lines = f.readlines() header_lines = int(lines[1].split()[-1]) - 1 - df = pd.read_csv(path, sep='\t', skiprows=header_lines) + df = pd.read_csv(path, sep='\t', skiprows=header_lines, encoding='cp1252') df.dropna(inplace=True, axis=1) return df diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index df977a5..0d3e2cc 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -161,7 +161,7 @@ def prettify_gc_plot(fig, ax, options=None): 'positions': {'xaxis': 'bottom', 'yaxis': 'left'}, 'x_vals': 'specific_capacity', 'y_vals': 'voltage', 'xlabel': None, 'ylabel': None, - 'units': None, + 'units': {'capacity': 'mAh', 'specific_capacity': r'mAh g$^{-1}$', 'time': 's', 'current': 'mA', 'energy': 'mWh', 'mass': 'g', 'voltage': 'V'}, 'sizes': None, 'title': None } From 514a20604bcc017bd6c815710fd8a3e35be03e02 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 22 Apr 2022 15:19:36 +0200 Subject: [PATCH 153/355] Standardise data flow --- nafuma/electrochemistry/io.py | 276 ++++++++++++++++---------------- nafuma/electrochemistry/plot.py | 107 ++++--------- 2 files changed, 169 insertions(+), 214 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index e3dd668..5abff3b 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -1,40 +1,28 @@ +from email.policy import default import pandas as pd import numpy as np import matplotlib.pyplot as plt import os +import nafuma.auxillary as aux +from sympy import re -def read_data(path, kind, options=None): +def read_data(data, options=None): - if kind == 'neware': - df = read_neware(path) - cycles = process_neware_data(df, options=options) + if data['kind'] == 'neware': + df = read_neware(data['path']) + cycles = process_neware_data(df=df, options=options) - elif kind == 'batsmall': - df = read_batsmall(path) + elif data['kind'] == 'batsmall': + df = read_batsmall(data['path']) cycles = process_batsmall_data(df=df, options=options) - elif kind == 'biologic': - df = read_biologic(path) + elif data['kind'] == 'biologic': + df = read_biologic(data['path']) cycles = process_biologic_data(df=df, options=options) return cycles -def read_batsmall(path): - ''' Reads BATSMALL-data into a DataFrame. - - Input: - path (required): string with path to datafile - - Output: - df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' - - df = pd.read_csv(path, skiprows=2, sep='\t') - df = df.loc[:, ~df.columns.str.contains('^Unnamed')] - - return df - - def read_neware(path, summary=False): @@ -43,6 +31,8 @@ def read_neware(path, summary=False): type is .csv, it will just open the datafile and it does not matter if summary is False or not.''' from xlsx2csv import Xlsx2csv + # FIXME Do a check if a .csv-file already exists even if the .xlsx is passed + # Convert from .xlsx to .csv to make readtime faster if path.split('.')[-1] == 'xlsx': csv_details = ''.join(path.split('.')[:-1]) + '_details.csv' @@ -66,6 +56,20 @@ def read_neware(path, summary=False): return df +def read_batsmall(path): + ''' Reads BATSMALL-data into a DataFrame. + + Input: + path (required): string with path to datafile + + Output: + df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' + + df = pd.read_csv(path, skiprows=2, sep='\t') + df = df.loc[:, ~df.columns.str.contains('^Unnamed')] + + return df + def read_biologic(path): ''' Reads Bio-Logic-data into a DataFrame. @@ -89,10 +93,6 @@ def read_biologic(path): - - - - def process_batsmall_data(df, options=None): ''' Takes BATSMALL-data in the form of a DataFrame and cleans the data up and converts units into desired units. Splits up into individual charge and discharge DataFrames per cycle, and outputs a list where each element is a tuple with the Chg and DChg-data. E.g. cycles[10][0] gives the charge data for the 11th cycle. @@ -111,26 +111,25 @@ def process_batsmall_data(df, options=None): ''' required_options = ['splice_cycles', 'molecular_weight', 'reverse_discharge', 'units'] - default_options = {'splice_cycles': False, 'molecular_weight': None, 'reverse_discharge': False, 'units': None} + + default_options = { + 'splice_cycles': False, + 'molecular_weight': None, + 'reverse_discharge': False, + 'units': None} - if not options: - options = default_options - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] + aux.update_options(options=options, required_options=required_options, default_options=default_options) + options['kind'] = 'batsmall' # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - new_units = set_units(units=options['units']) - old_units = get_old_units(df, kind='batsmall') - df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='batsmall') - - options['units'] = new_units + set_units(options) + options['old_units'] = get_old_units(df, options) + df = unit_conversion(df=df, options=options) if options['splice_cycles']: - df = splice_cycles(df=df, kind='batsmall') + df = splice_cycles(df=df, options=options) # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current df[["comment"]] = df[["comment"]].fillna(value={'comment': ''}) @@ -173,23 +172,21 @@ def process_batsmall_data(df, options=None): cycles.append((chg_df, dchg_df)) - - return cycles -def splice_cycles(df, kind): +def splice_cycles(df, options: dict) -> pd.DataFrame: + ''' Splices two cycles together - if e.g. one charge cycle are split into several cycles due to change in parameters. + + Incomplete, only accomodates BatSmall so far.''' - if kind == 'batsmall': + if options['kind'] == 'batsmall': # Creates masks for charge and discharge curves chg_mask = df['current'] >= 0 dchg_mask = df['current'] < 0 - # Get the number of cycles in the dataset - max_count = df["count"].max() - - # Loop through all the cycling steps, change the current and capacities in the + # Loop through all the cycling steps, change the current and capacities in the for i in range(df["count"].max()): sub_df = df.loc[df['count'] == i+1] sub_df_chg = sub_df.loc[chg_mask] @@ -233,7 +230,7 @@ def splice_cycles(df, kind): -def process_neware_data(df, options=None): +def process_neware_data(df, options={}): """ Takes data from NEWARE in a DataFrame as read by read_neware() and converts units, adds columns and splits into cycles. @@ -245,25 +242,26 @@ def process_neware_data(df, options=None): molecular_weight: the molar mass (in g mol^-1) of the active material, to calculate the number of ions extracted. Assumes one electron per Li+/Na+-ion """ required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles'] - default_options = {'units': None, 'active_material_weight': None, 'molecular_weight': None, 'reverse_discharge': False, 'splice_cycles': None} + + default_options = { + 'units': None, + 'active_material_weight': None, + 'molecular_weight': None, + 'reverse_discharge': False, + 'splice_cycles': None} - if not options: - options = default_options - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] + + aux.update_options(options=options, required_options=required_options, default_options=default_options) + options['kind'] = 'neware' # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - new_units = set_units(units=options['units']) - old_units = get_old_units(df=df, kind='neware') + set_units(options=options) # sets options['units'] + options['old_units'] = get_old_units(df=df, options=options) - df = add_columns(df=df, active_material_weight=options['active_material_weight'], molecular_weight=options['molecular_weight'], old_units=old_units, kind='neware') + df = add_columns(df=df, options=options) # adds columns to the DataFrame if active material weight and/or molecular weight has been passed in options - df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='neware') - - options['units'] = new_units + df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units # Creates masks for charge and discharge curves @@ -288,6 +286,8 @@ def process_neware_data(df, options=None): if chg_df.empty and dchg_df.empty: continue + + # Reverses the discharge curve if specified if options['reverse_discharge']: max_capacity = dchg_df['capacity'].max() dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) @@ -310,35 +310,34 @@ def process_neware_data(df, options=None): def process_biologic_data(df, options=None): required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles'] - default_options = {'units': None, 'active_material_weight': None, 'molecular_weight': None, 'reverse_discharge': False, 'splice_cycles': None} + + default_options = { + 'units': None, + 'active_material_weight': None, + 'molecular_weight': None, + 'reverse_discharge': False, + 'splice_cycles': None} - if not options: - options = default_options - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] + + aux.update_options(options=options, required_options=required_options, default_options=default_options) + options['kind'] = 'biologic' # Pick out necessary columns df = df[['Ns changes', 'Ns', 'time/s', 'Ewe/V', 'Energy charge/W.h', 'Energy discharge/W.h', '/mA', 'Capacity/mA.h', 'cycle number']].copy() # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - new_units = set_units(units=options['units']) - old_units = get_old_units(df=df, kind='biologic') + set_units(options) + options['old_units'] = get_old_units(df=df, options=options) - df = add_columns(df=df, active_material_weight=options['active_material_weight'], molecular_weight=options['molecular_weight'], old_units=old_units, kind='biologic') - - df = unit_conversion(df=df, new_units=new_units, old_units=old_units, kind='biologic') - - options['units'] = new_units + df = add_columns(df=df, options=options) + df = unit_conversion(df=df, options=options) # Creates masks for charge and discharge curves chg_mask = (df['status'] == 1) & (df['status_change'] != 1) dchg_mask = (df['status'] == 2) & (df['status_change'] != 1) - # Initiate cycles list cycles = [] @@ -376,62 +375,62 @@ def process_biologic_data(df, options=None): return cycles -def add_columns(df, active_material_weight, molecular_weight, old_units, kind): +def add_columns(df, options): - if kind == 'neware': - if active_material_weight: - df["SpecificCapacity({}/mg)".format(old_units["capacity"])] = df["Capacity({})".format(old_units['capacity'])] / (active_material_weight) + if options['kind'] == 'neware': + if options['active_material_weight']: + df["SpecificCapacity({}/mg)".format(options['old_units']["capacity"])] = df["Capacity({})".format(options['old_units']['capacity'])] / (options['active_material_weight']) - if molecular_weight: + if options['molecular_weight']: faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 seconds_per_hour = 3600 # s h^-1 f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 - df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(old_units['capacity'])]*molecular_weight)*1000/f + df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(options['old_units']['capacity'])]*options['molecular_weight'])*1000/f - if kind == 'biologic': - if active_material_weight: + if options['kind'] == 'biologic': + if options['active_material_weight']: - capacity = old_units['capacity'].split('h')[0] + '.h' + capacity = options['old_units']['capacity'].split('h')[0] + '.h' - df["SpecificCapacity({}/mg)".format(old_units["capacity"])] = df["Capacity/{}".format(capacity)] / (active_material_weight) + df["SpecificCapacity({}/mg)".format(options['old_units']["capacity"])] = df["Capacity/{}".format(capacity)] / (options['active_material_weight']) - if molecular_weight: + if options['molecular_weight']: faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 seconds_per_hour = 3600 # s h^-1 f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 - df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(old_units['capacity'])]*molecular_weight)*1000/f + df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(options['old_units']['capacity'])]*options['molecular_weight'])*1000/f return df -def unit_conversion(df, new_units, old_units, kind): +def unit_conversion(df, options): from . import unit_tables - if kind == 'batsmall': + if options['kind'] == 'batsmall': - df["TT [{}]".format(old_units["time"])] = df["TT [{}]".format(old_units["time"])] * unit_tables.time()[old_units["time"]].loc[new_units['time']] - df["U [{}]".format(old_units["voltage"])] = df["U [{}]".format(old_units["voltage"])] * unit_tables.voltage()[old_units["voltage"]].loc[new_units['voltage']] - df["I [{}]".format(old_units["current"])] = df["I [{}]".format(old_units["current"])] * unit_tables.current()[old_units["current"]].loc[new_units['current']] - df["C [{}/{}]".format(old_units["capacity"], old_units["mass"])] = df["C [{}/{}]".format(old_units["capacity"], old_units["mass"])] * (unit_tables.capacity()[old_units["capacity"]].loc[new_units["capacity"]] / unit_tables.mass()[old_units["mass"]].loc[new_units["mass"]]) + df["TT [{}]".format(options['old_units']["time"])] = df["TT [{}]".format(options['old_units']["time"])] * unit_tables.time()[options['old_units']["time"]].loc[options['units']['time']] + df["U [{}]".format(options['old_units']["voltage"])] = df["U [{}]".format(options['old_units']["voltage"])] * unit_tables.voltage()[options['old_units']["voltage"]].loc[options['units']['voltage']] + df["I [{}]".format(options['old_units']["current"])] = df["I [{}]".format(options['old_units']["current"])] * unit_tables.current()[options['old_units']["current"]].loc[options['units']['current']] + df["C [{}/{}]".format(options['old_units']["capacity"], options['old_units']["mass"])] = df["C [{}/{}]".format(options['old_units']["capacity"], options['old_units']["mass"])] * (unit_tables.capacity()[options['old_units']["capacity"]].loc[options['units']["capacity"]] / unit_tables.mass()[options['old_units']["mass"]].loc[options['units']["mass"]]) df.columns = ['time', 'voltage', 'current', 'count', 'specific_capacity', 'comment'] - if kind == 'neware': - df['Current({})'.format(old_units['current'])] = df['Current({})'.format(old_units['current'])] * unit_tables.current()[old_units['current']].loc[new_units['current']] - df['Voltage({})'.format(old_units['voltage'])] = df['Voltage({})'.format(old_units['voltage'])] * unit_tables.voltage()[old_units['voltage']].loc[new_units['voltage']] - df['Capacity({})'.format(old_units['capacity'])] = df['Capacity({})'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] - df['Energy({})'.format(old_units['energy'])] = df['Energy({})'.format(old_units['energy'])] * unit_tables.energy()[old_units['energy']].loc[new_units['energy']] - df['CycleTime({})'.format(new_units['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=new_units['time']), axis=1) - df['RunTime({})'.format(new_units['time'])] = df.apply(lambda row : convert_datetime_string(row['Real Time(h:min:s.ms)'], reference=df['Real Time(h:min:s.ms)'].iloc[0], unit=new_units['time']), axis=1) + if options['kind'] == 'neware': + df['Current({})'.format(options['old_units']['current'])] = df['Current({})'.format(options['old_units']['current'])] * unit_tables.current()[options['old_units']['current']].loc[options['units']['current']] + df['Voltage({})'.format(options['old_units']['voltage'])] = df['Voltage({})'.format(options['old_units']['voltage'])] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] + df['Capacity({})'.format(options['old_units']['capacity'])] = df['Capacity({})'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] + df['Energy({})'.format(options['old_units']['energy'])] = df['Energy({})'.format(options['old_units']['energy'])] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] + df['CycleTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=options['units']['time']), axis=1) + df['RunTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_datetime_string(row['Real Time(h:min:s.ms)'], reference=df['Real Time(h:min:s.ms)'].iloc[0], unit=options['units']['time']), axis=1) columns = ['status', 'jump', 'cycle', 'steps', 'current', 'voltage', 'capacity', 'energy'] - if 'SpecificCapacity({}/mg)'.format(old_units['capacity']) in df.columns: - df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] = df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] / unit_tables.mass()['mg'].loc[new_units["mass"]] + if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] columns.append('specific_capacity') if 'IonsExtracted' in df.columns: @@ -447,18 +446,18 @@ def unit_conversion(df, new_units, old_units, kind): df.columns = columns - if kind == 'biologic': - df['time/{}'.format(old_units['time'])] = df["time/{}".format(old_units["time"])] * unit_tables.time()[old_units["time"]].loc[new_units['time']] - df["Ewe/{}".format(old_units["voltage"])] = df["Ewe/{}".format(old_units["voltage"])] * unit_tables.voltage()[old_units["voltage"]].loc[new_units['voltage']] - df["/{}".format(old_units["current"])] = df["/{}".format(old_units["current"])] * unit_tables.current()[old_units["current"]].loc[new_units['current']] + if options['kind'] == 'biologic': + df['time/{}'.format(options['old_units']['time'])] = df["time/{}".format(options['old_units']["time"])] * unit_tables.time()[options['old_units']["time"]].loc[options['units']['time']] + df["Ewe/{}".format(options['old_units']["voltage"])] = df["Ewe/{}".format(options['old_units']["voltage"])] * unit_tables.voltage()[options['old_units']["voltage"]].loc[options['units']['voltage']] + df["/{}".format(options['old_units']["current"])] = df["/{}".format(options['old_units']["current"])] * unit_tables.current()[options['old_units']["current"]].loc[options['units']['current']] - capacity = old_units['capacity'].split('h')[0] + '.h' - df["Capacity/{}".format(capacity)] = df["Capacity/{}".format(capacity)] * (unit_tables.capacity()[old_units["capacity"]].loc[new_units["capacity"]]) + capacity = options['old_units']['capacity'].split('h')[0] + '.h' + df["Capacity/{}".format(capacity)] = df["Capacity/{}".format(capacity)] * (unit_tables.capacity()[options['old_units']["capacity"]].loc[options['units']["capacity"]]) columns = ['status_change', 'status', 'time', 'voltage', 'energy_charge', 'energy_discharge', 'current', 'capacity', 'cycle'] - if 'SpecificCapacity({}/mg)'.format(old_units['capacity']) in df.columns: - df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] = df['SpecificCapacity({}/mg)'.format(old_units['capacity'])] * unit_tables.capacity()[old_units['capacity']].loc[new_units['capacity']] / unit_tables.mass()['mg'].loc[new_units["mass"]] + if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] columns.append('specific_capacity') if 'IonsExtracted' in df.columns: @@ -469,38 +468,43 @@ def unit_conversion(df, new_units, old_units, kind): return df -def set_units(units=None): +def set_units(options: dict) -> None: # Complete the list of units - if not all are passed, then default value will be used required_units = ['time', 'current', 'voltage', 'capacity', 'mass', 'energy', 'specific_capacity'] - default_units = {'time': 'h', 'current': 'mA', 'voltage': 'V', 'capacity': 'mAh', 'mass': 'g', 'energy': 'mWh', 'specific_capacity': None} - - if not units: - units = default_units - - if units: - for unit in required_units: - if unit not in units.keys(): - units[unit] = default_units[unit] - - units['specific_capacity'] = r'{} {}'.format(units['capacity'], units['mass']) + '$^{-1}$' - - - return units - - - -def get_old_units(df, kind): - if kind=='batsmall': + default_units = { + 'time': 'h', + 'current': 'mA', + 'voltage': 'V', + 'capacity': 'mAh', + 'mass': 'g', + 'energy': 'mWh', + 'specific_capacity': None} + + if not options['units']: + options['units'] = default_units + + + aux.update_options(options=options['units'], required_options=required_units, default_options=default_units) + + options['units']['specific_capacity'] = r'{} {}'.format(options['units']['capacity'], options['units']['mass']) + '$^{-1}$' + + + +def get_old_units(df: pd.DataFrame, options: dict) -> dict: + ''' Reads a DataFrame with cycling data and determines which units have been used and returns these in a dictionary''' + + if options['kind'] == 'batsmall': + time = df.columns[0].split()[-1].strip('[]') voltage = df.columns[1].split()[-1].strip('[]') current = df.columns[2].split()[-1].strip('[]') capacity, mass = df.columns[4].split()[-1].strip('[]').split('/') old_units = {'time': time, 'current': current, 'voltage': voltage, 'capacity': capacity, 'mass': mass} - if kind=='neware': - + if options['kind']=='neware': + for column in df.columns: if 'Voltage' in column: voltage = column.split('(')[-1].strip(')') @@ -514,7 +518,7 @@ def get_old_units(df, kind): old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy} - if kind=='biologic': + if options['kind'] == 'biologic': for column in df.columns: if 'time' in column: @@ -530,8 +534,6 @@ def get_old_units(df, kind): old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy, 'time': time} - - return old_units def convert_time_string(time_string, unit='ms'): diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 0d3e2cc..dad866a 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -6,58 +6,57 @@ import numpy as np import math import nafuma.electrochemistry as ec +import nafuma.plotting as btp +import nafuma.auxillary as aux -def plot_gc(path, kind, options=None): - - # Prepare plot, and read and process data - fig, ax = prepare_gc_plot(options=options) - cycles = ec.io.read_data(path=path, kind=kind, options=options) +def plot_gc(data, options=None): # Update options - required_options = ['x_vals', 'y_vals', 'which_cycles', 'chg', 'dchg', 'colours', 'differentiate_charge_discharge', 'gradient'] - default_options = {'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', 'chg': True, 'dchg': True, 'colours': None, 'differentiate_charge_discharge': True, 'gradient': False} + required_options = ['x_vals', 'y_vals', 'which_cycles', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'rc_params', 'format_params'] + + default_options = { + 'x_vals': 'capacity', 'y_vals': 'voltage', + 'which_cycles': 'all', + 'charge': True, 'discharge': True, + 'colours': None, + 'differentiate_charge_discharge': True, + 'gradient': False, + 'rc_params': {}, + 'format_params': {}} - options = update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + # Prepare plot, and read and process data + + fig, ax = btp.prepare_plot(options=options) + data['cycles'] = ec.io.read_data(data=data, options=options) # Update list of cycles to correct indices - update_cycles_list(cycles=cycles, options=options) + update_cycles_list(cycles=data['cycles'], options=options) - colours = generate_colours(cycles=cycles, options=options) + colours = generate_colours(cycles=data['cycles'], options=options) - for i, cycle in enumerate(cycles): + for i, cycle in enumerate(data['cycles']): if i in options['which_cycles']: - if options['chg']: + if options['charge']: cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) - if options['dchg']: + if options['discharge']: cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) - fig, ax = prettify_gc_plot(fig=fig, ax=ax, options=options) + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) - return cycles, fig, ax + return data['cycles'], fig, ax -def update_options(options, required_options, default_options): - - if not options: - options = default_options - else: - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] - - return options - -def update_cycles_list(cycles, options): - - if not options: - options['which_cycles'] +def update_cycles_list(cycles, options: dict) -> None: if options['which_cycles'] == 'all': options['which_cycles'] = [i for i in range(len(cycles))] @@ -81,52 +80,6 @@ def update_cycles_list(cycles, options): options['which_cycles'] = [i-1 for i in range(which_cycles[0], which_cycles[1]+1)] - return options - - -def prepare_gc_plot(options=None): - - - # First take care of the options for plotting - set any values not specified to the default values - required_options = ['columns', 'width', 'height', 'format', 'dpi', 'facecolor'] - default_options = {'columns': 1, 'width': 14, 'format': 'golden_ratio', 'dpi': None, 'facecolor': 'w'} - - # If none are set at all, just pass the default_options - if not options: - options = default_options - options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 - options['figsize'] = (options['width'], options['height']) - - - # If options is passed, go through to fill out the rest. - else: - # Start by setting the width: - if 'width' not in options.keys(): - options['width'] = default_options['width'] - - # Then set height - check options for format. If not given, set the height to the width scaled by the golden ratio - if the format is square, set the same. This should possibly allow for the tweaking of custom ratios later. - if 'height' not in options.keys(): - if 'format' not in options.keys(): - options['height'] = options['width'] * (math.sqrt(5) - 1) / 2 - elif options['format'] == 'square': - options['height'] = options['width'] - - options['figsize'] = (options['width'], options['height']) - - # After height and width are set, go through the rest of the options to make sure that all the required options are filled - for option in required_options: - if option not in options.keys(): - options[option] = default_options[option] - - fig, ax = plt.subplots(figsize=(options['figsize']), dpi=options['dpi'], facecolor=options['facecolor']) - - linewidth = 1*options['columns'] - axeswidth = 3*options['columns'] - - plt.rc('lines', linewidth=linewidth) - plt.rc('axes', linewidth=axeswidth) - - return fig, ax def prettify_gc_plot(fig, ax, options=None): @@ -166,7 +119,7 @@ def prettify_gc_plot(fig, ax, options=None): 'title': None } - update_options(options, required_options, default_options) + aux.update_options(options, required_options, default_options) ################################################################## From 5735d011aa7573d5eb62e6c5ca063e92a7f4d613 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 22 Apr 2022 15:49:02 +0200 Subject: [PATCH 154/355] Add correct formatting of x- and y-labels for EC-plots --- nafuma/electrochemistry/plot.py | 30 ++++++++++++++++++++++++++++++ nafuma/plotting.py | 5 ++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index dad866a..a074899 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -49,12 +49,42 @@ def plot_gc(data, options=None): cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + update_labels(options) + print(options['xunit']) fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) return data['cycles'], fig, ax + +def update_labels(options): + + if 'xlabel' not in options.keys(): + options['xlabel'] = options['x_vals'].capitalize().replace('_', ' ') + + if 'ylabel' not in options.keys(): + options['ylabel'] = options['y_vals'].capitalize().replace('_', ' ') + + + if 'xunit' not in options.keys(): + if options['x_vals'] == 'capacity': + options['xunit'] = options['units']['capacity'] + elif options['x_vals'] == 'specific_capacity': + options['xunit'] = f"{options['units']['capacity']} {options['units']['mass']}$^{{-1}}$" + elif options['x_vals'] == 'time': + options['xunit'] = options['units']['time'] + elif options['x_vals'] == 'ions': + options['xunit'] = None + + + if 'yunit' not in options.keys(): + if options['y_vals'] == 'voltage': + options['yunit'] = options['units']['voltage'] + + + + def update_cycles_list(cycles, options: dict) -> None: diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 2233ab7..416c1cb 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -135,7 +135,10 @@ def adjust_plot(fig, ax, options): ax.set_ylabel('') if not options['hide_x_labels']: - ax.set_xlabel(f'{options["xlabel"]}') + if not options['xunit']: + ax.set_xlabel(f'{options["xlabel"]}') + else: + ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]') else: ax.set_xlabel('') From 7af1dc4228fe81fdb5943650bdbcd8e92f3c0904 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 22 Apr 2022 16:31:04 +0200 Subject: [PATCH 155/355] Add initial interactive capabilities to GC-plots --- nafuma/electrochemistry/plot.py | 44 +++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index a074899..a7e319e 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -5,40 +5,55 @@ import pandas as pd import numpy as np import math +import ipywidgets as widgets +from IPython.display import display + import nafuma.electrochemistry as ec import nafuma.plotting as btp import nafuma.auxillary as aux + + def plot_gc(data, options=None): # Update options - required_options = ['x_vals', 'y_vals', 'which_cycles', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'rc_params', 'format_params'] - + required_options = ['x_vals', 'y_vals', 'which_cycles', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params'] default_options = { 'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', 'charge': True, 'discharge': True, 'colours': None, 'differentiate_charge_discharge': True, - 'gradient': False, + 'gradient': False, + 'interactive': False, + 'interactive_session_active': False, 'rc_params': {}, 'format_params': {}} options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - # Prepare plot, and read and process data + - fig, ax = btp.prepare_plot(options=options) - data['cycles'] = ec.io.read_data(data=data, options=options) + if not 'cycles' in data.keys(): + data['cycles'] = ec.io.read_data(data=data, options=options) # Update list of cycles to correct indices update_cycles_list(cycles=data['cycles'], options=options) colours = generate_colours(cycles=data['cycles'], options=options) + if options['interactive']: + options['interactive'], options['interactive_session_active'] = False, True + plot_gc_interactive(data=data, options=options) + return + + + # Prepare plot, and read and process data + + fig, ax = btp.prepare_plot(options=options) for i, cycle in enumerate(data['cycles']): if i in options['which_cycles']: @@ -50,12 +65,25 @@ def plot_gc(data, options=None): update_labels(options) - print(options['xunit']) fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) - return data['cycles'], fig, ax + #if options['interactive_session_active']: + + return data['cycles'], fig, ax + + +def plot_gc_interactive(data, options): + + w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_gc), data=widgets.fixed(data), options=widgets.fixed(options), + charge=widgets.ToggleButton(value=True), + discharge=widgets.ToggleButton(value=True) + ) + + options['widget'] = w + + display(w) def update_labels(options): From d30c9c3b162d230cc84c0b53fd7d8290a76cd46a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 22 Apr 2022 16:45:28 +0200 Subject: [PATCH 156/355] Add change of x-values in interactive mode --- nafuma/electrochemistry/plot.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index a7e319e..9e80471 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -64,7 +64,10 @@ def plot_gc(data, options=None): cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) - update_labels(options) + if options['interactive_session_active']: + update_labels(options, force=True) + else: + update_labels(options) fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) @@ -78,7 +81,8 @@ def plot_gc_interactive(data, options): w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_gc), data=widgets.fixed(data), options=widgets.fixed(options), charge=widgets.ToggleButton(value=True), - discharge=widgets.ToggleButton(value=True) + discharge=widgets.ToggleButton(value=True), + x_vals=widgets.Dropdown(options=['specific_capacity', 'capacity', 'ions', 'voltage', 'time', 'energy'], value='specific_capacity', description='X-values') ) options['widget'] = w @@ -86,16 +90,16 @@ def plot_gc_interactive(data, options): display(w) -def update_labels(options): +def update_labels(options, force=False): - if 'xlabel' not in options.keys(): + if 'xlabel' not in options.keys() or force: options['xlabel'] = options['x_vals'].capitalize().replace('_', ' ') - if 'ylabel' not in options.keys(): + if 'ylabel' not in options.keys() or force: options['ylabel'] = options['y_vals'].capitalize().replace('_', ' ') - if 'xunit' not in options.keys(): + if 'xunit' not in options.keys() or force: if options['x_vals'] == 'capacity': options['xunit'] = options['units']['capacity'] elif options['x_vals'] == 'specific_capacity': @@ -106,7 +110,7 @@ def update_labels(options): options['xunit'] = None - if 'yunit' not in options.keys(): + if 'yunit' not in options.keys() or force: if options['y_vals'] == 'voltage': options['yunit'] = options['units']['voltage'] From c9660109cb32297838e1c3ac100401c4a56a86aa Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 27 Apr 2022 10:37:02 +0200 Subject: [PATCH 157/355] adding a smoothing function --- nafuma/xanes/calib.py | 59 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index ad3443f..fb436cb 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -2,9 +2,11 @@ import pandas as pd import numpy as np import os import matplotlib.pyplot as plt -import beamtime.auxillary as aux -import beamtime.xanes as xas -import beamtime.xanes.io as io +import nafuma.auxillary as aux +import nafuma.xanes as xas +import nafuma.xanes.io as io +from scipy.signal import savgol_filter + def rbkerbest(): print("ROSENBORG!<3") @@ -24,11 +26,10 @@ def finding_edge(df): edge='Ni' return(edge) -<<<<<<< HEAD:beamtime/xanes/calib.py #def pre_edge_subtraction(df,filenames, options={}): def test(innmat): df_test= xas.io.put_in_dataframe(innmat) - print(df_test) + #print(df_test) def pre_edge_subtraction(path, options={}): required_options = ['print','troubleshoot'] @@ -41,11 +42,6 @@ def pre_edge_subtraction(path, options={}): filenames = xas.io.get_filenames(path) df= xas.io.put_in_dataframe(path) edge=finding_edge(df) -======= -def split_xanes_scan(filename, destination=None, replace=False): - #root is the path to the beamtime-folder - #destination should be the path to the processed data ->>>>>>> master:nafuma/xanes/calib.py #Defining the end of the region used to define the background, thus start of the edge #implement widget @@ -154,4 +150,45 @@ def post_edge_normalization(path, options={}): df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) plt.axvline(x = min(df_end["ZapEnergy"])) - return df_bkgd_sub, df_postedge \ No newline at end of file + return df_bkgd_sub, df_postedge, filenames, edge + +def smoothing(path, options={}): + required_options = ['print','window_length','polyorder'] + default_options = { + 'print': False, + 'window_length': 3, + 'polyorder': 2 + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + df_bkgd_sub, df_postedge, filenames, edge = post_edge_normalization(path) + #================= SMOOTHING + df_smooth = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) + df_default = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) + #df_smooth[filenames] = df_bkgd_sub.iloc[:,2].rolling(window=rolling_av).mean() + #df_smooth[filenames] = df_smooth[filenames].shift(-int((rolling_av)/2)) + for filename in filenames: + x=savgol_filter(df_bkgd_sub[filename], options['window_length'],options['polyorder']) + df_smooth[filename] = x + x_default=savgol_filter(df_bkgd_sub[filename],default_options['window_length'],default_options['polyorder']) + df_default[filename] = x_default + + if options['print'] == True: + fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) + x_range_zoom=[6.54,6.55] #make into widget + y_range_zoom=[20000,80000] #make into widget + + df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, ax=ax1, color="Red") + df_smooth.plot(x = "ZapEnergy",y=filenames, ax=ax1, color="Blue") + ax1.set_xlim(x_range_zoom) + ax1.set_ylim(y_range_zoom) + ax1.set_title("Smoothed curve (blue) vs data (red) used for further analysis") + + df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, ax=ax2, color="Red") + df_default.plot(x = "ZapEnergy",y=filenames, ax=ax2, color="Blue") + ax2.set_xlim(x_range_zoom) + ax2.set_ylim(y_range_zoom) + ax2.set_title("Smoothed curve (blue) vs data (red) using default window_length and polyorder") + + + From 8d8cad966df30b631fb14fe98d4cd8482b3349dd Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 10 May 2022 17:29:22 +0200 Subject: [PATCH 158/355] Allow plotting of multiple beamline scans (quickfix) --- nafuma/xrd/io.py | 2 +- nafuma/xrd/plot.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index b3f3951..2511ff1 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -59,7 +59,7 @@ def integrate_1d(data, options={}, index=0): # Get image array from filename if not passed - if 'image' not in data.keys(): + if 'image' not in data.keys() or not data['image']: data['image'] = get_image_array(data['path'][index]) # Instanciate the azimuthal integrator from pyFAI from the calibrant (.poni-file) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 8dd6a27..c45582b 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -75,6 +75,9 @@ def plot_diffractogram(data, options={}): data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength + # FIXME This is a quick fix as the image is not reloaded when passing multiple beamline datasets + data['image'] = None + # Sets the xlim if this has not bee specified if not options['xlim']: options['xlim'] = [data['diffractogram'][0][options['x_vals']].min(), data['diffractogram'][0][options['x_vals']].max()] From 8f94fa4dc6d70d3698e7f7a9ba7491a73084daec Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 10 May 2022 17:30:00 +0200 Subject: [PATCH 159/355] Remove outcommented lines in splice_cycles --- nafuma/electrochemistry/io.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 5abff3b..8256749 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -178,19 +178,17 @@ def process_batsmall_data(df, options=None): def splice_cycles(df, options: dict) -> pd.DataFrame: ''' Splices two cycles together - if e.g. one charge cycle are split into several cycles due to change in parameters. - Incomplete, only accomodates BatSmall so far.''' + Incomplete, only accomodates BatSmall so far, and only for charge.''' if options['kind'] == 'batsmall': # Creates masks for charge and discharge curves chg_mask = df['current'] >= 0 - dchg_mask = df['current'] < 0 # Loop through all the cycling steps, change the current and capacities in the for i in range(df["count"].max()): sub_df = df.loc[df['count'] == i+1] sub_df_chg = sub_df.loc[chg_mask] - #sub_df_dchg = sub_df.loc[dchg_mask] # get indices where the program changed chg_indices = sub_df_chg[sub_df_chg["comment"].str.contains("program")==True].index.to_list() @@ -202,34 +200,17 @@ def splice_cycles(df, options: dict) -> pd.DataFrame: if chg_indices: last_chg = chg_indices.pop() - - - #dchg_indices = sub_df_dchg[sub_df_dchg["comment"].str.contains("program")==True].index.to_list() - #if dchg_indices: - # del dchg_indices[0] - - - + if chg_indices: for i in chg_indices: add = df['specific_capacity'].iloc[i-1] df['specific_capacity'].iloc[i:last_chg] = df['specific_capacity'].iloc[i:last_chg] + add - #if dchg_indices: - # for i in dchg_indices: - # add = df['specific_capacity'].iloc[i-1] - # df['specific_capacity'].iloc[i:last_dchg] = df['specific_capacity'].iloc[i:last_dchg] + add - - - - return df - - def process_neware_data(df, options={}): """ Takes data from NEWARE in a DataFrame as read by read_neware() and converts units, adds columns and splits into cycles. From 23f037c0ef3ec65ef93834bda643dbc928068d86 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 11 May 2022 18:35:11 +0200 Subject: [PATCH 160/355] Add function to strip headers from .xy --- nafuma/xrd/io.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 2511ff1..495f2d1 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -326,6 +326,39 @@ def read_xy(data, options={}, index=0): return diffractogram, wavelength + + +def strip_headers_from_xy(path: str, filename=None) -> None: + ''' Strips headers from a .xy-file''' + + + xy = [] + with open(path, 'r') as f: + lines = f.readlines() + + headerlines = 0 + for line in lines: + if line[0] == '#': + headerlines += 1 + else: + xy.append(line) + + + if not filename: + ext = path.split('.')[-1] + filename = path.split(f'.{ext}')[0] + f'_noheaders.{ext}' + + print(filename) + + with open(filename, 'w') as f: + for line in xy: + f.write(line) + + + + + + def read_data(data, options={}, index=0): beamline_extensions = ['mar3450', 'edf', 'cbf'] From 55b22d5bf19047578862b3f6d443b086d11d6ee4 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 12 May 2022 19:17:43 +0200 Subject: [PATCH 161/355] Small improvements to integrate function --- nafuma/xrd/io.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 495f2d1..6b5a762 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -62,6 +62,7 @@ def integrate_1d(data, options={}, index=0): if 'image' not in data.keys() or not data['image']: data['image'] = get_image_array(data['path'][index]) + # Instanciate the azimuthal integrator from pyFAI from the calibrant (.poni-file) ai = pyFAI.load(data['calibrant']) @@ -72,6 +73,9 @@ def integrate_1d(data, options={}, index=0): if not os.path.isdir(options['extract_folder']): os.makedirs(options['extract_folder']) + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + res = ai.integrate1d(data['image'], options['nbins'], unit=options['unit'], filename=filename) @@ -348,7 +352,6 @@ def strip_headers_from_xy(path: str, filename=None) -> None: ext = path.split('.')[-1] filename = path.split(f'.{ext}')[0] + f'_noheaders.{ext}' - print(filename) with open(filename, 'w') as f: for line in xy: From da8907083bc138a03cc7308d14187723d6b3a9bd Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 19 May 2022 21:32:10 +0200 Subject: [PATCH 162/355] Fix arcsin issue and allow plotting w/o reflection data --- nafuma/xrd/io.py | 42 ++++++++++++++++++++++++++++++++++-------- nafuma/xrd/plot.py | 25 ++++++++++++++++++------- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 6b5a762..56154ea 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -80,7 +80,8 @@ def integrate_1d(data, options={}, index=0): res = ai.integrate1d(data['image'], options['nbins'], unit=options['unit'], filename=filename) data['path'][index] = filename - diffractogram, wavelength = read_xy(data=data, options=options, index=index) + diffractogram, _ = read_xy(data=data, options=options, index=index) + wavelength = find_wavelength_from_poni(path=data['calibrant']) if not options['save']: os.remove(filename) @@ -282,8 +283,12 @@ def read_brml(data, options={}, index=0): #if 'wavelength' not in data.keys(): # Find wavelength - for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): - wavelength = float(chain.attrib['Value']) + + if not data['wavelength'][index]: + for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): + wavelength = float(chain.attrib['Value']) + else: + wavelength = data['wavelength'][index] diffractogram = pd.DataFrame(diffractogram) @@ -306,7 +311,11 @@ def read_xy(data, options={}, index=0): #if 'wavelength' not in data.keys(): # Get wavelength from scan - wavelength = find_wavelength_from_xy(path=data['path'][index]) + + if not data['wavelength'][index]: + wavelength = find_wavelength_from_xy(path=data['path'][index]) + else: + wavelength = data['wavelength'][index] with open(data['path'][index], 'r') as f: position = 0 @@ -378,7 +387,7 @@ def read_data(data, options={}, index=0): - + if options['offset'] or options['normalise']: # Make copy of the original intensities before any changes are made through normalisation or offset, to easily revert back if need to update. diffractogram['I_org'] = diffractogram['I'] @@ -387,6 +396,7 @@ def read_data(data, options={}, index=0): diffractogram = apply_offset(diffractogram, wavelength, index, options) + diffractogram = translate_wavelengths(data=diffractogram, wavelength=wavelength) return diffractogram, wavelength @@ -506,7 +516,7 @@ def translate_wavelengths(data: pd.DataFrame, wavelength: float, to_wavelength=N data['2th_cuka'] = np.NAN - data['2th_cuka'].loc[data['2th'] <= max_2th_cuka] = 2*np.arcsin(cuka/wavelength * np.sin((data['2th']/2) * np.pi/180)) * 180/np.pi + data['2th_cuka'].loc[data['2th'] <= max_2th_cuka] = 2*np.arcsin(cuka/wavelength * np.sin((data['2th'].loc[data['2th'] <= max_2th_cuka]/2) * np.pi/180)) * 180/np.pi # Translate to MoKalpha moka = 0.71073 # Å @@ -518,7 +528,7 @@ def translate_wavelengths(data: pd.DataFrame, wavelength: float, to_wavelength=N data['2th_moka'] = np.NAN - data['2th_moka'].loc[data['2th'] <= max_2th_moka] = 2*np.arcsin(moka/wavelength * np.sin((data['2th']/2) * np.pi/180)) * 180/np.pi + data['2th_moka'].loc[data['2th'] <= max_2th_moka] = 2*np.arcsin(moka/wavelength * np.sin((data['2th'].loc[data['2th'] <= max_2th_moka]/2) * np.pi/180)) * 180/np.pi # Convert to other parameters @@ -537,7 +547,7 @@ def translate_wavelengths(data: pd.DataFrame, wavelength: float, to_wavelength=N data['2th'] = np.NAN - data['2th'].loc[data['2th_cuka'] <= max_2th] = 2*np.arcsin(to_wavelength/cuka * np.sin((data['2th_cuka']/2) * np.pi/180)) * 180/np.pi + data['2th'].loc[data['2th_cuka'] <= max_2th] = 2*np.arcsin(to_wavelength/cuka * np.sin((data['2th_cuka'].loc[data['2th_cuka'] <= max_2th]/2) * np.pi/180)) * 180/np.pi @@ -564,6 +574,22 @@ def find_wavelength_from_xy(path): elif 'Wavelength' in line: wavelength = float(line.split()[2])*10**10 + else: + wavelength = None + + + + return wavelength + + +def find_wavelength_from_poni(path): + + with open(path, 'r') as f: + lines = f.readlines() + + for line in lines: + if 'Wavelength' in line: + wavelength = float(line.split()[-1])*10**10 return wavelength diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index c45582b..b562eeb 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -13,7 +13,6 @@ import nafuma.xrd as xrd import nafuma.auxillary as aux import nafuma.plotting as btp - def plot_diffractogram(data, options={}): ''' Plots a diffractogram. @@ -67,7 +66,14 @@ def plot_diffractogram(data, options={}): if not 'diffractogram' in data.keys(): # Initialise empty list for diffractograms and wavelengths data['diffractogram'] = [None for _ in range(len(data['path']))] - data['wavelength'] = [None for _ in range(len(data['path']))] + + # If wavelength is not manually passed it should be automatically gathered from the .xy-file + if 'wavelength' not in data.keys(): + data['wavelength'] = [None for _ in range(len(data['path']))] + else: + # If only a single value is passed it should be set to be the same for all diffractograms passed + if not isinstance(data['wavelength'], list): + data['wavelength'] = [data['wavelength'] for _ in range(len(data['path']))] for index in range(len(data['path'])): diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) @@ -117,7 +123,7 @@ def plot_diffractogram(data, options={}): options['reflections_data'] = [options['reflections_data']] # Determine number of subplots and height ratios between them - if len(options['reflections_data']) >= 1: + if options['reflections_data'] and len(options['reflections_data']) >= 1: options = determine_grid_layout(options=options) @@ -334,10 +340,10 @@ def plot_diffractogram_interactive(data, options): 'heatmap_default': {'min': xminmax['heatmap'][0], 'max': xminmax['heatmap'][1], 'value': [xminmax['heatmap'][0], xminmax['heatmap'][1]], 'step': 10} }, 'ylim': { - 'w': widgets.FloatRangeSlider(value=[yminmax['start'][2], yminmax['start'][3]], min=yminmax['start'][0], max=yminmax['start'][1], step=0.5, layout=widgets.Layout(width='95%')), + 'w': widgets.FloatRangeSlider(value=[yminmax['start'][2], yminmax['start'][3]], min=yminmax['start'][0], max=yminmax['start'][1], step=0.01, layout=widgets.Layout(width='95%')), 'state': 'heatmap' if options['heatmap'] else 'diff', - 'diff_default': {'min': yminmax['diff'][0], 'max': yminmax['diff'][1], 'value': [yminmax['diff'][2], yminmax['diff'][3]], 'step': 0.1}, - 'heatmap_default': {'min': yminmax['heatmap'][0], 'max': yminmax['heatmap'][1], 'value': [yminmax['heatmap'][0], yminmax['heatmap'][1]], 'step': 0.1} + 'diff_default': {'min': yminmax['diff'][0], 'max': yminmax['diff'][1], 'value': [yminmax['diff'][2], yminmax['diff'][3]], 'step': 0.01}, + 'heatmap_default': {'min': yminmax['heatmap'][0], 'max': yminmax['heatmap'][1], 'value': [yminmax['heatmap'][0], yminmax['heatmap'][1]], 'step': 0.01} } } @@ -359,7 +365,12 @@ def plot_diffractogram_interactive(data, options): w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), - xlim=options['widgets']['xlim']['w']) + heatmap=widgets.ToggleButton(value=options['heatmap']), + x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), + xlim=options['widgets']['xlim']['w'], + ylim=options['widgets']['ylim']['w'], + offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01, description='offset_y'), + offset_x=widgets.BoundedFloatText(value=options['offset_x'], min=-1, max=1, step=0.01, description='offset_x')) options['widget'] = w From a1106ac88d68f3c936365ff1907f0b67ef150733 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 20 May 2022 17:41:54 +0200 Subject: [PATCH 163/355] Add option to use mask with pyfai-integrations --- nafuma/xrd/io.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 56154ea..9b9e308 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -40,12 +40,13 @@ def integrate_1d(data, options={}, index=0): df: DataFrame contianing 1D diffractogram if option 'return' is True ''' - required_options = ['unit', 'nbins', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite', 'extract_folder'] + required_options = ['unit', 'npt', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite', 'extract_folder', 'error_model'] default_options = { 'unit': '2th_deg', - 'nbins': 3000, + 'npt': 3000, 'extract_folder': 'tmp', + 'error_model': None, 'save': False, 'save_filename': None, 'save_extension': '_integrated.xy', @@ -61,6 +62,13 @@ def integrate_1d(data, options={}, index=0): # Get image array from filename if not passed if 'image' not in data.keys() or not data['image']: data['image'] = get_image_array(data['path'][index]) + + + # Load mask + if 'mask' in data.keys(): + mask = get_image_array(data['mask']) + else: + mask = None # Instanciate the azimuthal integrator from pyFAI from the calibrant (.poni-file) @@ -76,8 +84,10 @@ def integrate_1d(data, options={}, index=0): if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) + - res = ai.integrate1d(data['image'], options['nbins'], unit=options['unit'], filename=filename) + + res = ai.integrate1d(data['image'], npt=options['npt'], mask=mask, error_model=options['error_model'], unit=options['unit'], filename=filename) data['path'][index] = filename diffractogram, _ = read_xy(data=data, options=options, index=index) @@ -312,7 +322,7 @@ def read_xy(data, options={}, index=0): #if 'wavelength' not in data.keys(): # Get wavelength from scan - if not data['wavelength'][index]: + if not 'wavelength' in data.keys() or data['wavelength'][index]: wavelength = find_wavelength_from_xy(path=data['path'][index]) else: wavelength = data['wavelength'][index] From ebc77c1b9e9a33dd430eaebefbcb8fe4692ed288 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 13 Jun 2022 13:50:24 +0200 Subject: [PATCH 164/355] Extract data from .brml heatscans and save as .xy --- nafuma/xrd/io.py | 112 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 9b9e308..ef77b0e 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -4,6 +4,7 @@ import pandas as pd import numpy as np import os import shutil +import sys import zipfile import xml.etree.ElementTree as ET @@ -317,6 +318,117 @@ def read_brml(data, options={}, index=0): return diffractogram, wavelength +def read_htxrd(data, options={}, index=0): + + required_options = ['extract_folder', 'save_folder', 'save_filename'] + default_options = { + 'extract_folder': 'tmp', + 'save_folder': None, + 'save_filename': None + } + + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + # Extract the RawData0.xml file from the brml-file + with zipfile.ZipFile(data['path'][index], 'r') as brml: + for info in brml.infolist(): + if "RawData" in info.filename: + brml.extract(info.filename, options['extract_folder']) + + + # Get all filenames + files = os.listdir(os.path.join(options['extract_folder'], 'Experiment0')) + + # initalise empty list to store all DataFrames + diffractograms = [] + wavelengths = [] + + # Loop through all RawData-files and extract all data and temperatures + for i, file in enumerate(files): + + # Create all filenames as strings + filename = os.path.join('tmp/Experiment0/', f'RawData{i}.xml') + + # Parse the .xml-files + tree = ET.parse(filename) + root = tree.getroot() + + # initalise empty list to store data from this particular scan + diffractogram = [] + + + for chain in root.findall('./DataRoutes/DataRoute'): + + scantypes = chain.findall('ScanInformation') + + for scantype in scantypes: + if scantype.get('VisibleName') == 'Still (TCU1000N)': + continue + + else: + if chain.get('RouteFlag') == 'Final': + for scandata in chain.findall('Datum'): + scandata = scandata.text.split(',') + twotheta, intensity, temperature = float(scandata[2]), float(scandata[3]), float(scandata[5]) + + diffractogram.append({'2th': twotheta, 'I': intensity, 'T': temperature}) + + diffractogram = pd.DataFrame(diffractogram) + diffractograms.append(diffractogram) + + + if not data['wavelength'][index]: + for chain in root.findall('./FixedInformation/Instrument/PrimaryTracks/TrackInfoData/MountedOptics/InfoData/Tube/WaveLengthAlpha1'): + wavelength = float(chain.attrib['Value']) + else: + wavelength = data['wavelength'][index] + + wavelengths.append(wavelength) + + + if options['save_folder']: + for i, (diffractogram, wavelength) in enumerate(zip(diffractograms, wavelengths)): + if not options['save_filename']: + filename = os.path.basename(data['path'][index]).split('.')[0] + '_' + str(i).zfill(4) +'.xy' + else: + filename = options['save_filename'] + '_' + str(i).zfill(4) +'.xy' + + + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + save_htxrd_as_xy(diffractogram, wavelength, filename, options['save_folder']) + + + + + + shutil.rmtree(options['extract_folder']) + + return diffractograms, wavelengths + +def save_htxrd_as_xy(diffractogram, wavelength, filename, save_path): + + headers = '\n'.join( + [line for line in + [f'# Temperature {np.round(diffractogram["T"].mean())}', + f'# Wavelength {wavelength}', + ] + ] + ) + + diffractogram = diffractogram.drop('T', axis=1) + + with open(os.path.join(save_path, filename), 'w') as f: + for line in headers: + f.write(line) + + f.write('\n') + + diffractogram.to_csv(f, index=False, sep='\t') + + def read_xy(data, options={}, index=0): #if 'wavelength' not in data.keys(): From 8ce15574398d848354f5e4ad2983d32e01eaf7f4 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 15 Jun 2022 10:00:13 +0200 Subject: [PATCH 165/355] finding e0 --- nafuma/xanes/calib.py | 191 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 183 insertions(+), 8 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index fb436cb..f7b57c6 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -44,9 +44,19 @@ def pre_edge_subtraction(path, options={}): edge=finding_edge(df) #Defining the end of the region used to define the background, thus start of the edge + + #######================================================================================================================================================ + #Trying to implement automatical region determination based on an estimate of the edge shift + #print(df) + #estimated_edge_shift, df_diff, df_diff_max = find_pos_maxdiff(df, filenames,options=options) + + #print(estimated_edge_shift) + #estimated_edge_shift + ###========================================================================================================================================================================= #implement widget if edge == 'Mn': edge_start = 6.42 + #edge_start = estimated_edge_shift if edge == 'Ni': edge_start = 8.3 @@ -120,7 +130,7 @@ def post_edge_normalization(path, options={}): } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - df_bkgd_sub,filenames,edge = pre_edge_subtraction(path) + df_bkgd_sub,filenames,edge = pre_edge_subtraction(path, options=options) #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge #Implement widget if edge == 'Mn': @@ -161,34 +171,199 @@ def smoothing(path, options={}): } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - df_bkgd_sub, df_postedge, filenames, edge = post_edge_normalization(path) + df_bkgd_sub, df_postedge, filenames, edge = post_edge_normalization(path,options=options) #================= SMOOTHING df_smooth = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) df_default = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) #df_smooth[filenames] = df_bkgd_sub.iloc[:,2].rolling(window=rolling_av).mean() #df_smooth[filenames] = df_smooth[filenames].shift(-int((rolling_av)/2)) for filename in filenames: - x=savgol_filter(df_bkgd_sub[filename], options['window_length'],options['polyorder']) - df_smooth[filename] = x + x_smooth=savgol_filter(df_bkgd_sub[filename], options['window_length'],options['polyorder']) + df_smooth[filename] = x_smooth x_default=savgol_filter(df_bkgd_sub[filename],default_options['window_length'],default_options['polyorder']) df_default[filename] = x_default + + + #printing the smoothed curves vs data if options['print'] == True: + + ## ================================================ + #df_diff = pd.DataFrame(df_smooth["ZapEnergy"]) + #df_diff_estimated_max = df_diff[filenames].dropna().max() + + + #estimated_edge_shift=df_diff.loc[df_diff[filenames] == df_diff_max,'ZapEnergy'].values[0] + # ========================================== + + fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) x_range_zoom=[6.54,6.55] #make into widget y_range_zoom=[20000,80000] #make into widget - df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, ax=ax1, color="Red") + df_bkgd_sub.plot.scatter(x = "ZapEnergy",y=filenames, ax=ax1, color="Red") df_smooth.plot(x = "ZapEnergy",y=filenames, ax=ax1, color="Blue") ax1.set_xlim(x_range_zoom) ax1.set_ylim(y_range_zoom) ax1.set_title("Smoothed curve (blue) vs data (red) used for further analysis") - df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, ax=ax2, color="Red") - df_default.plot(x = "ZapEnergy",y=filenames, ax=ax2, color="Blue") + df_bkgd_sub.plot.scatter(x = "ZapEnergy",y=filenames, ax=ax2, color="Red") + df_default.plot(x = "ZapEnergy",y=filenames, ax=ax2, color="Green") ax2.set_xlim(x_range_zoom) ax2.set_ylim(y_range_zoom) - ax2.set_title("Smoothed curve (blue) vs data (red) using default window_length and polyorder") + ax2.set_title("Smoothed curve (green) vs data (red) using default window_length and polyorder") + return df_smooth, filenames + +def find_pos_maxdiff(df, filenames,options={}): + #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. + required_options = ['print','periods'] + default_options = { + 'print': False, + 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #making new dataframe to keep the differentiated data + df_diff = pd.DataFrame(df["ZapEnergy"]) + df_diff[filenames]=df[filenames].diff(periods=options['periods']) + + #shifting column values up so that average differential fits right between the points used in the calculation + df_diff[filenames]=df_diff[filenames].shift(-int(options['periods']/2)) + df_diff_max = df_diff[filenames].dropna().max() + estimated_edge_shift =df_diff.loc[df_diff[filenames] == df_diff_max,'ZapEnergy'].values[0] + + return estimated_edge_shift, df_diff, df_diff_max + +def find_nearest(array, value): + #function to find the value closes to "value" in an "array" + array = np.asarray(array) + idx = (np.abs(array - value)).argmin() + return array[idx] + +def finding_e0(path, options={}): + required_options = ['print','periods'] + default_options = { + 'print': False, + 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + df_smooth, filenames = smoothing(path, options=options) #This way the smoothing is printed as long as the "finding e0" is printed. + + if options['periods'] % 2 == 1: + print("NB!!!!!!!!!!!!!!!!! Periods needs to be an even number for the shifting of values to work properly") + ###df_diff = pd.DataFrame(df_smooth["ZapEnergy"]) # + if len(filenames) == 1: + filenames=filenames[0] + else: + print("MORE THAN ONE FILE --> generalize") + + ##### + estimated_edge_shift, df_diff, df_diff_max = find_pos_maxdiff(df_smooth, filenames,options=options) + print(estimated_edge_shift) + #### + ###df_diff[filenames]=df_smooth[filenames].diff(periods=options['periods']) # + df_doublediff=pd.DataFrame(df_smooth["ZapEnergy"]) + df_doublediff[filenames]=df_diff[filenames].diff(periods=options['periods']) + + if options['print'] == True: + fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) + + df_diff.plot(x = "ZapEnergy",y=filenames, ax=ax1) #defining x and y + df_doublediff.plot(x = "ZapEnergy",y=filenames,ax=ax2) #defining x and y + + #shifting column values up so that average differential fits right between the points used in the calculation + #df_diff[filenames]=df_diff[filenames].shift(-int(options['periods']/2)) # + df_doublediff[filenames]=df_doublediff[filenames].shift(-int(options['periods'])) + + #finding maximum value to maneuver to the correct part of the data set + #df_diff_max = df_diff[filenames].dropna().max() + + + estimated_edge_shift=df_diff.loc[df_diff[filenames] == df_diff_max,'ZapEnergy'].values[0] + + fit_region = 0.0004 + df_diff_edge=df_diff.loc[(df_diff["ZapEnergy"] < estimated_edge_shift+fit_region)]# and (df_diff["ZapEnergy"] > estimated_edge_shift-0.05)] + df_diff_edge=df_diff_edge.loc[(df_diff["ZapEnergy"] > estimated_edge_shift-fit_region)] + + + + + df_doublediff_edge=df_doublediff.loc[(df_doublediff["ZapEnergy"] < estimated_edge_shift+fit_region)]# and (df_diff["ZapEnergy"] > estimated_edge_shift-0.05)] + df_doublediff_edge=df_doublediff_edge.loc[(df_doublediff["ZapEnergy"] > estimated_edge_shift-fit_region)] + #df_diff_edge=df_diff.loc[(df_diff["ZapEnergy"] > estimated_edge_shift-0.15) and (df_diff["ZapEnergy"] < estimated_edge_shift+0.15)] + + #df_diff_edge=df_diff.loc[df_diff["ZapEnergy"] > estimated_edge_shift-0.15] + #print(df_diff_edge) + if options['print'] == True: + fig, (ax3,ax4) = plt.subplots(1,2,figsize=(15,5)) + + df_diff_edge.plot(x = "ZapEnergy",y=filenames,ax=ax3) #defining x and y + ax3.set_title("Zoomed into edge region (derivative))") + ax3.axvline(x = estimated_edge_shift) + + df_doublediff_edge.plot(x = "ZapEnergy",y=filenames,ax=ax4,kind="scatter") #defining x and y + ax4.set_title("Zoomed into edge region (double derivative)") + ax4.axvline(x = estimated_edge_shift) + ax4.axhline(0) + + #ax1.set_xlim([estimated_edge_shift-fit_region,estimated_edge_shift+fit_region]) + #ax1.set_title("not sure what this is tbh") + + #ax2.set_xlim([estimated_edge_shift-fit_region,estimated_edge_shift+fit_region]) + #ax2.set_title("not sure what this is either tbh") + + #============== + #df_smooth=df_smooth2 + #================= + + + + + #========================== fitting first differential ========== + df_diff = df_diff[df_diff[filenames].notna()] + + #fitting a function to the chosen interval + d = np.polyfit(df_diff_edge["ZapEnergy"],df_diff_edge[filenames],2) + function_diff = np.poly1d(d) + + x_diff=np.linspace(df_diff_edge["ZapEnergy"].iloc[0],df_diff_edge["ZapEnergy"].iloc[-1],num=1000) + y_diff=function_diff(x_diff) + #print(df_diff_edge["ZapEnergy"].iloc[-1]) + if options['print'] == True: + ax3.plot(x_diff,y_diff,color='Green') + + #y_diff_max=np.amax(y_diff,0) + y_diff_max_index = np.where(y_diff == np.amax(y_diff)) + #print(y_diff_max_index[0]) + edge_shift_diff=float(x_diff[y_diff_max_index]) + print("Edge shift estimated by the differential maximum is "+str(round(edge_shift_diff,5))) + if options['print'] == True: + ax3.axvline(x=edge_shift_diff,color="green") + #print(df_doublediff_edge["ZapEnergy"].iloc[0]) + #ax4.plot(x_doublediff,y_doublediff,color='Green')) + + + #fitting double differentiate + df_doublediff = df_doublediff[df_doublediff[filenames].notna()] + d = np.polyfit(df_doublediff_edge["ZapEnergy"],df_doublediff_edge[filenames],2) + function_doublediff = np.poly1d(d) + + x_doublediff=np.linspace(df_doublediff_edge["ZapEnergy"].iloc[0],df_doublediff_edge["ZapEnergy"].iloc[-1],num=10000) + y_doublediff=function_doublediff(x_doublediff) + + if options['print'] == True: + ax4.plot(x_doublediff,y_doublediff,color='Green') + + y_doublediff_zero=find_nearest(y_doublediff,0) + y_doublediff_zero_index = np.where(y_doublediff == y_doublediff_zero) + + edge_shift_doublediff=float(x_doublediff[y_doublediff_zero_index]) + + print("Edge shift estimated by the double differential zero-point is "+str(round(edge_shift_doublediff,5))) + if options['print'] == True: + ax4.axvline(x=edge_shift_doublediff,color="green") + From 7676bd06af5720bb10cfea269e9d38a20e75b8a2 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 15 Jun 2022 13:21:26 +0200 Subject: [PATCH 166/355] Adding FIXME's --- nafuma/xanes/calib.py | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index f7b57c6..f11ff93 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -7,18 +7,12 @@ import nafuma.xanes as xas import nafuma.xanes.io as io from scipy.signal import savgol_filter -def rbkerbest(): - print("ROSENBORG!<3") - -#def split_xanes_scan(filename, destination=None): - - # with open(filename, 'r') as f: - ##Better to make a new function that loops through the files, and performing the split_xanes_scan on -#Tryiung to make a function that can decide which edge it is based on the first ZapEnergy-value +#Trying to make a function that can decide which edge it is based on the first ZapEnergy-value def finding_edge(df): + #FIXME add Fe and Co if 5.9 < df["ZapEnergy"][0] < 6.5: edge='Mn' return(edge) @@ -26,12 +20,8 @@ def finding_edge(df): edge='Ni' return(edge) -#def pre_edge_subtraction(df,filenames, options={}): -def test(innmat): - df_test= xas.io.put_in_dataframe(innmat) - #print(df_test) - def pre_edge_subtraction(path, options={}): + #FIXME add log-file instead of the troubleshoot-option required_options = ['print','troubleshoot'] default_options = { 'print': False, @@ -46,7 +36,7 @@ def pre_edge_subtraction(path, options={}): #Defining the end of the region used to define the background, thus start of the edge #######================================================================================================================================================ - #Trying to implement automatical region determination based on an estimate of the edge shift + #FIXME Trying to implement automatical region determination based on an estimate of the edge shift #print(df) #estimated_edge_shift, df_diff, df_diff_max = find_pos_maxdiff(df, filenames,options=options) @@ -122,8 +112,8 @@ def pre_edge_subtraction(path, options={}): return df_bkgd_sub,filenames,edge -def post_edge_normalization(path, options={}): - +def post_edge_fit(path, options={}): + #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) required_options = ['print'] default_options = { 'print': False @@ -132,7 +122,7 @@ def post_edge_normalization(path, options={}): df_bkgd_sub,filenames,edge = pre_edge_subtraction(path, options=options) #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge - #Implement widget + #FIXME Use rought edge shift estimate, add X eV as first guess, have an option to adjust this value with widget if edge == 'Mn': edge_stop = 6.565 if edge == 'Ni': @@ -171,7 +161,7 @@ def smoothing(path, options={}): } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - df_bkgd_sub, df_postedge, filenames, edge = post_edge_normalization(path,options=options) + df_bkgd_sub, df_postedge, filenames, edge = post_edge_fit(path,options=options) #================= SMOOTHING df_smooth = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) df_default = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) From d88a302d2a91ae48a4cea57ff7c24a0a9b4ef6d4 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Jun 2022 13:44:42 +0200 Subject: [PATCH 167/355] Add Fe and Co to find_element and refactor --- nafuma/xanes/calib.py | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index f11ff93..c93813d 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -11,14 +11,27 @@ from scipy.signal import savgol_filter ##Better to make a new function that loops through the files, and performing the split_xanes_scan on #Trying to make a function that can decide which edge it is based on the first ZapEnergy-value -def finding_edge(df): - #FIXME add Fe and Co - if 5.9 < df["ZapEnergy"][0] < 6.5: - edge='Mn' - return(edge) - if 8.0 < df["ZapEnergy"][0] < 8.6: - edge='Ni' - return(edge) +def find_element(data: dict) -> str: + ''' Takes the data dictionary and determines based on the start value of the ZapEnergy-column which element the edge is from.''' + + element_energy_intervals = { + 'Mn': [5.9, 6.5], + 'Fe': [7.0, 7.2], + 'Co': [7.6, 7.8], + 'Ni': [8.0, 8.6] + } + + if element_energy_intervals['Mn'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Mn'][1]: + edge = 'Mn' + elif element_energy_intervals['Co'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Fe'][1]: + edge = 'Fe' + elif element_energy_intervals['Co'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Co'][1]: + edge = 'Co' + elif element_energy_intervals['Ni'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Ni'][1]: + edge = 'Ni' + + + return(edge) def pre_edge_subtraction(path, options={}): #FIXME add log-file instead of the troubleshoot-option @@ -31,7 +44,7 @@ def pre_edge_subtraction(path, options={}): filenames = xas.io.get_filenames(path) df= xas.io.put_in_dataframe(path) - edge=finding_edge(df) + edge=find_element(df) #Defining the end of the region used to define the background, thus start of the edge From 909c616c508ce7bdea4c7fbb2e0fc31ef94cfd08 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Jun 2022 14:28:50 +0200 Subject: [PATCH 168/355] Add function to write out log messages --- nafuma/auxillary.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 68785f7..33a62fe 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -52,4 +52,23 @@ def floor(a, roundto=1): a = np.floor(a*fac) / fac - return a \ No newline at end of file + return a + + + +def write_log(message, options={}): + from datetime import datetime + + required_options = ['logfile'] + default_options = { + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}' + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + now = datetime.now().strftime('%Y/%m/%d %H:%M:%S') + message = f'{now} {message} \n' + + + with open(options['logfile'], 'a') as f: + f.write(message) \ No newline at end of file From d17e715d82cdd1243a08c2c7dfaf8655c7168283 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 15 Jun 2022 14:50:32 +0200 Subject: [PATCH 169/355] Separating and refactoring pre_edge_normalisation --- nafuma/xanes/calib.py | 63 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index c93813d..56d944b 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -33,7 +33,68 @@ def find_element(data: dict) -> str: return(edge) -def pre_edge_subtraction(path, options={}): + + +def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: + from datetime import datetime + + # FIXME Add log-file + + required_options = ['edge_start', 'log', 'troubleshoot'] + default_options = { + 'edge_start': None, + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_pre_edge_fit.log', + 'save_fit': False, + 'save_folder': './' + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + if options['log']: + aux.write_log(message='Starting pre edge fit', options=options) + + + + # FIXME Implement with finding accurate edge position + # Find the cutoff point at which the edge starts - everything to the LEFT of this point will be used in the pre edge function fit + if not options['edge_start']: + edge_starts = { + 'Mn': 6.42, + 'Fe': 7.11, + 'Co': 7.705, + 'Ni': 8.3 + } + + edge_start = edge_starts[data['edge']] + + # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) + pre_edge_data = data['xanes_data'].loc[data['xanes_data']["ZapEnergy"] < edge_start] + + # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data + pre_edge_fit_data = pd.DataFrame(data['xanes_data']["ZapEnergy"]) + + for filename in data['path']: + if options['log']: + aux.write_log(message=f'Fitting background on {filename}', options=options) + + #Fitting linear function to the background + params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],1) + fit_function = np.poly1d(params) + + #making a list, y_pre,so the background will be applied to all ZapEnergy-values + background=fit_function(pre_edge_fit_data["ZapEnergy"]) + + #adding a new column in df_background with the y-values of the background + pre_edge_fit_data.insert(1,filename,background) + + if options['log']: + aux.write_log(message=f'Pre edge fitting done.', options=options) + + return pre_edge_fit_data + + +def pre_edge_subtraction(data: dict, options={}): #FIXME add log-file instead of the troubleshoot-option required_options = ['print','troubleshoot'] default_options = { From 7485adef07e9df407647ea061e78711a0f72956f Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 15 Jun 2022 16:00:47 +0200 Subject: [PATCH 170/355] Adding sketch for normalization and flattening --- nafuma/xanes/calib.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index f11ff93..5579574 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -356,4 +356,30 @@ def finding_e0(path, options={}): print("Edge shift estimated by the double differential zero-point is "+str(round(edge_shift_doublediff,5))) if options['print'] == True: ax4.axvline(x=edge_shift_doublediff,color="green") + + return df_smooth, filenames, edge_shift_diff + +def normalization(data,options={}): + required_options = ['print'] + default_options = { + 'print': False, + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #Finding the normalization constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 + normalization_constant=post_edge_fit_function(e0) - pre_edge_fit_function(e0) + #subtracting background (as in pre_edge_subtraction) + + #dividing the background-subtracted data with the normalization constant + + +def flattening(data,options={}): + #only picking out zapenergy-values higher than edge position (edge pos and below remains untouched) + df_e0_and_above=df.loc[df['ZapEnergy'] > edge_shift_diff] + + flattened_data = post_edge_fit_function(df_e0_and_above['ZapEnergy']) - pre_edge_fit_function(df_e0_and_above['ZapEnergy']) + + #make a new dataframe with flattened values + + From e0b71a85b788936e929e2b4a2868ab1606b07498 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 14:58:41 +0200 Subject: [PATCH 171/355] Add save fit function to pre edge fit function --- nafuma/xanes/calib.py | 50 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 56d944b..238aa3e 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -6,6 +6,7 @@ import nafuma.auxillary as aux import nafuma.xanes as xas import nafuma.xanes.io as io from scipy.signal import savgol_filter +from datetime import datetime ##Better to make a new function that loops through the files, and performing the split_xanes_scan on @@ -36,7 +37,7 @@ def find_element(data: dict) -> str: def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: - from datetime import datetime + # FIXME Add log-file @@ -68,15 +69,17 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: edge_start = edge_starts[data['edge']] + # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to + # limit the interval # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) pre_edge_data = data['xanes_data'].loc[data['xanes_data']["ZapEnergy"] < edge_start] # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data pre_edge_fit_data = pd.DataFrame(data['xanes_data']["ZapEnergy"]) - for filename in data['path']: + for i, filename in enumerate(data['path']): if options['log']: - aux.write_log(message=f'Fitting background on {filename}', options=options) + aux.write_log(message=f'Fitting background on {filename} ({i} / {len(data["path"])}', options=options) #Fitting linear function to the background params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],1) @@ -88,13 +91,50 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: #adding a new column in df_background with the y-values of the background pre_edge_fit_data.insert(1,filename,background) - if options['log']: - aux.write_log(message=f'Pre edge fitting done.', options=options) + if options['save_fit']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], filename) + '.png' + + fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) + pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) + ax1.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') + ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) + + data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) + pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) + ax2.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') + ax2.set_xlim([min(pre_edge_data['ZapEnergy']), max(pre_edge_data['ZapEnergy'])]) + ax2.set_ylim([min(pre_edge_data[filename]), max(pre_edge_data[filename])]) + ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) + + + plt.savefig(dst) + plt.close() + + + if options['log']: + aux.write_log(message=f'Pre edge fitting done.', options=options) return pre_edge_fit_data + def pre_edge_subtraction(data: dict, options={}): + + required_options = ['log', 'logfile'] + default_options = { + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_pre_edge_subtraction.log', + } + + + + + +def pre_edge_subtraction_legacy(data: dict, options={}): #FIXME add log-file instead of the troubleshoot-option required_options = ['print','troubleshoot'] default_options = { From bac137042e55c109450146c0d05bd23291214ea7 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 15:42:50 +0200 Subject: [PATCH 172/355] Refactor pre edge subtraction --- nafuma/xanes/calib.py | 120 ++++++++++-------------------------------- 1 file changed, 27 insertions(+), 93 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 238aa3e..833e98e 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -41,12 +41,12 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Add log-file - required_options = ['edge_start', 'log', 'troubleshoot'] + required_options = ['edge_start', 'log', 'logfile', 'save_plots', 'save_folder'] default_options = { 'edge_start': None, 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_pre_edge_fit.log', - 'save_fit': False, + 'save_plots': False, 'save_folder': './' } @@ -72,10 +72,10 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to # limit the interval # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) - pre_edge_data = data['xanes_data'].loc[data['xanes_data']["ZapEnergy"] < edge_start] + pre_edge_data = data['xanes_data_original'].loc[data['xanes_data_original']["ZapEnergy"] < edge_start] # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data - pre_edge_fit_data = pd.DataFrame(data['xanes_data']["ZapEnergy"]) + pre_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) for i, filename in enumerate(data['path']): if options['log']: @@ -91,11 +91,11 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: #adding a new column in df_background with the y-values of the background pre_edge_fit_data.insert(1,filename,background) - if options['save_fit']: + if options['save_plots']: if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) - dst = os.path.join(options['save_folder'], filename) + '.png' + dst = os.path.join(options['save_folder'], filename) + '_pre_edge_fit.png' fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) @@ -124,107 +124,41 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: def pre_edge_subtraction(data: dict, options={}): - required_options = ['log', 'logfile'] + required_options = ['log', 'logfile', 'save_plots', 'save_folder'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_pre_edge_subtraction.log', + 'save_plots': False, + 'save_folder': './' } + if options['log']: + aux.write_log(message='Starting pre edge subtraction', options=options) + xanes_data_bkgd_subtracted = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) + for i, filename in enumerate(data['path']): + if options['log']: + aux.write_log(message=f'Subtracting background on {filename} ({i} / {len(data["path"])}', options=options) + xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data'][filename] - data['pre_edge_fit_data'][filename]) -def pre_edge_subtraction_legacy(data: dict, options={}): - #FIXME add log-file instead of the troubleshoot-option - required_options = ['print','troubleshoot'] - default_options = { - 'print': False, - 'troubleshoot': False - } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - filenames = xas.io.get_filenames(path) - df= xas.io.put_in_dataframe(path) - edge=find_element(df) - - #Defining the end of the region used to define the background, thus start of the edge - - #######================================================================================================================================================ - #FIXME Trying to implement automatical region determination based on an estimate of the edge shift - #print(df) - #estimated_edge_shift, df_diff, df_diff_max = find_pos_maxdiff(df, filenames,options=options) + dst = os.path.join(options['save_folder'], filename) + '_pre_edge_subtraction.png' - #print(estimated_edge_shift) - #estimated_edge_shift - ###========================================================================================================================================================================= - #implement widget - if edge == 'Mn': - edge_start = 6.42 - #edge_start = estimated_edge_shift - if edge == 'Ni': - edge_start = 8.3 + fig, ax = plt.subplots(1,2,figsize=(10,5)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax) + xanes_data_bkgd_subtracted.plot(x='ZapEnergy', y=filename, color='red', ax=ax) + ax.set_title(f'{os.path.basename(filename)} - After subtraction', size=20) - #making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) - df_start=df.loc[df["ZapEnergy"] < edge_start] - - #Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data - df_bkgd = pd.DataFrame(df["ZapEnergy"]) + plt.savefig(dst) + plt.close() - for files in filenames: + return xanes_data_bkgd_subtracted - #Fitting linear function to the background - d = np.polyfit(df_start["ZapEnergy"],df_start[files],1) - function_bkgd = np.poly1d(d) - - #making a list, y_pre,so the background will be applied to all ZapEnergy-values - y_bkgd=function_bkgd(df["ZapEnergy"]) - - #adding a new column in df_background with the y-values of the background - df_bkgd.insert(1,files,y_bkgd) - - - if options['troubleshoot'] == True: - ### FOR FIGURING OUT WHERE IT GOES WRONG/WHICH FILE IS CORRUPT - ax = df.plot(x = "ZapEnergy",y=files) - #Plotting the calculated pre-edge background with the region used for the regression - if options['print'] == True: - #Plotting an example of the edge_start region and the fitted background that will later be subtracted - fig, (ax1,ax2,ax3) = plt.subplots(1,3,figsize=(15,5)) - df.plot(x="ZapEnergy", y=filenames,color="Black",ax=ax1) - df_bkgd.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax1) - plt.axvline(x = max(df_start["ZapEnergy"])) - #fig = plt.figure(figsize=(15,15)) - df_bkgd.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax2) - ax1.set_title('Data and fitted background') - #Zooming into bacground region to confirm fit and limits looks reasonable - df.plot(x = "ZapEnergy",y=filenames,ax=ax2) #defining x and y) - ax2.set_xlim([min(df_start["ZapEnergy"]),max(df_start["ZapEnergy"])+0.01]) - #finding maximum and minimum values in the backgrounds - min_values=[] - max_values=[] - for file in filenames: - min_values.append(min(df_start[file])) - max_values.append(max(df_start[file])) - ax2.set_ylim([min(min_values),max(max_values)]) - plt.axvline(x = max(df_start["ZapEnergy"])) - #ax2.set_xlim([25, 50]) - ###################### Subtracting the pre edge from xmap_roi00 ################ - - #making a new dataframe to insert the background subtracted intensities - df_bkgd_sub = pd.DataFrame(df["ZapEnergy"]) - #inserting the background subtracted original xmap_roi00 data - - for files in filenames: - newintensity_calc=df[files]-df_bkgd[files] - df_bkgd_sub.insert(1,files,newintensity_calc) - - if options['print'] == True: - df.plot(x = "ZapEnergy",y=filenames, color="Black", ax=ax3, legend=False) - #plt.axvline(x = max(df_start["ZapEnergy"])) - df_bkgd_sub.plot(x="ZapEnergy", y=filenames,color="Red",ax=ax3, legend=False) - ax3.set_title('Data and background-subtracted data') - - return df_bkgd_sub,filenames,edge def post_edge_fit(path, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) From 2baa765806d5035b9c4c4d9bab7d5a18c7feb92a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 15:55:21 +0200 Subject: [PATCH 173/355] Quasi-fixed linting issue causing automatic test to fail --- nafuma/xanes/io.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 527f300..f8a3e78 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -8,6 +8,9 @@ def split_xanes_scan(root, destination=None, replace=False): #destination should be the path to the processed data #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw + + # FIXME Only adding this variable to pass the Linting-tests - will refactor this later + filename = 'dummy' with open(filename, 'r') as f: lines = f.readlines() From a49fc8b0d23ec7b26f02fd1d60afd901205a400c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 16:18:22 +0200 Subject: [PATCH 174/355] Refactor read_data --- nafuma/xanes/io.py | 49 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index f8a3e78..b5faf2c 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -2,6 +2,8 @@ import pandas as pd import matplotlib.pyplot as plt import os import numpy as np +import nafuma.auxillary as aux + def split_xanes_scan(root, destination=None, replace=False): #root is the path to the beamtime-folder @@ -105,8 +107,51 @@ def get_filenames(path): return filenames -def put_in_dataframe(path): - filenames = get_filenames(path) + + +def read_data(data: dict, options={}) -> pd.DataFrame: + + required_options = [] + default_options = { + + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + columns = ['ZapEnergy'] + + # Initialise DataFrame with only ZapEnergy-column + xanes_data = pd.read_csv(data['path'][0])[['ZapEnergy']] + + for filename in data['path']: + columns.append(filename) + + scan_data = pd.read_csv(filename) + scan_data = scan_data[[determine_active_roi(scan_data)]] + xanes_data.insert(1, filename, scan_data) + + + return xanes_data + + + + + +def determine_active_roi(scan_data): + + #Trying to pick the roi with the highest difference between maximum and minimum intensity --> biggest edge shift + if max(scan_data["xmap_roi00"])-min(scan_data["xmap_roi00"])>max(scan_data["xmap_roi01"])-min(scan_data["xmap_roi01"]): + active_roi = 'xmap_roi00' + else: + active_roi = 'xmap_roi01' + + return active_roi + + + + +def put_into_dataframe(data: dict, options={}) -> pd.DataFrame: + filenames = get_filenames(data) #making the column names to be used in the dataframe, making sure the first column is the ZapEnergy column_names = ["ZapEnergy"] From 0d757ce36501f1aa1a7add8abb470b4055cf070b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 16:26:41 +0200 Subject: [PATCH 175/355] Move get_filenames to auxillary and generalise --- nafuma/auxillary.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 33a62fe..072ee83 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -1,5 +1,6 @@ import json import numpy as np +import os def update_options(options, required_options, default_options): ''' Takes a dictionary of options along with a list of required options and dictionary of default options, and sets all keyval-pairs of options that is not already defined to the default values''' @@ -71,4 +72,17 @@ def write_log(message, options={}): with open(options['logfile'], 'a') as f: - f.write(message) \ No newline at end of file + f.write(message) + + +#Function that "collects" all the files in a folder, only accepting .dat-files from xanes-measurements +def get_filenames(path, ext): + ''' Collects all filenames from specified path with a specificed extension + + Input: + path: path to find all filenames (relative or absolute) + ext: extension (including ".")''' + + filenames = [os.path.join(path, filename) for filename in os.listdir(path) if os.path.isfile(os.path.join(path, filename)) and filename.endswith(ext)] + + return filenames \ No newline at end of file From 303704c3577a2762041b1d4b647c8bea016536d8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 17:54:51 +0200 Subject: [PATCH 176/355] Add filter --- nafuma/auxillary.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 072ee83..2b87479 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -68,7 +68,7 @@ def write_log(message, options={}): options = update_options(options=options, required_options=required_options, default_options=default_options) now = datetime.now().strftime('%Y/%m/%d %H:%M:%S') - message = f'{now} {message} \n' + message = f'[{now}] {message} \n' with open(options['logfile'], 'a') as f: @@ -76,13 +76,13 @@ def write_log(message, options={}): #Function that "collects" all the files in a folder, only accepting .dat-files from xanes-measurements -def get_filenames(path, ext): +def get_filenames(path, ext, filter=''): ''' Collects all filenames from specified path with a specificed extension Input: path: path to find all filenames (relative or absolute) ext: extension (including ".")''' - filenames = [os.path.join(path, filename) for filename in os.listdir(path) if os.path.isfile(os.path.join(path, filename)) and filename.endswith(ext)] + filenames = [os.path.join(path, filename) for filename in os.listdir(path) if os.path.isfile(os.path.join(path, filename)) and filename.endswith(ext) and filter in filename] return filenames \ No newline at end of file From 0b89524ef19b8a2c6f8dd76302f4c6a51c3c7258 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 17:55:42 +0200 Subject: [PATCH 177/355] Clear up small bugs encountered during testing --- nafuma/xanes/calib.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 833e98e..a1211e2 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -22,13 +22,13 @@ def find_element(data: dict) -> str: 'Ni': [8.0, 8.6] } - if element_energy_intervals['Mn'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Mn'][1]: + if (element_energy_intervals['Mn'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Mn'][1]): edge = 'Mn' - elif element_energy_intervals['Co'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Fe'][1]: + elif (element_energy_intervals['Fe'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Fe'][1]): edge = 'Fe' - elif element_energy_intervals['Co'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Co'][1]: + elif (element_energy_intervals['Co'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Co'][1]): edge = 'Co' - elif element_energy_intervals['Ni'][0] < data['xanes_data']["ZapEnergy"][0] < element_energy_intervals['Ni'][1]: + elif (element_energy_intervals['Ni'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Ni'][1]): edge = 'Ni' @@ -45,7 +45,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: default_options = { 'edge_start': None, 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_pre_edge_fit.log', + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', 'save_plots': False, 'save_folder': './' } @@ -62,11 +62,12 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: if not options['edge_start']: edge_starts = { 'Mn': 6.42, - 'Fe': 7.11, + 'Fe': 7.09, 'Co': 7.705, 'Ni': 8.3 } + data['edge'] = find_element(data) edge_start = edge_starts[data['edge']] # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to @@ -79,7 +80,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: for i, filename in enumerate(data['path']): if options['log']: - aux.write_log(message=f'Fitting background on {filename} ({i} / {len(data["path"])}', options=options) + aux.write_log(message=f'Fitting background on {os.path.basename(filename)} ({i+1} / {len(data["path"])})', options=options) #Fitting linear function to the background params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],1) @@ -95,15 +96,15 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) - dst = os.path.join(options['save_folder'], filename) + '_pre_edge_fit.png' + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_fit.png' fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) - data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) ax1.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) - data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) ax2.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') ax2.set_xlim([min(pre_edge_data['ZapEnergy']), max(pre_edge_data['ZapEnergy'])]) @@ -111,7 +112,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) - plt.savefig(dst) + plt.savefig(dst, transparent=False) plt.close() From e7a95d65edd7a04d8dd38cce1e537e4008af30fd Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 16 Jun 2022 17:56:08 +0200 Subject: [PATCH 178/355] Refactor read_data and move get_filenames --- nafuma/xanes/io.py | 71 ++++------------------------------------------ 1 file changed, 6 insertions(+), 65 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index b5faf2c..458b38f 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -89,28 +89,15 @@ def split_xanes_scan(root, destination=None, replace=False): df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) -#Function that "collects" all the files in a folder, only accepting .dat-files from xanes-measurements -def get_filenames(path): - - - cwd = os.getcwd() - - # Change into path provided - os.chdir(path) - - filenames = [os.path.join(path, filename) for filename in os.listdir() if os.path.isfile(filename) and filename[-4:] == '.dat'] #changed - - - - # Change directory back to where you ran the script from - os.chdir(cwd) - - return filenames + def read_data(data: dict, options={}) -> pd.DataFrame: + + # FIXME Handle the case when dataseries are not the same size + required_options = [] default_options = { @@ -138,6 +125,8 @@ def read_data(data: dict, options={}) -> pd.DataFrame: def determine_active_roi(scan_data): + + # FIXME For Co-edge, this gave a wrong scan #Trying to pick the roi with the highest difference between maximum and minimum intensity --> biggest edge shift if max(scan_data["xmap_roi00"])-min(scan_data["xmap_roi00"])>max(scan_data["xmap_roi01"])-min(scan_data["xmap_roi01"]): @@ -146,51 +135,3 @@ def determine_active_roi(scan_data): active_roi = 'xmap_roi01' return active_roi - - - - -def put_into_dataframe(data: dict, options={}) -> pd.DataFrame: - filenames = get_filenames(data) - - #making the column names to be used in the dataframe, making sure the first column is the ZapEnergy - column_names = ["ZapEnergy"] - - for i in range(len(filenames)): - column_names.append(filenames[i]) - - #Taking the first file in the folder and extracting ZapEnergies and intensity from that (only need the intensity from the rest) - first = pd.read_csv(filenames[0], skiprows=0) - - #Making a data frame with the correct columns, and will fill inn data afterwards - df = pd.DataFrame(columns = column_names) - #First putting in the 2theta-values - df["ZapEnergy"]=first["ZapEnergy"] - - #filling in the intensities from all files into the corresponding column in the dataframe - for i in range(len(filenames)): - df2 = pd.read_csv(filenames[i]) - df2 = df2.drop(['Mon','Det1','Det2','Det3','Det4','Det5', 'Det6','Ion1'], axis=1) #, axis=1) - df2 = df2.drop(['MonEx','Ion2','Htime','MusstEnc1','MusstEnc3','MusstEnc4', 'TwoTheta', 'ZCryo'], axis=1) - df2 = df2.drop(['ZBlower1', 'ZBlower2', 'ZSrcur'], axis=1)#, axis=19) #removing the sigma at this point - - ############## THIS PART PICKS OUT WHICH ROI IS OF INTEREST, BUT MUST BE FIXED IF LOOKING AT THREE EDGES (roi00,roi01,roi02) ##################### - if 'xmap_roi01' in df2.columns: - #Trying to pick the roi with the highest difference between maximum and minimum intensity --> biggest edge shift - if max(df2["xmap_roi00"])-min(df2["xmap_roi00"])>max(df2["xmap_roi01"])-min(df2["xmap_roi01"]): - df[filenames[i]]=df2["xmap_roi00"] #forMn - else: - df[filenames[i]]=df2["xmap_roi01"] #forNi - else: - df[filenames[i]]=df2["xmap_roi00"] - ############################################################################################### - - i=i+1 - - - #print(df) - #If I want to make a csv-file of the raw data. Decided that was not necessary: - #df.to_csv('static-Mn-edge.csv') #writing it to a csv, first row is datapoint (index), second column is 2theta, and from there the scans starts - - - return df \ No newline at end of file From 880722d7784d7cc6c3a44ceb98e87fb3aa7bd1b8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 17 Jun 2022 15:35:24 +0200 Subject: [PATCH 179/355] Load correct xmap_roi for more cases --- nafuma/xanes/io.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 458b38f..20aa717 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -115,7 +115,10 @@ def read_data(data: dict, options={}) -> pd.DataFrame: scan_data = pd.read_csv(filename) scan_data = scan_data[[determine_active_roi(scan_data)]] - xanes_data.insert(1, filename, scan_data) + xanes_data = pd.concat([xanes_data, scan_data], axis=1) + + + xanes_data.columns = columns return xanes_data @@ -129,9 +132,24 @@ def determine_active_roi(scan_data): # FIXME For Co-edge, this gave a wrong scan #Trying to pick the roi with the highest difference between maximum and minimum intensity --> biggest edge shift - if max(scan_data["xmap_roi00"])-min(scan_data["xmap_roi00"])>max(scan_data["xmap_roi01"])-min(scan_data["xmap_roi01"]): - active_roi = 'xmap_roi00' - else: - active_roi = 'xmap_roi01' + # if max(scan_data["xmap_roi00"])-min(scan_data["xmap_roi00"])>max(scan_data["xmap_roi01"])-min(scan_data["xmap_roi01"]): + # active_roi = 'xmap_roi00' + # else: + # active_roi = 'xmap_roi01' + if (scan_data['xmap_roi00'].iloc[0:100].mean() < scan_data['xmap_roi00'].iloc[-100:].mean()) and (scan_data['xmap_roi01'].iloc[0:100].mean() < scan_data['xmap_roi01'].iloc[-100:].mean()): + if (scan_data['xmap_roi00'].max()-scan_data['xmap_roi00'].min()) > (scan_data['xmap_roi01'].max() - scan_data['xmap_roi01'].min()): + active_roi = 'xmap_roi00' + else: + active_roi = 'xmap_roi01' + + elif scan_data['xmap_roi00'].iloc[0:100].mean() < scan_data['xmap_roi00'].iloc[-100:].mean(): + active_roi = 'xmap_roi00' + + elif scan_data['xmap_roi01'].iloc[0:100].mean() < scan_data['xmap_roi01'].iloc[-100:].mean(): + active_roi = 'xmap_roi01' + + else: + active_roi = None + return active_roi From cb2c7532e6ac48073a3cbbccb809ae7631019b2b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 17 Jun 2022 15:46:33 +0200 Subject: [PATCH 180/355] Make sure data['path'] is in a list --- nafuma/xanes/io.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 20aa717..816b8f5 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -110,6 +110,9 @@ def read_data(data: dict, options={}) -> pd.DataFrame: # Initialise DataFrame with only ZapEnergy-column xanes_data = pd.read_csv(data['path'][0])[['ZapEnergy']] + if not isinstance(data['path'], list): + data['path'] = [data['path']] + for filename in data['path']: columns.append(filename) From ba349a5892f480b398444e095f80779e5d7111c3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 17 Jun 2022 15:58:39 +0200 Subject: [PATCH 181/355] Refactor estimation of edge position and automatise pre edge limit setting --- nafuma/xanes/calib.py | 70 ++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index a1211e2..e540e9b 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -12,7 +12,7 @@ from datetime import datetime ##Better to make a new function that loops through the files, and performing the split_xanes_scan on #Trying to make a function that can decide which edge it is based on the first ZapEnergy-value -def find_element(data: dict) -> str: +def find_element(data: dict, index=0) -> str: ''' Takes the data dictionary and determines based on the start value of the ZapEnergy-column which element the edge is from.''' element_energy_intervals = { @@ -22,13 +22,13 @@ def find_element(data: dict) -> str: 'Ni': [8.0, 8.6] } - if (element_energy_intervals['Mn'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Mn'][1]): + if (element_energy_intervals['Mn'][0] < data['xanes_data_original']["ZapEnergy"].iloc[index]) & (data['xanes_data_original']["ZapEnergy"].iloc[index] < element_energy_intervals['Mn'][1]): edge = 'Mn' - elif (element_energy_intervals['Fe'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Fe'][1]): + elif (element_energy_intervals['Fe'][0] < data['xanes_data_original']["ZapEnergy"].iloc[index]) & (data['xanes_data_original']["ZapEnergy"].iloc[index] < element_energy_intervals['Fe'][1]): edge = 'Fe' - elif (element_energy_intervals['Co'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Co'][1]): + elif (element_energy_intervals['Co'][0] < data['xanes_data_original']["ZapEnergy"].iloc[index]) & (data['xanes_data_original']["ZapEnergy"].iloc[index] < element_energy_intervals['Co'][1]): edge = 'Co' - elif (element_energy_intervals['Ni'][0] < data['xanes_data_original']["ZapEnergy"].iloc[0]) & (data['xanes_data_original']["ZapEnergy"].iloc[0] < element_energy_intervals['Ni'][1]): + elif (element_energy_intervals['Ni'][0] < data['xanes_data_original']["ZapEnergy"].iloc[index]) & (data['xanes_data_original']["ZapEnergy"].iloc[index] < element_energy_intervals['Ni'][1]): edge = 'Ni' @@ -58,22 +58,25 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Implement with finding accurate edge position + # FIXME Allow specification of start of pre-edge area # Find the cutoff point at which the edge starts - everything to the LEFT of this point will be used in the pre edge function fit if not options['edge_start']: - edge_starts = { - 'Mn': 6.42, - 'Fe': 7.09, - 'Co': 7.705, - 'Ni': 8.3 + pre_edge_limit_offsets = { + 'Mn': 0.03, + 'Fe': 0.03, + 'Co': 0.03, + 'Ni': 0.03 } data['edge'] = find_element(data) - edge_start = edge_starts[data['edge']] + + edge_position = estimate_edge_position(data, options, index=0) + pre_edge_limit = edge_position - pre_edge_limit_offsets[data['edge']] # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to # limit the interval # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) - pre_edge_data = data['xanes_data_original'].loc[data['xanes_data_original']["ZapEnergy"] < edge_start] + pre_edge_data = data['xanes_data_original'].loc[data['xanes_data_original']["ZapEnergy"] < pre_edge_limit] # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data pre_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) @@ -161,6 +164,31 @@ def pre_edge_subtraction(data: dict, options={}): return xanes_data_bkgd_subtracted +def estimate_edge_position(data: dict, options={}, index=0): + #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. + required_options = ['print','periods'] + default_options = { + 'print': False, + 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #making new dataframe to keep the differentiated data + df_diff = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) + df_diff[data['path'][index]]=data['xanes_data_original'][data['path'][index]].diff(periods=options['periods']) + + #shifting column values up so that average differential fits right between the points used in the calculation + df_diff[data['path'][index]]=df_diff[data['path'][index]].shift(-int(options['periods']/2)) + df_diff_max = df_diff[data['path'][index]].dropna().max() + estimated_edge_shift =df_diff.loc[df_diff[data['path'][index]] == df_diff_max,'ZapEnergy'].values[0] + + # FIXME Add logging option to see the result + + print(estimated_edge_shift) + + return estimated_edge_shift + + def post_edge_fit(path, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) required_options = ['print'] @@ -254,25 +282,7 @@ def smoothing(path, options={}): return df_smooth, filenames -def find_pos_maxdiff(df, filenames,options={}): - #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. - required_options = ['print','periods'] - default_options = { - 'print': False, - 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly - } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #making new dataframe to keep the differentiated data - df_diff = pd.DataFrame(df["ZapEnergy"]) - df_diff[filenames]=df[filenames].diff(periods=options['periods']) - - #shifting column values up so that average differential fits right between the points used in the calculation - df_diff[filenames]=df_diff[filenames].shift(-int(options['periods']/2)) - df_diff_max = df_diff[filenames].dropna().max() - estimated_edge_shift =df_diff.loc[df_diff[filenames] == df_diff_max,'ZapEnergy'].values[0] - - return estimated_edge_shift, df_diff, df_diff_max def find_nearest(array, value): #function to find the value closes to "value" in an "array" From 672d5549fe9143bb724c65b195cc323f049448f5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 17 Jun 2022 16:14:21 +0200 Subject: [PATCH 182/355] Fix lint issue --- nafuma/xanes/calib.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index e540e9b..3a37eab 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -136,6 +136,7 @@ def pre_edge_subtraction(data: dict, options={}): 'save_folder': './' } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) if options['log']: aux.write_log(message='Starting pre edge subtraction', options=options) @@ -145,16 +146,16 @@ def pre_edge_subtraction(data: dict, options={}): if options['log']: aux.write_log(message=f'Subtracting background on {filename} ({i} / {len(data["path"])}', options=options) - xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data'][filename] - data['pre_edge_fit_data'][filename]) + xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data_original'][filename] - data['pre_edge_fit_data'][filename]) if options['save_plots']: if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) - dst = os.path.join(options['save_folder'], filename) + '_pre_edge_subtraction.png' + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_subtraction.png' - fig, ax = plt.subplots(1,2,figsize=(10,5)) - data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax) + fig, ax = plt.subplots(figsize=(10,5)) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax) xanes_data_bkgd_subtracted.plot(x='ZapEnergy', y=filename, color='red', ax=ax) ax.set_title(f'{os.path.basename(filename)} - After subtraction', size=20) @@ -184,7 +185,8 @@ def estimate_edge_position(data: dict, options={}, index=0): # FIXME Add logging option to see the result - print(estimated_edge_shift) + if options['log']: + aux.write_log(message=f'Estimated edge shift for determination of pre-edge area is: {estimated_edge_shift} keV', options=options) return estimated_edge_shift @@ -309,7 +311,7 @@ def finding_e0(path, options={}): print("MORE THAN ONE FILE --> generalize") ##### - estimated_edge_shift, df_diff, df_diff_max = find_pos_maxdiff(df_smooth, filenames,options=options) + estimated_edge_shift, df_diff, df_diff_max = estimate_edge_position(df_smooth, filenames,options=options) print(estimated_edge_shift) #### ###df_diff[filenames]=df_smooth[filenames].diff(periods=options['periods']) # From 9c6a7d5991af452759fdf2f11afb76623e7d6aae Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 17 Jun 2022 16:59:37 +0200 Subject: [PATCH 183/355] Refactor post_edge_fit --- nafuma/xanes/calib.py | 100 ++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index a45457b..10afda9 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -41,9 +41,9 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Add log-file - required_options = ['edge_start', 'log', 'logfile', 'save_plots', 'save_folder'] + required_options = ['pre_edge_start', 'log', 'logfile', 'save_plots', 'save_folder'] default_options = { - 'edge_start': None, + 'pre_edge_start': None, 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', 'save_plots': False, @@ -60,18 +60,13 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Implement with finding accurate edge position # FIXME Allow specification of start of pre-edge area # Find the cutoff point at which the edge starts - everything to the LEFT of this point will be used in the pre edge function fit - if not options['edge_start']: - pre_edge_limit_offsets = { - 'Mn': 0.03, - 'Fe': 0.03, - 'Co': 0.03, - 'Ni': 0.03 - } + if not options['pre_edge_start']: + pre_edge_limit_offset = 0.03 data['edge'] = find_element(data) edge_position = estimate_edge_position(data, options, index=0) - pre_edge_limit = edge_position - pre_edge_limit_offsets[data['edge']] + pre_edge_limit = edge_position - pre_edge_limit_offset # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to # limit the interval @@ -169,6 +164,7 @@ def estimate_edge_position(data: dict, options={}, index=0): #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. required_options = ['print','periods'] default_options = { + 'print': False, 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly } @@ -191,45 +187,71 @@ def estimate_edge_position(data: dict, options={}, index=0): return estimated_edge_shift -def post_edge_fit(path, options={}): +def post_edge_fit(data: dict, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) - required_options = ['print'] + required_options = ['post_edge_start', 'print'] default_options = { + 'post_edge_start': None, 'print': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + #FIXME Allow min and max limits + + if not options['post_edge_start']: + post_edge_limit_offset = 0.03 + + data['edge'] = find_element(data) + + edge_position = estimate_edge_position(data, options, index=0) + post_edge_limit = edge_position + post_edge_limit_offset + + + post_edge_data = data['xanes_data_original'].loc[data['xanes_data_original']["ZapEnergy"] > post_edge_limit] + post_edge_data.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + + # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data + post_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) - df_bkgd_sub,filenames,edge = pre_edge_subtraction(path, options=options) - #Defining the end of the pre-edge-region for Mn/Ni, thus start of the edge - #FIXME Use rought edge shift estimate, add X eV as first guess, have an option to adjust this value with widget - if edge == 'Mn': - edge_stop = 6.565 - if edge == 'Ni': - edge_stop = 8.361 + for i, filename in enumerate(data['path']): + if options['log']: + aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])})', options=options) - df_end= df_bkgd_sub.loc[df_bkgd_sub["ZapEnergy"] > edge_stop] # new dataframe only containing the post edge, where a regression line will be calculated in the for-loop below - df_end.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit - df_postedge = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) #making a new dataframe + #Fitting linear function to the background + params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], 2) + fit_function = np.poly1d(params) + + #making a list, y_pre,so the background will be applied to all ZapEnergy-values + background=fit_function(post_edge_fit_data["ZapEnergy"]) + + #adding a new column in df_background with the y-values of the background + post_edge_fit_data.insert(1,filename,background) + + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - function_post_list=[] - for files in filenames: - d = np.polyfit(df_end["ZapEnergy"],df_end[files],1) - function_post = np.poly1d(d) - y_post=function_post(df_bkgd_sub["ZapEnergy"]) - function_post_list.append(function_post) - df_postedge.insert(1,files,y_post) #adding a new column with the y-values of the fitted post edge + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_post_edge_fit.png' - #Plotting the background subtracted signal with the post-edge regression line and the start point for the linear regression line - if options['print'] == True: - ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames) #defining x and y - plt.axvline(x = min(df_end["ZapEnergy"])) - fig = plt.figure(figsize=(15,15)) - df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) - ax = df_bkgd_sub.plot(x = "ZapEnergy",y=filenames, legend=False) #defining x and y - df_postedge.plot(x="ZapEnergy", y=filenames,color="Green",ax=ax, legend=False) - plt.axvline(x = min(df_end["ZapEnergy"])) + fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) + post_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) + ax1.axvline(x = max(post_edge_data['ZapEnergy']), ls='--') + ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) - return df_bkgd_sub, df_postedge, filenames, edge + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) + post_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) + ax2.axvline(x = max(post_edge_data['ZapEnergy']), ls='--') + ax2.set_xlim([min(post_edge_data['ZapEnergy']), max(post_edge_data['ZapEnergy'])]) + ax2.set_ylim([min(post_edge_data[filename]), max(post_edge_data[filename])]) + ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) + + + plt.savefig(dst, transparent=False) + plt.close() + + + return post_edge_fit_data def smoothing(path, options={}): required_options = ['print','window_length','polyorder'] From 7214746af18475257e041e54483f456424aca8dc Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 20 Jun 2022 16:08:36 +0200 Subject: [PATCH 184/355] Refactor split_scans --- nafuma/xanes/io.py | 159 ++++++++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 66 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 816b8f5..53ad1ca 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -2,94 +2,120 @@ import pandas as pd import matplotlib.pyplot as plt import os import numpy as np -import nafuma.auxillary as aux +import nafuma.auxillary as aux +from nafuma.xanes.calib import find_element -def split_xanes_scan(root, destination=None, replace=False): +def split_scan_data(data: dict, options={}): + + + required_options = ['save', 'save_folder', 'replace', 'add_rois'] + + default_options = { + 'save': False, + 'save_folder': '.', + 'replace': False, + 'add_rois': False + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) #root is the path to the beamtime-folder #destination should be the path to the processed data #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw # FIXME Only adding this variable to pass the Linting-tests - will refactor this later - filename = 'dummy' + + if not isinstance(data['path'], list): + data['path'] = [data['path']] + + all_scans = [] - with open(filename, 'r') as f: - lines = f.readlines() + for filename in data['path']: + + with open(filename, 'r') as f: + lines = f.readlines() + + scan_datas, scan_data = [], [] + headers, header = [], '' + read_data = False - datas = [] - data = [] - headers = [] - header = '' - start = False - - for line in lines: - if line[0:2] == "#L": - start = True - header = line[2:].split() - continue - - elif line[0:2] == "#C": - start = False - - if data: - datas.append(data) - data = [] + for line in lines: + # Header line starts with #L - reads headers, and toggles data read-in on + if line[0:2] == "#L": + header, read_data = line[2:].split(), True + continue + + # First line after data started with #C - stops data read-in + elif line[0:2] == "#C": + read_data = False - if header: - headers.append(header) - header = '' + if scan_data: + scan_datas.append(scan_data); scan_data = [] + + if header: + headers.append(header); header = '' + + # Ignore line if read-in not toggled + if read_data == False: + continue + + # Read in data if it is + else: + scan_data.append(line.split()) - - if start == False: - continue - - else: - data.append(line.split()) - - - - - edges = {'Mn': [6.0, 6.1, 6.2, 6.3, 6.4, 6.5], 'Fe': [6.8, 6.9, 7.0, 7.1, 7.2], 'Co': [7.6, 7.7, 7.8, 7.9], 'Ni': [8.1, 8.2, 8.3, 8.4, 8.5]} - edge_count = {'Mn': 0, 'Fe': 0, 'Co': 0, 'Ni': 0} - - - for ind, data in enumerate(datas): - df = pd.DataFrame(data) - df.columns = headers[ind] - - edge_start = np.round((float(df["ZapEnergy"].min())), 1) - - for edge, energies in edges.items(): - if edge_start in energies: - edge_actual = edge - edge_count[edge] += 1 - + edges = {'Mn': [], 'Fe': [], 'Co': [], 'Ni': []} + + for i, scan_data in enumerate(scan_datas): + xanes_df = pd.DataFrame(scan_data).apply(pd.to_numeric) + xanes_df.columns = headers[i] + + if not ('xmap_roi00' in headers[i]) and (not 'xmap_roi01' in headers[i]): + continue + + + edge = find_element({'xanes_data_original': xanes_df}) + edges[edge].append(xanes_df) + - filename = filename.split('/')[-1] - count = str(edge_count[edge_actual]).zfill(4) + if options['add']: + + added_edges = {'Mn': [], 'Fe': [], 'Co': [], 'Ni': []} + for edge, scans in edges.items(): + if scans: + xanes_df = scans[0] - - # Save - if destination: - cwd = os.getcwd() + for i, scan in enumerate(scans): + if i > 0: - if not os.path.isdir(destination): - os.mkdir(destination) - - os.chdir(destination) + if 'xmap_roi00' in xanes_df.columns: + xanes_df['xmap_roi00'] += scan['xmap_roi00'] + if 'xmap_roi01' in xanes_df.columns: + xanes_df['xmap_roi01'] += scan['xmap_roi01'] - df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + added_edges[edge].append(xanes_df) - os.chdir(cwd) - - else: - df.to_csv('{}_{}_{}.dat'.format(filename.split('.')[0], edge_actual, count)) + edges = added_edges + + if options['save']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + filename = os.path.basename(filename).split('.')[0] + for edge, scans in edges.items(): + for i, scan in enumerate(scans): + count = '' if options['add'] else '_'+str(i).zfill(4) + path = os.path.join(options['save_folder'], f'{filename}_{edge}{count}.dat') + scan.to_csv(path) + + all_scans.append(edges) + + + return all_scans @@ -117,6 +143,7 @@ def read_data(data: dict, options={}) -> pd.DataFrame: columns.append(filename) scan_data = pd.read_csv(filename) + scan_data = scan_data[[determine_active_roi(scan_data)]] xanes_data = pd.concat([xanes_data, scan_data], axis=1) From cc80a48259dfae683a46752e76001488d035d605 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 20 Jun 2022 19:16:20 +0200 Subject: [PATCH 185/355] Add logging --- nafuma/xanes/io.py | 78 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 10 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 53ad1ca..40ea0c2 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -4,18 +4,23 @@ import os import numpy as np import nafuma.auxillary as aux from nafuma.xanes.calib import find_element +from datetime import datetime -def split_scan_data(data: dict, options={}): +def split_scan_data(data: dict, options={}) -> list: + ''' Splits a XANES-file from BM31 into different files depending on the edge. Has the option to add intensities of all scans of same edge into the same file. + As of now only picks out xmap_rois (fluoresence mode) and for Mn, Fe, Co and Ni K-edges.''' - required_options = ['save', 'save_folder', 'replace', 'add_rois'] + required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'add_rois'] default_options = { - 'save': False, - 'save_folder': '.', - 'replace': False, - 'add_rois': False + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_split_edges.log', + 'save': False, # whether to save the files or not + 'save_folder': '.', # root folder of where to save the files + 'replace': False, # whether to replace the files if they already exist + 'add_rois': False # Whether to add the rois of individual scans of the same edge together } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -30,9 +35,15 @@ def split_scan_data(data: dict, options={}): data['path'] = [data['path']] all_scans = [] + + if options['log']: + aux.write_log(message='Starting file splitting...', options=options) for filename in data['path']: + if options['log']: + aux.write_log(message=f'Reading {filename}...', options=options) + with open(filename, 'r') as f: lines = f.readlines() @@ -44,6 +55,9 @@ def split_scan_data(data: dict, options={}): # Header line starts with #L - reads headers, and toggles data read-in on if line[0:2] == "#L": header, read_data = line[2:].split(), True + + if options['log']: + aux.write_log(message='... Found scan data. Starting read-in...', options=options) continue # First line after data started with #C - stops data read-in @@ -69,27 +83,46 @@ def split_scan_data(data: dict, options={}): for i, scan_data in enumerate(scan_datas): + xanes_df = pd.DataFrame(scan_data).apply(pd.to_numeric) xanes_df.columns = headers[i] + edge = find_element({'xanes_data_original': xanes_df}) + + if options['log']: + aux.write_log(message=f'Starting data clean-up ({edge}-edge)... ({i+1}/{len(scan_datas)})', options=options) + if not ('xmap_roi00' in headers[i]) and (not 'xmap_roi01' in headers[i]): + if options['log']: + aux.write_log(message='... Did not find fluoresence data. Skipping...', options=options) + continue - edge = find_element({'xanes_data_original': xanes_df}) + edges[edge].append(xanes_df) if options['add']: + + if options['log']: + aux.write_log(message=f'Addition of rois enabled. Starting addition...', options=options) added_edges = {'Mn': [], 'Fe': [], 'Co': [], 'Ni': []} for edge, scans in edges.items(): + + if options['log']: + aux.write_log(message=f'... Adding rois of the {edge}-edge...', options=options) + if scans: xanes_df = scans[0] for i, scan in enumerate(scans): if i > 0: + if options['log']: + aux.write_log(message=f'... ... Adding {i}/{len(scans)}', options=options) + if 'xmap_roi00' in xanes_df.columns: xanes_df['xmap_roi00'] += scan['xmap_roi00'] if 'xmap_roi01' in xanes_df.columns: @@ -100,7 +133,14 @@ def split_scan_data(data: dict, options={}): edges = added_edges if options['save']: + + if options['log']: + aux.write_log(message=f'Saving data to {options["save_folder"]}', options=options) + if not os.path.isdir(options['save_folder']): + if options['log']: + aux.write_log(message=f'... {options["save_folder"]} does not exist. Creating folder.', options=options) + os.makedirs(options['save_folder']) @@ -110,10 +150,26 @@ def split_scan_data(data: dict, options={}): for i, scan in enumerate(scans): count = '' if options['add'] else '_'+str(i).zfill(4) path = os.path.join(options['save_folder'], f'{filename}_{edge}{count}.dat') - scan.to_csv(path) + + if not os.path.isfile(path): + scan.to_csv(path) + if options['log']: + aux.write_log(message=f'... Scan saved to {path}', options=options) + + elif options['replace'] and os.path.isfile(path): + scan.to_csv(path) + if options['log']: + aux.write_log(message=f'... File already exists. Overwriting to {path}', options=options) + + elif not options['replace'] and os.path.isfile(path): + if options['log']: + aux.write_log(message=f'... File already exists. Skipping...', options=options) all_scans.append(edges) + if options['log']: + aux.write_log(message=f'All done!', options=options) + return all_scans @@ -124,9 +180,9 @@ def read_data(data: dict, options={}) -> pd.DataFrame: # FIXME Handle the case when dataseries are not the same size - required_options = [] + required_options = ['adjust'] default_options = { - + 'adjust': 0 } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -135,6 +191,7 @@ def read_data(data: dict, options={}) -> pd.DataFrame: # Initialise DataFrame with only ZapEnergy-column xanes_data = pd.read_csv(data['path'][0])[['ZapEnergy']] + xanes_data['ZapEnergy'] += options['adjust'] if not isinstance(data['path'], list): data['path'] = [data['path']] @@ -157,6 +214,7 @@ def read_data(data: dict, options={}) -> pd.DataFrame: + def determine_active_roi(scan_data): # FIXME For Co-edge, this gave a wrong scan From 054311ca102893b2aca555a9360e13f847dd01f2 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 21 Jun 2022 18:01:53 +0200 Subject: [PATCH 186/355] Small adjustments to logging --- nafuma/xanes/io.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 40ea0c2..f623a38 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -16,7 +16,7 @@ def split_scan_data(data: dict, options={}) -> list: default_options = { 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_split_edges.log', + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_split_edges.log', 'save': False, # whether to save the files or not 'save_folder': '.', # root folder of where to save the files 'replace': False, # whether to replace the files if they already exist @@ -24,12 +24,6 @@ def split_scan_data(data: dict, options={}) -> list: } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #root is the path to the beamtime-folder - #destination should be the path to the processed data - - #insert a for-loop to go through all the folders.dat-files in the folder root\xanes\raw - - # FIXME Only adding this variable to pass the Linting-tests - will refactor this later if not isinstance(data['path'], list): data['path'] = [data['path']] @@ -89,12 +83,12 @@ def split_scan_data(data: dict, options={}) -> list: edge = find_element({'xanes_data_original': xanes_df}) if options['log']: - aux.write_log(message=f'Starting data clean-up ({edge}-edge)... ({i+1}/{len(scan_datas)})', options=options) + aux.write_log(message=f'... Starting data clean-up ({edge}-edge)... ({i+1}/{len(scan_datas)})', options=options) if not ('xmap_roi00' in headers[i]) and (not 'xmap_roi01' in headers[i]): if options['log']: - aux.write_log(message='... Did not find fluoresence data. Skipping...', options=options) + aux.write_log(message='... ... Did not find fluoresence data. Skipping...', options=options) continue @@ -106,13 +100,13 @@ def split_scan_data(data: dict, options={}) -> list: if options['add']: if options['log']: - aux.write_log(message=f'Addition of rois enabled. Starting addition...', options=options) + aux.write_log(message=f'... Addition of rois enabled. Starting addition...', options=options) added_edges = {'Mn': [], 'Fe': [], 'Co': [], 'Ni': []} for edge, scans in edges.items(): if options['log']: - aux.write_log(message=f'... Adding rois of the {edge}-edge...', options=options) + aux.write_log(message=f'... ... Adding rois of the {edge}-edge...', options=options) if scans: xanes_df = scans[0] @@ -121,7 +115,7 @@ def split_scan_data(data: dict, options={}) -> list: if i > 0: if options['log']: - aux.write_log(message=f'... ... Adding {i}/{len(scans)}', options=options) + aux.write_log(message=f'... ... ... Adding {i+1}/{len(scans)}', options=options) if 'xmap_roi00' in xanes_df.columns: xanes_df['xmap_roi00'] += scan['xmap_roi00'] @@ -135,11 +129,11 @@ def split_scan_data(data: dict, options={}) -> list: if options['save']: if options['log']: - aux.write_log(message=f'Saving data to {options["save_folder"]}', options=options) + aux.write_log(message=f'... Saving data to {options["save_folder"]}', options=options) if not os.path.isdir(options['save_folder']): if options['log']: - aux.write_log(message=f'... {options["save_folder"]} does not exist. Creating folder.', options=options) + aux.write_log(message=f'... ... {options["save_folder"]} does not exist. Creating folder.', options=options) os.makedirs(options['save_folder']) @@ -154,16 +148,16 @@ def split_scan_data(data: dict, options={}) -> list: if not os.path.isfile(path): scan.to_csv(path) if options['log']: - aux.write_log(message=f'... Scan saved to {path}', options=options) + aux.write_log(message=f'... ... Scan saved to {path}', options=options) elif options['replace'] and os.path.isfile(path): scan.to_csv(path) if options['log']: - aux.write_log(message=f'... File already exists. Overwriting to {path}', options=options) + aux.write_log(message=f'... ... File already exists. Overwriting to {path}', options=options) elif not options['replace'] and os.path.isfile(path): if options['log']: - aux.write_log(message=f'... File already exists. Skipping...', options=options) + aux.write_log(message=f'... ... File already exists. Skipping...', options=options) all_scans.append(edges) From 1cf949e36bbd1c9ef8e377535c0782f7507ccefa Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 21 Jun 2022 18:02:08 +0200 Subject: [PATCH 187/355] Start clean-up of smoothing --- nafuma/xanes/calib.py | 97 ++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 38 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 10afda9..4692446 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -126,7 +126,7 @@ def pre_edge_subtraction(data: dict, options={}): required_options = ['log', 'logfile', 'save_plots', 'save_folder'] default_options = { 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S.log")}_pre_edge_subtraction.log', + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_subtraction.log', 'save_plots': False, 'save_folder': './' } @@ -162,10 +162,10 @@ def pre_edge_subtraction(data: dict, options={}): def estimate_edge_position(data: dict, options={}, index=0): #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. - required_options = ['print','periods'] + required_options = ['log','logfile', 'periods'] default_options = { - - 'print': False, + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -189,25 +189,29 @@ def estimate_edge_position(data: dict, options={}, index=0): def post_edge_fit(data: dict, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) - required_options = ['post_edge_start', 'print'] + required_options = ['log', 'logfile', 'post_edge_interval'] default_options = { - 'post_edge_start': None, - 'print': False + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', + 'post_edge_interval': [None, None], } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #FIXME Allow min and max limits - if not options['post_edge_start']: + if not options['post_edge_interval'][0]: post_edge_limit_offset = 0.03 data['edge'] = find_element(data) edge_position = estimate_edge_position(data, options, index=0) - post_edge_limit = edge_position + post_edge_limit_offset + options['post_edge_interval'][0] = edge_position + post_edge_limit_offset - post_edge_data = data['xanes_data_original'].loc[data['xanes_data_original']["ZapEnergy"] > post_edge_limit] + if not options['post_edge_interval'][1]: + options['post_edge_interval'][1] = data['xanes_data_original']['ZapEnergy'].max() + + + post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_interval'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_interval'][1])] post_edge_data.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data @@ -253,40 +257,31 @@ def post_edge_fit(data: dict, options={}): return post_edge_fit_data -def smoothing(path, options={}): - required_options = ['print','window_length','polyorder'] +def smoothing(data: dict, options={}): + + # FIXME Add logging + # FIXME Add saving of files + + required_options = ['log', 'logfile', 'window_length','polyorder'] default_options = { - 'print': False, + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', + 'save_plots': False, + 'save_folder': './', 'window_length': 3, 'polyorder': 2 } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - df_bkgd_sub, df_postedge, filenames, edge = post_edge_fit(path,options=options) - #================= SMOOTHING - df_smooth = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) - df_default = pd.DataFrame(df_bkgd_sub["ZapEnergy"]) - #df_smooth[filenames] = df_bkgd_sub.iloc[:,2].rolling(window=rolling_av).mean() - #df_smooth[filenames] = df_smooth[filenames].shift(-int((rolling_av)/2)) - for filename in filenames: - x_smooth=savgol_filter(df_bkgd_sub[filename], options['window_length'],options['polyorder']) - df_smooth[filename] = x_smooth - x_default=savgol_filter(df_bkgd_sub[filename],default_options['window_length'],default_options['polyorder']) - df_default[filename] = x_default - - + + # FIXME Add other types of filters + for filename in data['path']: + xanes_smooth = savgol_filter(data['xanes_data'][filename], options['window_length'], options['polyorder']) + default_smooth = savgol_filter(data['xanes_data'][filename], default_options['window_length'], default_options['polyorder']) + #printing the smoothed curves vs data - if options['print'] == True: - - ## ================================================ - #df_diff = pd.DataFrame(df_smooth["ZapEnergy"]) - #df_diff_estimated_max = df_diff[filenames].dropna().max() - - - #estimated_edge_shift=df_diff.loc[df_diff[filenames] == df_diff_max,'ZapEnergy'].values[0] - # ========================================== - + if options['save_folder'] == True: fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) x_range_zoom=[6.54,6.55] #make into widget @@ -303,8 +298,34 @@ def smoothing(path, options={}): ax2.set_xlim(x_range_zoom) ax2.set_ylim(y_range_zoom) ax2.set_title("Smoothed curve (green) vs data (red) using default window_length and polyorder") + + + # FIXME Clear up these two plotting functions + + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_fit.png' + + fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) + pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) + ax1.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') + ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) + + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) + pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) + ax2.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') + ax2.set_xlim([min(pre_edge_data['ZapEnergy']), max(pre_edge_data['ZapEnergy'])]) + ax2.set_ylim([min(pre_edge_data[filename]), max(pre_edge_data[filename])]) + ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) + + + plt.savefig(dst, transparent=False) + plt.close() - return df_smooth, filenames + return xanes_smooth, default_smooth From 9e39135f0022903502fd81d6ba879da6e4bd8de0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 21 Jun 2022 19:04:04 +0200 Subject: [PATCH 188/355] Update smoothing function --- nafuma/xanes/calib.py | 89 +++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 4692446..d81ac0f 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -262,70 +262,75 @@ def smoothing(data: dict, options={}): # FIXME Add logging # FIXME Add saving of files - required_options = ['log', 'logfile', 'window_length','polyorder'] + required_options = ['log', 'logfile', 'window_length','polyorder', 'save_default'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', 'save_plots': False, 'save_folder': './', 'window_length': 3, - 'polyorder': 2 + 'polyorder': 2, + 'save_default': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + if options['save_default']: + data['xanes_data_smooth_default'] = data['xanes_data']['ZapEnergy'] # FIXME Add other types of filters + # FIXME Instead of assigning values directly to the data dictionary, these should be made into an own DataFrame that you can decide later what to do with - these variables should + # then be returned for filename in data['path']: xanes_smooth = savgol_filter(data['xanes_data'][filename], options['window_length'], options['polyorder']) - default_smooth = savgol_filter(data['xanes_data'][filename], default_options['window_length'], default_options['polyorder']) + if options['save_default']: + default_smooth = savgol_filter(data['xanes_data'][filename], default_options['window_length'], default_options['polyorder']) + data['xanes_data'][filename] = xanes_smooth - #printing the smoothed curves vs data - if options['save_folder'] == True: - - fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) - x_range_zoom=[6.54,6.55] #make into widget - y_range_zoom=[20000,80000] #make into widget - - df_bkgd_sub.plot.scatter(x = "ZapEnergy",y=filenames, ax=ax1, color="Red") - df_smooth.plot(x = "ZapEnergy",y=filenames, ax=ax1, color="Blue") - ax1.set_xlim(x_range_zoom) - ax1.set_ylim(y_range_zoom) - ax1.set_title("Smoothed curve (blue) vs data (red) used for further analysis") - - df_bkgd_sub.plot.scatter(x = "ZapEnergy",y=filenames, ax=ax2, color="Red") - df_default.plot(x = "ZapEnergy",y=filenames, ax=ax2, color="Green") - ax2.set_xlim(x_range_zoom) - ax2.set_ylim(y_range_zoom) - ax2.set_title("Smoothed curve (green) vs data (red) using default window_length and polyorder") + if options['save_default']: + data['xanes_data_smooth_default'][filename] = default_smooth - # FIXME Clear up these two plotting functions + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - if options['save_plots']: - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_smooth.png' - dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_fit.png' - - fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) - data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) - pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) - ax1.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') - ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) - - data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) - pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) - ax2.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') - ax2.set_xlim([min(pre_edge_data['ZapEnergy']), max(pre_edge_data['ZapEnergy'])]) - ax2.set_ylim([min(pre_edge_data[filename]), max(pre_edge_data[filename])]) - ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) + edge_pos = estimate_edge_position(data=data, options=options) + intensity_midpoint = data['xanes_data'][filename].max() - data['xanes_data'][filename].min() - plt.savefig(dst, transparent=False) - plt.close() + if options['save_default']: + fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) + xanes_smooth.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) + ax1.set_xlim([edge_pos-0.5, edge_pos+0.5]) + ax1.set_ylim([intensity_midpoint*0.98, intensity_midpoint*1.02]) + + ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) + + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) + data['xanes_data_smooth_default'].plot(x='ZapEnergy', y=filename, color='green', ax=ax2) + ax2.set_xlim([edge_pos-0.5, edge_pos+0.5]) + ax2.set_ylim([intensity_midpoint*0.98, intensity_midpoint*1.02]) + ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20) + + elif not options['save_default']: + fig, ax = plt.subplots(figsize=(10,5)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) + xanes_smooth.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) + ax1.set_xlim([edge_pos-0.5, edge_pos+0.5]) + ax1.set_ylim([intensity_midpoint*0.98, intensity_midpoint*1.02]) + + ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) + + + plt.savefig(dst, transparent=False) + plt.close() - return xanes_smooth, default_smooth + # FIXME See comment above about return values + return None From 4d501adb729b4d48f0a3f5417e418a33462e2547 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 22 Jun 2022 15:56:34 +0200 Subject: [PATCH 189/355] Complete smooth and get determine_edge_position going --- nafuma/xanes/calib.py | 351 ++++++++++++++++++++++-------------------- 1 file changed, 187 insertions(+), 164 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index d81ac0f..dfe5b07 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -1,3 +1,5 @@ +from logging import raiseExceptions +from jinja2 import TemplateRuntimeError import pandas as pd import numpy as np import os @@ -160,31 +162,7 @@ def pre_edge_subtraction(data: dict, options={}): return xanes_data_bkgd_subtracted -def estimate_edge_position(data: dict, options={}, index=0): - #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. - required_options = ['log','logfile', 'periods'] - default_options = { - 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', - 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly - } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #making new dataframe to keep the differentiated data - df_diff = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) - df_diff[data['path'][index]]=data['xanes_data_original'][data['path'][index]].diff(periods=options['periods']) - - #shifting column values up so that average differential fits right between the points used in the calculation - df_diff[data['path'][index]]=df_diff[data['path'][index]].shift(-int(options['periods']/2)) - df_diff_max = df_diff[data['path'][index]].dropna().max() - estimated_edge_shift =df_diff.loc[df_diff[data['path'][index]] == df_diff_max,'ZapEnergy'].values[0] - - # FIXME Add logging option to see the result - - if options['log']: - aux.write_log(message=f'Estimated edge shift for determination of pre-edge area is: {estimated_edge_shift} keV', options=options) - - return estimated_edge_shift def post_edge_fit(data: dict, options={}): @@ -274,22 +252,20 @@ def smoothing(data: dict, options={}): } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + df_smooth = pd.DataFrame(data['xanes_data']['ZapEnergy']) + if options['save_default']: - data['xanes_data_smooth_default'] = data['xanes_data']['ZapEnergy'] + df_smooth_default = pd.DataFrame(data['xanes_data']['ZapEnergy']) # FIXME Add other types of filters # FIXME Instead of assigning values directly to the data dictionary, these should be made into an own DataFrame that you can decide later what to do with - these variables should # then be returned for filename in data['path']: - xanes_smooth = savgol_filter(data['xanes_data'][filename], options['window_length'], options['polyorder']) - if options['save_default']: - default_smooth = savgol_filter(data['xanes_data'][filename], default_options['window_length'], default_options['polyorder']) - - data['xanes_data'][filename] = xanes_smooth + df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['window_length'], options['polyorder'])) if options['save_default']: - data['xanes_data_smooth_default'][filename] = default_smooth - + df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['window_length'], default_options['polyorder'])) + if options['save_plots']: if not os.path.isdir(options['save_folder']): @@ -298,39 +274,35 @@ def smoothing(data: dict, options={}): dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_smooth.png' edge_pos = estimate_edge_position(data=data, options=options) - intensity_midpoint = data['xanes_data'][filename].max() - data['xanes_data'][filename].min() - - + intensity_midpoint = df_smooth[filename].iloc[np.where(df_smooth['ZapEnergy'] == find_nearest(df_smooth['ZapEnergy'], edge_pos))].values[0] + if options['save_default']: fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5)) - data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) - xanes_smooth.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) - ax1.set_xlim([edge_pos-0.5, edge_pos+0.5]) - ax1.set_ylim([intensity_midpoint*0.98, intensity_midpoint*1.02]) - + data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter') + df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax1) ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) - data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) - data['xanes_data_smooth_default'].plot(x='ZapEnergy', y=filename, color='green', ax=ax2) - ax2.set_xlim([edge_pos-0.5, edge_pos+0.5]) - ax2.set_ylim([intensity_midpoint*0.98, intensity_midpoint*1.02]) + data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter') + df_smooth_default.loc[(df_smooth_default['ZapEnergy'] > edge_pos-0.0015) & (df_smooth_default['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20) elif not options['save_default']: fig, ax = plt.subplots(figsize=(10,5)) - data['xanes_data'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) - xanes_smooth.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) - ax1.set_xlim([edge_pos-0.5, edge_pos+0.5]) - ax1.set_ylim([intensity_midpoint*0.98, intensity_midpoint*1.02]) + data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax, kind='scatter') + df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax) + ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015]) + ax.set_ylim([intensity_midpoint*0.9, intensity_midpoint*1.1]) - ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) + ax.set_title(f'{os.path.basename(filename)} - Smooth', size=20) plt.savefig(dst, transparent=False) plt.close() - # FIXME See comment above about return values - return None + if not options['save_default']: + df_smooth_default = None + + return df_smooth, df_smooth_default @@ -340,133 +312,184 @@ def find_nearest(array, value): idx = (np.abs(array - value)).argmin() return array[idx] -def finding_e0(path, options={}): - required_options = ['print','periods'] + +def estimate_edge_position(data: dict, options={}, index=0): + #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. + required_options = ['log','logfile', 'periods'] default_options = { - 'print': False, + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - df_smooth, filenames = smoothing(path, options=options) #This way the smoothing is printed as long as the "finding e0" is printed. - - if options['periods'] % 2 == 1: - print("NB!!!!!!!!!!!!!!!!! Periods needs to be an even number for the shifting of values to work properly") - ###df_diff = pd.DataFrame(df_smooth["ZapEnergy"]) # - if len(filenames) == 1: - filenames=filenames[0] - else: - print("MORE THAN ONE FILE --> generalize") - - ##### - estimated_edge_shift, df_diff, df_diff_max = estimate_edge_position(df_smooth, filenames,options=options) - print(estimated_edge_shift) - #### - ###df_diff[filenames]=df_smooth[filenames].diff(periods=options['periods']) # - df_doublediff=pd.DataFrame(df_smooth["ZapEnergy"]) - df_doublediff[filenames]=df_diff[filenames].diff(periods=options['periods']) - - if options['print'] == True: - fig, (ax1,ax2) = plt.subplots(1,2,figsize=(15,5)) - - df_diff.plot(x = "ZapEnergy",y=filenames, ax=ax1) #defining x and y - df_doublediff.plot(x = "ZapEnergy",y=filenames,ax=ax2) #defining x and y + #making new dataframe to keep the differentiated data + df_diff = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) + df_diff[data['path'][index]]=data['xanes_data_original'][data['path'][index]].diff(periods=options['periods']) #shifting column values up so that average differential fits right between the points used in the calculation - #df_diff[filenames]=df_diff[filenames].shift(-int(options['periods']/2)) # - df_doublediff[filenames]=df_doublediff[filenames].shift(-int(options['periods'])) + df_diff[data['path'][index]]=df_diff[data['path'][index]].shift(-int(options['periods']/2)) + df_diff_max = df_diff[data['path'][index]].dropna().max() + estimated_edge_shift =df_diff.loc[df_diff[data['path'][index]] == df_diff_max,'ZapEnergy'].values[0] + + # FIXME Add logging option to see the result + + if options['log']: + aux.write_log(message=f'Estimated edge shift for determination of pre-edge area is: {estimated_edge_shift} keV', options=options) + + return estimated_edge_shift + +def determine_edge_position(data: dict, options={}): - #finding maximum value to maneuver to the correct part of the data set - #df_diff_max = df_diff[filenames].dropna().max() - + required_options = ['log', 'logfile', 'save_plots', 'save_folder', 'periods', 'diff', 'double_diff', 'fit_region'] + default_options = { + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_determine_edge_position.log', + 'save_plots': False, + 'save_folder': './', + 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly, + 'diff': True, + 'double_diff': False, + 'fit_region': 0.0005 - estimated_edge_shift=df_diff.loc[df_diff[filenames] == df_diff_max,'ZapEnergy'].values[0] + } - fit_region = 0.0004 - df_diff_edge=df_diff.loc[(df_diff["ZapEnergy"] < estimated_edge_shift+fit_region)]# and (df_diff["ZapEnergy"] > estimated_edge_shift-0.05)] - df_diff_edge=df_diff_edge.loc[(df_diff["ZapEnergy"] > estimated_edge_shift-fit_region)] - - - - - df_doublediff_edge=df_doublediff.loc[(df_doublediff["ZapEnergy"] < estimated_edge_shift+fit_region)]# and (df_diff["ZapEnergy"] > estimated_edge_shift-0.05)] - df_doublediff_edge=df_doublediff_edge.loc[(df_doublediff["ZapEnergy"] > estimated_edge_shift-fit_region)] - #df_diff_edge=df_diff.loc[(df_diff["ZapEnergy"] > estimated_edge_shift-0.15) and (df_diff["ZapEnergy"] < estimated_edge_shift+0.15)] + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #df_diff_edge=df_diff.loc[df_diff["ZapEnergy"] > estimated_edge_shift-0.15] - #print(df_diff_edge) - if options['print'] == True: - fig, (ax3,ax4) = plt.subplots(1,2,figsize=(15,5)) + if options['periods'] % 2 == 1: + raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") - df_diff_edge.plot(x = "ZapEnergy",y=filenames,ax=ax3) #defining x and y - ax3.set_title("Zoomed into edge region (derivative))") - ax3.axvline(x = estimated_edge_shift) - - df_doublediff_edge.plot(x = "ZapEnergy",y=filenames,ax=ax4,kind="scatter") #defining x and y - ax4.set_title("Zoomed into edge region (double derivative)") - ax4.axvline(x = estimated_edge_shift) - ax4.axhline(0) - - - - #ax1.set_xlim([estimated_edge_shift-fit_region,estimated_edge_shift+fit_region]) - #ax1.set_title("not sure what this is tbh") - - #ax2.set_xlim([estimated_edge_shift-fit_region,estimated_edge_shift+fit_region]) - #ax2.set_title("not sure what this is either tbh") - - #============== - #df_smooth=df_smooth2 - #================= - - - - - #========================== fitting first differential ========== - df_diff = df_diff[df_diff[filenames].notna()] - - #fitting a function to the chosen interval - d = np.polyfit(df_diff_edge["ZapEnergy"],df_diff_edge[filenames],2) - function_diff = np.poly1d(d) - - x_diff=np.linspace(df_diff_edge["ZapEnergy"].iloc[0],df_diff_edge["ZapEnergy"].iloc[-1],num=1000) - y_diff=function_diff(x_diff) - #print(df_diff_edge["ZapEnergy"].iloc[-1]) - if options['print'] == True: - ax3.plot(x_diff,y_diff,color='Green') - - #y_diff_max=np.amax(y_diff,0) - y_diff_max_index = np.where(y_diff == np.amax(y_diff)) - #print(y_diff_max_index[0]) - edge_shift_diff=float(x_diff[y_diff_max_index]) - print("Edge shift estimated by the differential maximum is "+str(round(edge_shift_diff,5))) - if options['print'] == True: - ax3.axvline(x=edge_shift_diff,color="green") - #print(df_doublediff_edge["ZapEnergy"].iloc[0]) - #ax4.plot(x_doublediff,y_doublediff,color='Green')) - - - #fitting double differentiate - df_doublediff = df_doublediff[df_doublediff[filenames].notna()] - d = np.polyfit(df_doublediff_edge["ZapEnergy"],df_doublediff_edge[filenames],2) - function_doublediff = np.poly1d(d) - - x_doublediff=np.linspace(df_doublediff_edge["ZapEnergy"].iloc[0],df_doublediff_edge["ZapEnergy"].iloc[-1],num=10000) - y_doublediff=function_doublediff(x_doublediff) - - if options['print'] == True: - ax4.plot(x_doublediff,y_doublediff,color='Green') - - y_doublediff_zero=find_nearest(y_doublediff,0) - y_doublediff_zero_index = np.where(y_doublediff == y_doublediff_zero) - - edge_shift_doublediff=float(x_doublediff[y_doublediff_zero_index]) - print("Edge shift estimated by the double differential zero-point is "+str(round(edge_shift_doublediff,5))) - if options['print'] == True: - ax4.axvline(x=edge_shift_doublediff,color="green") + ##### - return df_smooth, filenames, edge_shift_diff + if options['diff']: + df_diff = pd.DataFrame(data['xanes_data']['ZapEnergy']) + if options['double_diff']: + df_double_diff = pd.DataFrame(data['xanes_data']['ZapEnergy']) + + for i, filename in enumerate(data['path']): + estimated_edge_pos = estimate_edge_position(data, options=options, index=i) + + + #========================== fitting first differential ========== + + if options['diff']: + df_diff[filename] = data['xanes_data'][filename].diff(periods=options['periods']) + df_diff[filename]=df_diff[filename].shift(-int(options['periods']/2)) + + df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] + + + # Fitting a function to the chosen interval + params = np.polyfit(df_diff_edge["ZapEnergy"], df_diff_edge[filename], 2) + diff_function = np.poly1d(params) + + x_diff=np.linspace(df_diff_edge["ZapEnergy"].iloc[0],df_diff_edge["ZapEnergy"].iloc[-1],num=10000) + y_diff=diff_function(x_diff) + + df_diff_fit_function = pd.DataFrame(x_diff) + df_diff_fit_function['y_diff'] = y_diff + df_diff_fit_function.columns = ['x_diff', 'y_diff'] + + # Picks out the x-value where the y-value is at a maximum + edge_pos_diff=x_diff[np.where(y_diff == np.amax(y_diff))][0] + + if options['log']: + aux.write_log(message=f"Edge shift estimated by the differential maximum is: {str(round(edge_pos_diff,5))}", options=options) + + + if options['double_diff']: + df_double_diff[filename] = data['xanes_data'][filename].diff(periods=options['periods']).diff(periods=options['periods']) + df_double_diff[filename]=df_double_diff[filename].shift(-int(options['periods'])) + + # Pick out region of interest + df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] + + # Fitting a function to the chosen interval + params = np.polyfit(df_double_diff_edge["ZapEnergy"], df_double_diff_edge[filename], 2) + double_diff_function = np.poly1d(params) + + x_double_diff=np.linspace(df_double_diff_edge["ZapEnergy"].iloc[0], df_double_diff_edge["ZapEnergy"].iloc[-1],num=10000) + y_double_diff=double_diff_function(x_double_diff) + + df_double_diff_fit_function = pd.DataFrame(x_double_diff) + df_double_diff_fit_function['y_diff'] = y_double_diff + df_double_diff_fit_function.columns = ['x_diff', 'y_diff'] + + + # Picks out the x-value where the y-value is closest to 0 + edge_pos_double_diff=x_double_diff[np.where(y_double_diff == find_nearest(y_double_diff,0))][0] + + if options['log']: + aux.write_log(message=f"Edge shift estimated by the double differential zero-point is {str(round(edge_pos_double_diff,5))}", options=options) + + if options['save_plots']: + + if options['diff'] and options['double_diff']: + + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, figsize=(20,20)) + df_diff.plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax1) + ax1.set_xlim([edge_pos_diff-0.0015, edge_pos_diff+0.0015]) + ax1.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') + ax1.axvline(x=edge_pos_diff, ls='--', c='green') + ax1.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') + ax1.set_title('Fit region of differentiated data') + + df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.axvline(x=edge_pos_diff, ls='--', c='green') + ax2.axvline(x=estimated_edge_pos, ls='--', c='red') + ax2.set_title('Fit of differentiated data') + + + df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015]) + ax3.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') + ax3.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax3.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') + + df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax4, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax4) + ax4.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax4.axvline(x=estimated_edge_pos, ls='--', c='red') + + + + + elif options['diff']: + fig, (ax1, ax2) = plt.subplots(ncols=2,nrows=1, figsize=(20, 10)) + df_diff.plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter') + ax1.set_xlim([edge_pos_diff-0.5, edge_pos_diff+0.5]) + ax1.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') + ax1.axvline(x=edge_pos_diff, ls='--', c='green') + ax1.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') + + df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax2) + ax2.axvline(x=edge_pos_diff, ls='--', c='green') + ax2.axvline(x=estimated_edge_pos, ls='--', c='red') + + + elif options['double_diff']: + fig, (ax1, ax2) = plt.subplots(ncols=2,nrows=1, figsize=(20, 10)) + df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter') + ax1.set_xlim([edge_pos_double_diff-0.5, edge_pos_double_diff+0.5]) + ax1.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') + ax1.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax1.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') + + df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax2) + ax2.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax2.axvline(x=estimated_edge_pos, ls='--', c='red') + + + if not options['diff']: + edge_pos_diff = None + if not options['double_diff']: + edge_pos_double_diff = None + + return edge_pos_diff, edge_pos_double_diff def normalization(data,options={}): required_options = ['print'] From ec1fba1c829a478d4e3ac3364617660dd9c37009 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 23 Jun 2022 11:46:06 +0200 Subject: [PATCH 190/355] Refactor normalisation and flattening functions --- nafuma/xanes/calib.py | 45 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index dfe5b07..5c111f5 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -491,25 +491,52 @@ def determine_edge_position(data: dict, options={}): return edge_pos_diff, edge_pos_double_diff -def normalization(data,options={}): - required_options = ['print'] +def normalise(data: dict, options={}): + required_options = ['log', 'logfile', 'save_values'] default_options = { - 'print': False, + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log', + 'save_values': True } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - #Finding the normalization constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 - normalization_constant=post_edge_fit_function(e0) - pre_edge_fit_function(e0) - - #subtracting background (as in pre_edge_subtraction) + normalised_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) - #dividing the background-subtracted data with the normalization constant + #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 + for filename in data['path']: + normalisation_constant = data['post_edge_fit_function'][filename].loc[data['post_edge_fit_function']['ZapEnergy'] == data['e0'][filename]] - data['pre_edge_fit_function'].loc[data['pre_edge_fit_function']['ZapEnergy'] == data['e0'][filename]] + + normalised_df.insert(1, filename, data['xanes_data'] / normalisation_constant) + + if options['save_values']: + data['xanes_data'] = normalised_df + + + return normalised_df -def flattening(data,options={}): +def flatten(data:dict, options={}): #only picking out zapenergy-values higher than edge position (edge pos and below remains untouched) + + required_options = ['log', 'logfile', 'save_values'] + default_options = { + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_flattening.log', + 'save_values': True + } + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + df_e0_and_above=df.loc[df['ZapEnergy'] > edge_shift_diff] + flattened_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) + + for filename in data['path']: + above_e0 = data['xanes_data'][filename].loc(data['xanes_data']['ZapEnergy'] > data['e0'][filename]) + flattened_data = data['post_edge_fit_function'][filename] - + + + flattened_data = post_edge_fit_function(df_e0_and_above['ZapEnergy']) - pre_edge_fit_function(df_e0_and_above['ZapEnergy']) #make a new dataframe with flattened values From 2b14a64c4bbab1590e6028a0f75ed5d699372ff6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 23 Jun 2022 15:32:29 +0200 Subject: [PATCH 191/355] Attempt to get flattening and normalisation to behave properly --- nafuma/xanes/calib.py | 45 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 5c111f5..2b12c78 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -78,6 +78,8 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data pre_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) + data['pre_edge_params'] = {} + for i, filename in enumerate(data['path']): if options['log']: aux.write_log(message=f'Fitting background on {os.path.basename(filename)} ({i+1} / {len(data["path"])})', options=options) @@ -85,6 +87,8 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: #Fitting linear function to the background params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],1) fit_function = np.poly1d(params) + + data['pre_edge_params'][filename] = params #making a list, y_pre,so the background will be applied to all ZapEnergy-values background=fit_function(pre_edge_fit_data["ZapEnergy"]) @@ -194,6 +198,8 @@ def post_edge_fit(data: dict, options={}): # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data post_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) + + data['post_edge_params'] = {} for i, filename in enumerate(data['path']): if options['log']: @@ -202,6 +208,8 @@ def post_edge_fit(data: dict, options={}): #Fitting linear function to the background params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], 2) fit_function = np.poly1d(params) + + data['post_edge_params'][filename] = params #making a list, y_pre,so the background will be applied to all ZapEnergy-values background=fit_function(post_edge_fit_data["ZapEnergy"]) @@ -341,8 +349,9 @@ def estimate_edge_position(data: dict, options={}, index=0): def determine_edge_position(data: dict, options={}): - required_options = ['log', 'logfile', 'save_plots', 'save_folder', 'periods', 'diff', 'double_diff', 'fit_region'] + required_options = ['save_values', 'log', 'logfile', 'save_plots', 'save_folder', 'periods', 'diff', 'double_diff', 'fit_region'] default_options = { + 'save_values': True, 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_determine_edge_position.log', 'save_plots': False, @@ -366,6 +375,9 @@ def determine_edge_position(data: dict, options={}): df_diff = pd.DataFrame(data['xanes_data']['ZapEnergy']) if options['double_diff']: df_double_diff = pd.DataFrame(data['xanes_data']['ZapEnergy']) + if options['save_values']: + data['e0'] = {} + for i, filename in enumerate(data['path']): estimated_edge_pos = estimate_edge_position(data, options=options, index=i) @@ -395,7 +407,10 @@ def determine_edge_position(data: dict, options={}): edge_pos_diff=x_diff[np.where(y_diff == np.amax(y_diff))][0] if options['log']: - aux.write_log(message=f"Edge shift estimated by the differential maximum is: {str(round(edge_pos_diff,5))}", options=options) + aux.write_log(message=f"Edge position estimated by the differential maximum is: {str(round(edge_pos_diff,5))}", options=options) + + if options['save_values']: + data['e0'][filename] = edge_pos_diff if options['double_diff']: @@ -501,12 +516,21 @@ def normalise(data: dict, options={}): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) normalised_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) + data['normalisation_constants'] = {} #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 for filename in data['path']: - normalisation_constant = data['post_edge_fit_function'][filename].loc[data['post_edge_fit_function']['ZapEnergy'] == data['e0'][filename]] - data['pre_edge_fit_function'].loc[data['pre_edge_fit_function']['ZapEnergy'] == data['e0'][filename]] + e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0'][filename])].index.values[0] + #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] + normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] + normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant) - normalised_df.insert(1, filename, data['xanes_data'] / normalisation_constant) + + # Normalise the pre-edge and post-edge fit function data + data['pre_edge_fit_data'][filename] = data['pre_edge_fit_data'][filename] / normalisation_constant + data['post_edge_fit_data'][filename] = data['post_edge_fit_data'][filename] / normalisation_constant + + data['normalisation_constants'][filename] = normalisation_constant if options['save_values']: data['xanes_data'] = normalised_df @@ -527,17 +551,20 @@ def flatten(data:dict, options={}): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - df_e0_and_above=df.loc[df['ZapEnergy'] > edge_shift_diff] - flattened_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) for filename in data['path']: - above_e0 = data['xanes_data'][filename].loc(data['xanes_data']['ZapEnergy'] > data['e0'][filename]) - flattened_data = data['post_edge_fit_function'][filename] - + fit_function_diff = -data['post_edge_fit_data'][filename] + data['pre_edge_params'][filename][0] + fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0'][filename]] = 0 + + flattened_df[filename] = data['xanes_data'][filename] - fit_function_diff + if options['save_values']: + data['xanes_data'] = flattened_df + - flattened_data = post_edge_fit_function(df_e0_and_above['ZapEnergy']) - pre_edge_fit_function(df_e0_and_above['ZapEnergy']) + return flattened_df, fit_function_diff #make a new dataframe with flattened values From 726535c66fab28b9863a923be16b55e6a4955476 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 24 Jun 2022 19:28:55 +0200 Subject: [PATCH 192/355] fixing the "add_rois"-option --- nafuma/xanes/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index f623a38..87e5ac3 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -97,7 +97,7 @@ def split_scan_data(data: dict, options={}) -> list: edges[edge].append(xanes_df) - if options['add']: + if options['add_rois']: if options['log']: aux.write_log(message=f'... Addition of rois enabled. Starting addition...', options=options) @@ -142,7 +142,7 @@ def split_scan_data(data: dict, options={}) -> list: for edge, scans in edges.items(): for i, scan in enumerate(scans): - count = '' if options['add'] else '_'+str(i).zfill(4) + count = '' if options['add_rois'] else '_'+str(i).zfill(4) path = os.path.join(options['save_folder'], f'{filename}_{edge}{count}.dat') if not os.path.isfile(path): From 931b3e42ae2aff55dbaa65b2cc6ab326b9d7104b Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 24 Jun 2022 19:39:11 +0200 Subject: [PATCH 193/355] adding a fixme for split_scan_data-function --- nafuma/xanes/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 87e5ac3..9676608 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -127,7 +127,7 @@ def split_scan_data(data: dict, options={}) -> list: edges = added_edges if options['save']: - + #FIXME If there is something wrong with the input file, the file will not be saved but log-file still sais it is saved. Goes from "Saving data to ..." to "All done!" no matter if it fals or not. if options['log']: aux.write_log(message=f'... Saving data to {options["save_folder"]}', options=options) From 8e0d8f486155cab13011287398009ecaf4db578a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 27 Jun 2022 12:20:49 +0200 Subject: [PATCH 194/355] ADd interactive mode for pre and post edge fitting --- nafuma/xanes/calib.py | 129 +++++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 33 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 2b12c78..911af8d 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -5,10 +5,13 @@ import numpy as np import os import matplotlib.pyplot as plt import nafuma.auxillary as aux +import nafuma.plotting as btp import nafuma.xanes as xas import nafuma.xanes.io as io from scipy.signal import savgol_filter from datetime import datetime +import ipywidgets as widgets +from IPython.display import display ##Better to make a new function that loops through the files, and performing the split_xanes_scan on @@ -43,13 +46,15 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Add log-file - required_options = ['pre_edge_start', 'log', 'logfile', 'save_plots', 'save_folder'] + required_options = ['pre_edge_limit', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'interactive'] default_options = { - 'pre_edge_start': None, + 'pre_edge_limit': [None, None], 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', + 'show_plots': False, 'save_plots': False, - 'save_folder': './' + 'save_folder': './', + 'interactive': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -57,23 +62,34 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: if options['log']: aux.write_log(message='Starting pre edge fit', options=options) - - # FIXME Implement with finding accurate edge position # FIXME Allow specification of start of pre-edge area # Find the cutoff point at which the edge starts - everything to the LEFT of this point will be used in the pre edge function fit - if not options['pre_edge_start']: + if not options['pre_edge_limit'][0]: + options['pre_edge_limit'][0] = data['xanes_data_original']['ZapEnergy'].min() + + if not options['pre_edge_limit'][1]: pre_edge_limit_offset = 0.03 data['edge'] = find_element(data) edge_position = estimate_edge_position(data, options, index=0) - pre_edge_limit = edge_position - pre_edge_limit_offset + options['pre_edge_limit'][1] = edge_position - pre_edge_limit_offset + + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session + if options['interactive']: + options['interactive'] = False + options['interactive_session_active'] = True + options['show_plots'] = True + pre_edge_fit_interactive(data=data, options=options) + return + + # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to # limit the interval # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) - pre_edge_data = data['xanes_data_original'].loc[data['xanes_data_original']["ZapEnergy"] < pre_edge_limit] + pre_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['pre_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['pre_edge_limit'][1])] # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data pre_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) @@ -96,16 +112,12 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: #adding a new column in df_background with the y-values of the background pre_edge_fit_data.insert(1,filename,background) - if options['save_plots']: - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) - - dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_fit.png' - - fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) + if options['show_plots'] or options['save_plots']: + fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,10)) data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) ax1.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') + ax1.axvline(x = min(pre_edge_data['ZapEnergy']), ls='--') ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) @@ -115,9 +127,15 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: ax2.set_ylim([min(pre_edge_data[filename]), max(pre_edge_data[filename])]) ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - plt.savefig(dst, transparent=False) - plt.close() + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_fit.png' + plt.savefig(dst, transparent=False) + + if not options['show_plots']: + plt.close() if options['log']: @@ -127,6 +145,21 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: +def pre_edge_fit_interactive(data: dict, options: dict) -> None: + + + w = widgets.interactive( + btp.ipywidgets_update, func=widgets.fixed(pre_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), + pre_edge_limit=widgets.FloatRangeSlider(value=[options['pre_edge_limit'][0], options['pre_edge_limit'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) + ) + + options['widget'] = w + + display(w) + + + + def pre_edge_subtraction(data: dict, options={}): required_options = ['log', 'logfile', 'save_plots', 'save_folder'] @@ -171,30 +204,43 @@ def pre_edge_subtraction(data: dict, options={}): def post_edge_fit(data: dict, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) - required_options = ['log', 'logfile', 'post_edge_interval'] + required_options = ['log', 'logfile', 'post_edge_limit', 'interactive'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', - 'post_edge_interval': [None, None], + 'post_edge_limit': [None, None], + 'interactive': False, + 'show_plots': False, + 'save_plots': False, + 'save_folder': './', } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - if not options['post_edge_interval'][0]: + if not options['post_edge_limit'][0]: post_edge_limit_offset = 0.03 data['edge'] = find_element(data) edge_position = estimate_edge_position(data, options, index=0) - options['post_edge_interval'][0] = edge_position + post_edge_limit_offset + options['post_edge_limit'][0] = edge_position + post_edge_limit_offset - if not options['post_edge_interval'][1]: - options['post_edge_interval'][1] = data['xanes_data_original']['ZapEnergy'].max() + if not options['post_edge_limit'][1]: + options['post_edge_limit'][1] = data['xanes_data_original']['ZapEnergy'].max() + + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session + if options['interactive']: + options['interactive'] = False + options['interactive_session_active'] = True + options['show_plots'] = True + post_edge_fit_interactive(data=data, options=options) + return - post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_interval'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_interval'][1])] - post_edge_data.dropna(inplace=True) #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + + post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_limit'][1])] + post_edge_data = post_edge_data.dropna() #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data post_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) @@ -217,16 +263,14 @@ def post_edge_fit(data: dict, options={}): #adding a new column in df_background with the y-values of the background post_edge_fit_data.insert(1,filename,background) - if options['save_plots']: - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) + if options['save_plots'] or options['show_plots']: - dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_post_edge_fit.png' - fig, (ax1, ax2) = plt.subplots(1,2,figsize=(10,5)) + fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,10)) data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax1) post_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax1) ax1.axvline(x = max(post_edge_data['ZapEnergy']), ls='--') + ax1.axvline(x = min(post_edge_data['ZapEnergy']), ls='--') ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) @@ -236,13 +280,32 @@ def post_edge_fit(data: dict, options={}): ax2.set_ylim([min(post_edge_data[filename]), max(post_edge_data[filename])]) ax2.set_title(f'{os.path.basename(filename)} - Fit region', size=20) + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - plt.savefig(dst, transparent=False) - plt.close() + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_post_edge_fit.png' + + plt.savefig(dst, transparent=False) + + if not options['show_plots']: + plt.close() return post_edge_fit_data + +def post_edge_fit_interactive(data: dict, options: dict) -> None: + + w = widgets.interactive( + btp.ipywidgets_update, func=widgets.fixed(post_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), + post_edge_limit=widgets.FloatRangeSlider(value=[options['post_edge_limit'][0], options['post_edge_limit'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) + ) + + options['widget'] = w + + display(w) + def smoothing(data: dict, options={}): # FIXME Add logging @@ -295,7 +358,7 @@ def smoothing(data: dict, options={}): ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20) elif not options['save_default']: - fig, ax = plt.subplots(figsize=(10,5)) + fig, ax = plt.subplots(figsize=(20,10)) data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax, kind='scatter') df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax) ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015]) From 537c7b3c5ad4d8eaa8c36f33f8b7752d6029d075 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 27 Jun 2022 13:34:18 +0200 Subject: [PATCH 195/355] Add masks to pre and post edge fitting --- nafuma/xanes/calib.py | 47 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 911af8d..2e03086 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -46,14 +46,16 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Add log-file - required_options = ['pre_edge_limit', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'interactive'] + required_options = ['pre_edge_limit', 'masks', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive'] default_options = { 'pre_edge_limit': [None, None], + 'masks': [], 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', 'show_plots': False, 'save_plots': False, 'save_folder': './', + 'ylim': [None, None], 'interactive': False } @@ -67,6 +69,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # Find the cutoff point at which the edge starts - everything to the LEFT of this point will be used in the pre edge function fit if not options['pre_edge_limit'][0]: options['pre_edge_limit'][0] = data['xanes_data_original']['ZapEnergy'].min() + if not options['pre_edge_limit'][1]: pre_edge_limit_offset = 0.03 @@ -89,8 +92,13 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to # limit the interval # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) - pre_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['pre_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['pre_edge_limit'][1])] - + pre_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['pre_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['pre_edge_limit'][1])].copy() + + for mask in options['masks']: + pre_edge_data.loc[(pre_edge_data['ZapEnergy'] > mask[0]) & (pre_edge_data['ZapEnergy'] < mask[1])] = np.nan + + pre_edge_data = pre_edge_data.dropna() + # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data pre_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) @@ -119,6 +127,14 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: ax1.axvline(x = max(pre_edge_data['ZapEnergy']), ls='--') ax1.axvline(x = min(pre_edge_data['ZapEnergy']), ls='--') ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) + + if options['ylim'][0] != None: + ax1.set_ylim(bottom=options['ylim'][0]) + if options['ylim'][1]: + ax1.set_ylim(top=options['ylim'][1]) + + for mask in options['masks']: + ax1.fill_between(x=mask, y1=0, y2=data['xanes_data_original'][filename].max()*2, alpha=0.2, color='black') data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) pre_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) @@ -204,11 +220,13 @@ def pre_edge_subtraction(data: dict, options={}): def post_edge_fit(data: dict, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) - required_options = ['log', 'logfile', 'post_edge_limit', 'interactive'] + required_options = ['log', 'logfile', 'masks', 'post_edge_limit', 'interactive', 'show_plots', 'save_plots', 'save_folder'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', 'post_edge_limit': [None, None], + 'masks': [], + 'polyorder': 2, 'interactive': False, 'show_plots': False, 'save_plots': False, @@ -239,7 +257,11 @@ def post_edge_fit(data: dict, options={}): - post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_limit'][1])] + post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_limit'][1])].copy() + + for mask in options['masks']: + post_edge_data.loc[(post_edge_data['ZapEnergy'] > mask[0]) & (post_edge_data['ZapEnergy'] < mask[1])] = np.nan + post_edge_data = post_edge_data.dropna() #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data @@ -249,12 +271,15 @@ def post_edge_fit(data: dict, options={}): for i, filename in enumerate(data['path']): if options['log']: - aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])})', options=options) + aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["polyorder"]}', options=options) #Fitting linear function to the background - params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], 2) + params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], options['polyorder']) fit_function = np.poly1d(params) + if options['log']: + aux.write_log(message=f'Post edge fitted with parameters: {params}') + data['post_edge_params'][filename] = params #making a list, y_pre,so the background will be applied to all ZapEnergy-values @@ -273,6 +298,14 @@ def post_edge_fit(data: dict, options={}): ax1.axvline(x = min(post_edge_data['ZapEnergy']), ls='--') ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) + for mask in options['masks']: + ax1.fill_between(x=mask, y1=0, y2=data['xanes_data_original'][filename].max()*2, alpha=0.2, color='black') + + if options['ylim'][0] != None: + ax1.set_ylim(bottom=options['ylim'][0]) + if options['ylim'][1] != None: + ax1.set_ylim(top=options['ylim'][1]) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) post_edge_fit_data.plot(x='ZapEnergy', y=filename, color='red', ax=ax2) ax2.axvline(x = max(post_edge_data['ZapEnergy']), ls='--') From 8c2723ee552bdc642aa984d0fca8df8a21f93b6f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 27 Jun 2022 16:16:46 +0200 Subject: [PATCH 196/355] Plot full scan with computed edge position --- nafuma/xanes/calib.py | 139 +++++++++++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 49 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 2e03086..66c4229 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -220,7 +220,7 @@ def pre_edge_subtraction(data: dict, options={}): def post_edge_fit(data: dict, options={}): #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) - required_options = ['log', 'logfile', 'masks', 'post_edge_limit', 'interactive', 'show_plots', 'save_plots', 'save_folder'] + required_options = ['log', 'logfile', 'masks', 'post_edge_limit', 'polyorder', 'interactive', 'show_plots', 'save_plots', 'save_folder'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', @@ -444,24 +444,31 @@ def estimate_edge_position(data: dict, options={}, index=0): return estimated_edge_shift def determine_edge_position(data: dict, options={}): + ''' Determines the edge position by 1) first differential maximum and/or 2) second differential zero-point. Calculates differential and/or double differential by periods''' - required_options = ['save_values', 'log', 'logfile', 'save_plots', 'save_folder', 'periods', 'diff', 'double_diff', 'fit_region'] + required_options = ['save_values', 'log', 'logfile', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'fit_region'] default_options = { 'save_values': True, 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_determine_edge_position.log', + 'show_plots': False, 'save_plots': False, 'save_folder': './', - 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly, 'diff': True, + 'diff.polyorder': 2, + 'diff.periods': 2, #Periods needs to be an even number for the shifting of values to work properly, 'double_diff': False, - 'fit_region': 0.0005 + 'double_diff.polyorder': 2, + 'double_diff.periods': 2, #Periods needs to be an even number for the shifting of values to work properly, + 'fit_region': None # The length of the region to find points to fit to a function } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - if options['periods'] % 2 == 1: + if options['diff'] and options['diff.periods'] % 2 != 0: + raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") + if options['double_diff'] and options['double_diff.periods'] % 2 != 0: raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") @@ -478,18 +485,21 @@ def determine_edge_position(data: dict, options={}): for i, filename in enumerate(data['path']): estimated_edge_pos = estimate_edge_position(data, options=options, index=i) - - #========================== fitting first differential ========== + if not options['fit_region']: + options['fit_region'] = (5)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]) + + + #========================== Fitting first differential ========== if options['diff']: df_diff[filename] = data['xanes_data'][filename].diff(periods=options['periods']) df_diff[filename]=df_diff[filename].shift(-int(options['periods']/2)) - df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] + df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] <= estimated_edge_pos+options['fit_region']) & ((df_diff["ZapEnergy"] >= estimated_edge_pos-options['fit_region']))] # Fitting a function to the chosen interval - params = np.polyfit(df_diff_edge["ZapEnergy"], df_diff_edge[filename], 2) + params = np.polyfit(df_diff_edge["ZapEnergy"], df_diff_edge[filename], options['diff.polyorder']) diff_function = np.poly1d(params) x_diff=np.linspace(df_diff_edge["ZapEnergy"].iloc[0],df_diff_edge["ZapEnergy"].iloc[-1],num=10000) @@ -503,7 +513,7 @@ def determine_edge_position(data: dict, options={}): edge_pos_diff=x_diff[np.where(y_diff == np.amax(y_diff))][0] if options['log']: - aux.write_log(message=f"Edge position estimated by the differential maximum is: {str(round(edge_pos_diff,5))}", options=options) + aux.write_log(message=f"Edge position estimated by the differential maximum is: {str(round(edge_pos_diff,5))} keV", options=options) if options['save_values']: data['e0'][filename] = edge_pos_diff @@ -517,7 +527,7 @@ def determine_edge_position(data: dict, options={}): df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] # Fitting a function to the chosen interval - params = np.polyfit(df_double_diff_edge["ZapEnergy"], df_double_diff_edge[filename], 2) + params = np.polyfit(df_double_diff_edge["ZapEnergy"], df_double_diff_edge[filename], options['double_diff.polyorder']) double_diff_function = np.poly1d(params) x_double_diff=np.linspace(df_double_diff_edge["ZapEnergy"].iloc[0], df_double_diff_edge["ZapEnergy"].iloc[-1],num=10000) @@ -532,67 +542,98 @@ def determine_edge_position(data: dict, options={}): edge_pos_double_diff=x_double_diff[np.where(y_double_diff == find_nearest(y_double_diff,0))][0] if options['log']: - aux.write_log(message=f"Edge shift estimated by the double differential zero-point is {str(round(edge_pos_double_diff,5))}", options=options) + aux.write_log(message=f"Edge position estimated by the double differential zero-point is {str(round(edge_pos_double_diff,5))} keV", options=options) - if options['save_plots']: + if options['diff']: + aux.write_log(message=f"Difference between edge position estimated from differential maximum and double differential zero-point is {(edge_pos_diff-edge_pos_double_diff)*1000} eV.") + + if options['save_plots'] or options['show_plots']: if options['diff'] and options['double_diff']: - - fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(ncols=2, nrows=2, figsize=(20,20)) - df_diff.plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter') - df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax1) - ax1.set_xlim([edge_pos_diff-0.0015, edge_pos_diff+0.0015]) - ax1.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') + + fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(ncols=3, nrows=2, figsize=(20,20)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') ax1.axvline(x=edge_pos_diff, ls='--', c='green') - ax1.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') - ax1.set_title('Fit region of differentiated data') - - df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + + df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.set_xlim([edge_pos_diff-0.0015, edge_pos_diff+0.0015]) + ax2.axvline(x=estimated_edge_pos-options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_diff, ls='--', c='green') - ax2.axvline(x=estimated_edge_pos, ls='--', c='red') - ax2.set_title('Fit of differentiated data') + ax2.axvline(x=estimated_edge_pos+options['fit_region'], ls='--', c='black') + ax2.set_title('Fit region of differentiated data') + + df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.axvline(x=edge_pos_diff, ls='--', c='green') + ax3.axvline(x=estimated_edge_pos, ls='--', c='red') + ax3.set_title('Fit of differentiated data') - df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter') - df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) - ax3.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015]) - ax3.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') - ax3.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax3.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') - - df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax4, kind='scatter') - df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax4) + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax4, c='black') ax4.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax4.axvline(x=estimated_edge_pos, ls='--', c='red') + df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax5, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax5) + ax5.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015]) + ax5.axvline(x=estimated_edge_pos-options['fit_region'], ls='--', c='black') + ax5.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax5.axvline(x=estimated_edge_pos+options['fit_region'], ls='--', c='black') + + df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax6, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax6) + ax6.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax6.axvline(x=estimated_edge_pos, ls='--', c='red') elif options['diff']: - fig, (ax1, ax2) = plt.subplots(ncols=2,nrows=1, figsize=(20, 10)) - df_diff.plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter') - ax1.set_xlim([edge_pos_diff-0.5, edge_pos_diff+0.5]) - ax1.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') + fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) + + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') ax1.axvline(x=edge_pos_diff, ls='--', c='green') - ax1.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') - df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax2) + df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.set_xlim([edge_pos_diff-0.5, edge_pos_diff+0.5]) + ax2.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_diff, ls='--', c='green') - ax2.axvline(x=estimated_edge_pos, ls='--', c='red') + ax2.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') + + df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.axvline(x=edge_pos_diff, ls='--', c='green') + ax3.axvline(x=estimated_edge_pos, ls='--', c='red') elif options['double_diff']: - fig, (ax1, ax2) = plt.subplots(ncols=2,nrows=1, figsize=(20, 10)) - df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter') - ax1.set_xlim([edge_pos_double_diff-0.5, edge_pos_double_diff+0.5]) - ax1.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') + fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) + + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') ax1.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax1.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') - df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax2) + df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.set_xlim([edge_pos_double_diff-0.5, edge_pos_double_diff+0.5]) + ax2.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') ax2.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax2.axvline(x=estimated_edge_pos, ls='--', c='red') + ax2.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') + + df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax3.axvline(x=estimated_edge_pos, ls='--', c='red') + + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_edge_position.png' + + plt.savefig(dst, transparent=False) + + if not options['show_plots']: + plt.close() if not options['diff']: From 1e147854a7173ca0d4191f99b64cc5d46726aa06 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 27 Jun 2022 16:43:42 +0200 Subject: [PATCH 197/355] Update documentation for determination of edge position --- nafuma/xanes/calib.py | 76 ++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 66c4229..d6fecc2 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -444,44 +444,56 @@ def estimate_edge_position(data: dict, options={}, index=0): return estimated_edge_shift def determine_edge_position(data: dict, options={}): - ''' Determines the edge position by 1) first differential maximum and/or 2) second differential zero-point. Calculates differential and/or double differential by periods''' + ''' Determines the edge position by 1) first differential maximum and/or 2) second differential zero-point. Calculates differential and/or double differential by diff.periods and double_diff.periods respectively. + The differentiated and/or doubly differentiated data is fitted to a polynomial of diff.polyorder and/or double_diff.polyorder around the estimated edge position. The estimated edge position is set to be the x-value of the data + point at maximum of the differentiated data. The region to be fitted to the polynomial is determined by fit_region, which defaults to 5 times the distance between two data points, giving five data points to fit to. + + Allows plotting and saving of three plots to assess the quality of the fit, and also allows logging. + + Requires that XANES-data is already loaded in data['xanes_data']. This allows the user to choose when to determine the edge position - whether before or after normalisation, flattening etc.''' required_options = ['save_values', 'log', 'logfile', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'fit_region'] default_options = { - 'save_values': True, - 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_determine_edge_position.log', - 'show_plots': False, - 'save_plots': False, - 'save_folder': './', - 'diff': True, - 'diff.polyorder': 2, - 'diff.periods': 2, #Periods needs to be an even number for the shifting of values to work properly, - 'double_diff': False, - 'double_diff.polyorder': 2, - 'double_diff.periods': 2, #Periods needs to be an even number for the shifting of values to work properly, + 'save_values': True, # Whether the edge positions should be stored in a dictionary within the main data dictionary. + 'log': False, # Toggles logging on/off + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_determine_edge_position.log', # Sets the path to the logfile. Ignored if log == False + 'show_plots': False, # Toggles on/off whether plots should be shown. For sequential data, saving the plots and inspecting them there is probably better. + 'save_plots': False, # Toggles on/off whether plots should be saved. + 'save_folder': './', # Sets the path to where the plots should be saved. Creates folder if doesn't exist. Ignored if save_plots == False + 'diff': True, # Toggles calculation of the edge position based on differential data + 'diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the differential to + 'diff.periods': 2, # Sets the number of data points between which the first order difference should be calculated. Needs to be even for subsequent shifting of data to function. + 'double_diff': False, # Toggles calculation of the edge position based on double differential data + 'double_diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the double differential to + 'double_diff.periods': 2, # Sets the number of data points between which the second order difference should be calculated. Needs to be even for subsequent shifting of data to function. 'fit_region': None # The length of the region to find points to fit to a function - } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + # Check if periods are even if options['diff'] and options['diff.periods'] % 2 != 0: + if options['log']: + aux.write_log(message='Periods for differentiation is not even. Ending run.', options=options) raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") if options['double_diff'] and options['double_diff.periods'] % 2 != 0: + aux.write_log(message='Periods for double differentiation is not even. Ending run.', options=options) raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") - ##### - + + # Prepare dataframes for differential data if options['diff']: df_diff = pd.DataFrame(data['xanes_data']['ZapEnergy']) if options['double_diff']: df_double_diff = pd.DataFrame(data['xanes_data']['ZapEnergy']) if options['save_values']: - data['e0'] = {} + data['e0_diff'] = {} + data['e0_double_diff'] = {} + # Get rough estimate of edge position for i, filename in enumerate(data['path']): estimated_edge_pos = estimate_edge_position(data, options=options, index=i) @@ -489,12 +501,13 @@ def determine_edge_position(data: dict, options={}): options['fit_region'] = (5)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]) - #========================== Fitting first differential ========== + #========================== Fitting the first order derivative ========== if options['diff']: - df_diff[filename] = data['xanes_data'][filename].diff(periods=options['periods']) - df_diff[filename]=df_diff[filename].shift(-int(options['periods']/2)) + df_diff[filename] = data['xanes_data'][filename].diff(periods=options['diff.periods']) + df_diff[filename]=df_diff[filename].shift(-int(options['diff.periods']/2)) # Shifts the data back so that the difference between the points is located in the middle of the two points the caluclated difference is between + # Picks out the points to be fitted df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] <= estimated_edge_pos+options['fit_region']) & ((df_diff["ZapEnergy"] >= estimated_edge_pos-options['fit_region']))] @@ -516,12 +529,12 @@ def determine_edge_position(data: dict, options={}): aux.write_log(message=f"Edge position estimated by the differential maximum is: {str(round(edge_pos_diff,5))} keV", options=options) if options['save_values']: - data['e0'][filename] = edge_pos_diff - + data['e0_diff'][filename] = edge_pos_diff + #========================== Fitting the second order derivative ========== if options['double_diff']: - df_double_diff[filename] = data['xanes_data'][filename].diff(periods=options['periods']).diff(periods=options['periods']) - df_double_diff[filename]=df_double_diff[filename].shift(-int(options['periods'])) + df_double_diff[filename] = data['xanes_data'][filename].diff(periods=options['double_diff.periods']).diff(periods=options['double_diff.periods']) + df_double_diff[filename]=df_double_diff[filename].shift(-int(options['double_diff.periods'])) # Pick out region of interest df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] @@ -547,8 +560,15 @@ def determine_edge_position(data: dict, options={}): if options['diff']: aux.write_log(message=f"Difference between edge position estimated from differential maximum and double differential zero-point is {(edge_pos_diff-edge_pos_double_diff)*1000} eV.") + if options['save_values']: + data['e0_double_diff'][filename] = edge_pos_double_diff + + + # Make and show / save plots if options['save_plots'] or options['show_plots']: + + # If both are enabled if options['diff'] and options['double_diff']: fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(ncols=3, nrows=2, figsize=(20,20)) @@ -586,7 +606,7 @@ def determine_edge_position(data: dict, options={}): ax6.axvline(x=estimated_edge_pos, ls='--', c='red') - + # If only first order differentials is enabled elif options['diff']: fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) @@ -605,7 +625,7 @@ def determine_edge_position(data: dict, options={}): ax3.axvline(x=edge_pos_diff, ls='--', c='green') ax3.axvline(x=estimated_edge_pos, ls='--', c='red') - + # If only second order differentials is enabled elif options['double_diff']: fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) @@ -624,6 +644,8 @@ def determine_edge_position(data: dict, options={}): ax3.axvline(x=edge_pos_double_diff, ls='--', c='green') ax3.axvline(x=estimated_edge_pos, ls='--', c='red') + + # Save plots if toggled if options['save_plots']: if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) @@ -632,6 +654,8 @@ def determine_edge_position(data: dict, options={}): plt.savefig(dst, transparent=False) + + # Close plots if show_plots not toggled if not options['show_plots']: plt.close() From cc3c4dc5b65206b2b36323b26f96d3560e383568 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 27 Jun 2022 20:46:01 +0200 Subject: [PATCH 198/355] Add interactive to smoothing + documentation updates --- nafuma/xanes/calib.py | 145 +++++++++++++++++++++++++++++------------- 1 file changed, 101 insertions(+), 44 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index d6fecc2..9bb13f4 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -48,8 +48,10 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: required_options = ['pre_edge_limit', 'masks', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive'] default_options = { - 'pre_edge_limit': [None, None], - 'masks': [], + 'pre_edge_limits': [None, None], + 'pre_edge_masks': [], + 'pre_edge_polyorder': 1, + 'pre_edge_save_data': False, 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', 'show_plots': False, @@ -67,17 +69,17 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Implement with finding accurate edge position # FIXME Allow specification of start of pre-edge area # Find the cutoff point at which the edge starts - everything to the LEFT of this point will be used in the pre edge function fit - if not options['pre_edge_limit'][0]: - options['pre_edge_limit'][0] = data['xanes_data_original']['ZapEnergy'].min() + if not options['pre_edge_limits'][0]: + options['pre_edge_limits'][0] = data['xanes_data_original']['ZapEnergy'].min() - if not options['pre_edge_limit'][1]: + if not options['pre_edge_limits'][1]: pre_edge_limit_offset = 0.03 data['edge'] = find_element(data) edge_position = estimate_edge_position(data, options, index=0) - options['pre_edge_limit'][1] = edge_position - pre_edge_limit_offset + options['pre_edge_limits'][1] = edge_position - pre_edge_limit_offset # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session if options['interactive']: @@ -92,9 +94,9 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME There should be an option to specify the interval in which to fit the background - now it is taking everything to the left of edge_start parameter, but if there are some artifacts in this area, it should be possible to # limit the interval # Making a dataframe only containing the rows that are included in the background subtraction (points lower than where the edge start is defined) - pre_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['pre_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['pre_edge_limit'][1])].copy() + pre_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['pre_edge_limits'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['pre_edge_limits'][1])].copy() - for mask in options['masks']: + for mask in options['pre_edge_masks']: pre_edge_data.loc[(pre_edge_data['ZapEnergy'] > mask[0]) & (pre_edge_data['ZapEnergy'] < mask[1])] = np.nan pre_edge_data = pre_edge_data.dropna() @@ -106,10 +108,10 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: for i, filename in enumerate(data['path']): if options['log']: - aux.write_log(message=f'Fitting background on {os.path.basename(filename)} ({i+1} / {len(data["path"])})', options=options) + aux.write_log(message=f'Fitting background on {os.path.basename(filename)} ({i+1}/{len(data["path"])})', options=options) #Fitting linear function to the background - params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],1) + params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],options['pre_edge_polyorder']) fit_function = np.poly1d(params) data['pre_edge_params'][filename] = params @@ -133,7 +135,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: if options['ylim'][1]: ax1.set_ylim(top=options['ylim'][1]) - for mask in options['masks']: + for mask in options['pre_edge_masks']: ax1.fill_between(x=mask, y1=0, y2=data['xanes_data_original'][filename].max()*2, alpha=0.2, color='black') data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax2) @@ -155,7 +157,10 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: if options['log']: - aux.write_log(message=f'Pre edge fitting done.', options=options) + aux.write_log(message=f'Pre edge fitting done.', options=options) + + if options['pre_edge_save_data']: + data['pre_edge_fit_data'] = pre_edge_fit_data return pre_edge_fit_data @@ -166,7 +171,7 @@ def pre_edge_fit_interactive(data: dict, options: dict) -> None: w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(pre_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), - pre_edge_limit=widgets.FloatRangeSlider(value=[options['pre_edge_limit'][0], options['pre_edge_limit'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) + pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) ) options['widget'] = w @@ -219,33 +224,40 @@ def pre_edge_subtraction(data: dict, options={}): def post_edge_fit(data: dict, options={}): - #FIXME should be called "fitting post edge" (normalization is not done here, need edge shift position) - required_options = ['log', 'logfile', 'masks', 'post_edge_limit', 'polyorder', 'interactive', 'show_plots', 'save_plots', 'save_folder'] + ''' Fit the post edge within the post_edge.limits to a polynomial of post_edge.polyorder order. Allows interactive plotting, as well as showing static plots and saving plots to drive. + + Requires data to have already been read to data['xanes_data_original'] + ''' + + + required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'interactive', 'show_plots', 'save_plots', 'save_folder'] default_options = { - 'log': False, + 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', - 'post_edge_limit': [None, None], - 'masks': [], - 'polyorder': 2, + 'post_edge_limits': [None, None], + 'post_edge_masks': [], + 'post_edge_polyorder': 2, + 'post_edge_save_data': False, 'interactive': False, 'show_plots': False, 'save_plots': False, 'save_folder': './', + 'ylim': [None, None] } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - if not options['post_edge_limit'][0]: + if not options['post_edge_limits'][0]: post_edge_limit_offset = 0.03 data['edge'] = find_element(data) edge_position = estimate_edge_position(data, options, index=0) - options['post_edge_limit'][0] = edge_position + post_edge_limit_offset + options['post_edge_limits'][0] = edge_position + post_edge_limit_offset - if not options['post_edge_limit'][1]: - options['post_edge_limit'][1] = data['xanes_data_original']['ZapEnergy'].max() + if not options['post_edge_limits'][1]: + options['post_edge_limits'][1] = data['xanes_data_original']['ZapEnergy'].max() # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session if options['interactive']: @@ -257,9 +269,9 @@ def post_edge_fit(data: dict, options={}): - post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_limit'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_limit'][1])].copy() + post_edge_data = data['xanes_data_original'].loc[(data['xanes_data_original']["ZapEnergy"] > options['post_edge_limits'][0]) & (data['xanes_data_original']["ZapEnergy"] < options['post_edge_limits'][1])].copy() - for mask in options['masks']: + for mask in options['post_edge_masks']: post_edge_data.loc[(post_edge_data['ZapEnergy'] > mask[0]) & (post_edge_data['ZapEnergy'] < mask[1])] = np.nan post_edge_data = post_edge_data.dropna() #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit @@ -298,7 +310,7 @@ def post_edge_fit(data: dict, options={}): ax1.axvline(x = min(post_edge_data['ZapEnergy']), ls='--') ax1.set_title(f'{os.path.basename(filename)} - Full view', size=20) - for mask in options['masks']: + for mask in options['post_edge_masks']: ax1.fill_between(x=mask, y1=0, y2=data['xanes_data_original'][filename].max()*2, alpha=0.2, color='black') if options['ylim'][0] != None: @@ -325,14 +337,22 @@ def post_edge_fit(data: dict, options={}): plt.close() + if options['log']: + aux.write_log(message='Post edge fitting done!', options=options) + + if options['post_edge_save_data']: + data['post_edge_fit_data'] = post_edge_fit_data + + return post_edge_fit_data def post_edge_fit_interactive(data: dict, options: dict) -> None: + ''' Defines the widgets to use with the ipywidgets interactive mode and calls the update function found in btp.ipywidgets. ''' w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(post_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), - post_edge_limit=widgets.FloatRangeSlider(value=[options['post_edge_limit'][0], options['post_edge_limit'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) + post_edge_limit=widgets.FloatRangeSlider(value=[options['post_edge.limits'][0], options['post_edge.limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) ) options['widget'] = w @@ -344,43 +364,59 @@ def smoothing(data: dict, options={}): # FIXME Add logging # FIXME Add saving of files - required_options = ['log', 'logfile', 'window_length','polyorder', 'save_default'] + required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'smooth_window_length', 'smooth_algorithm', 'smooth_polyorder', 'smooth_save_default'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', 'save_plots': False, 'save_folder': './', - 'window_length': 3, - 'polyorder': 2, - 'save_default': False + 'smooth_window_length': 3, + 'smooth_polyorder': 2, + 'smooth_algorithm': 'savgol', # At the present, only Savitzky-Golay filter is implemented. Add Gaussian and Boxcar later. + 'smooth_save_default': False, } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) df_smooth = pd.DataFrame(data['xanes_data']['ZapEnergy']) - if options['save_default']: + if options['smooth_save_default']: df_smooth_default = pd.DataFrame(data['xanes_data']['ZapEnergy']) + if options['log']: + aux.write_log(message='Starting smoothing.') + + + if options['interactive']: + options['interactive'] = False + options['interactive_session_active'] = True + options['show_plots'] = True + smoothing_interactive(data=data, options=options) + return + + # FIXME Add other types of filters # FIXME Instead of assigning values directly to the data dictionary, these should be made into an own DataFrame that you can decide later what to do with - these variables should # then be returned - for filename in data['path']: - df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['window_length'], options['polyorder'])) + for i, filename in enumerate(data['path']): + + if options['smooth_algorithm'] == 'savgol': + if options['log']: + aux.write_log(message=f'Smoothing {filename} with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options) + df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['smooth_window_length'], options['smooth_polyorder'])) - if options['save_default']: - df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['window_length'], default_options['polyorder'])) + if options['smooth_save_default']: + if options['smooth.algorithm'] == 'savgol': + if options['log']: + aux.write_log(message=f'Smoothing {filename} using default parameters with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options) + df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['smooth_window_length'], default_options['smooth_polyorder'])) - if options['save_plots']: - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) - - dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_smooth.png' + if options['save_plots'] or options['show_plots']: edge_pos = estimate_edge_position(data=data, options=options) intensity_midpoint = df_smooth[filename].iloc[np.where(df_smooth['ZapEnergy'] == find_nearest(df_smooth['ZapEnergy'], edge_pos))].values[0] - if options['save_default']: + if options['smooth_save_default']: fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5)) data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter') df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax1) @@ -399,9 +435,15 @@ def smoothing(data: dict, options={}): ax.set_title(f'{os.path.basename(filename)} - Smooth', size=20) + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - plt.savefig(dst, transparent=False) - plt.close() + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_smooth.png' + plt.savefig(dst, transparent=False) + + if not options['show_plots']: + plt.close() if not options['save_default']: df_smooth_default = None @@ -410,6 +452,21 @@ def smoothing(data: dict, options={}): +def smoothing_interactive(data: dict, options: dict) -> None: + ''' Defines the widgets to use with the ipywidgets interactive mode and calls the update function found in btp.ipywidgets. ''' + + w = widgets.interactive( + btp.ipywidgets_update, func=widgets.fixed(smoothing), data=widgets.fixed(data), options=widgets.fixed(options), + smooth_window_length=widgets.IntSlider(value=options['smooth_window_length'], min=1, max=20, step=1), + smooth_polyorder=widgets.IntSlider(value=options['smooth_polyorder'], min=1, max=5, step=1), + ) + + + options['widget'] = w + + display(w) + + def find_nearest(array, value): #function to find the value closes to "value" in an "array" array = np.asarray(array) From c522b73ca4170e639487564fd51daf98acf09b65 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 28 Jun 2022 12:10:46 +0200 Subject: [PATCH 199/355] Add K-edges from ITC Vol C --- nafuma/xanes/__init__.py | 2 +- nafuma/xanes/edges.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 nafuma/xanes/edges.py diff --git a/nafuma/xanes/__init__.py b/nafuma/xanes/__init__.py index b11c1f3..a3834e8 100644 --- a/nafuma/xanes/__init__.py +++ b/nafuma/xanes/__init__.py @@ -1 +1 @@ -from . import io, calib \ No newline at end of file +from . import io, calib, edges \ No newline at end of file diff --git a/nafuma/xanes/edges.py b/nafuma/xanes/edges.py new file mode 100644 index 0000000..dc02601 --- /dev/null +++ b/nafuma/xanes/edges.py @@ -0,0 +1,27 @@ +import pandas as pd +import numpy as np +from scipy.constants import c, h + +# From 2019 redefinition of SI base units: https://en.wikipedia.org/wiki/2019_redefinition_of_the_SI_base_units +keV_per_J = (1 / 1.602176634e-19) / 1000 + +# kXu values taken from International Tables for Crystallography Volume , Kulwer Academic Publishers - Dordrect / Boston / London (1992) +k_edge = { 'Z': [ 1, 2, + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48], + 'Atom': [ 'H', 'He', + 'Li', 'Be', 'B', 'C', 'N', 'O', 'F', 'Ne', + 'Na', 'Mg', 'Al', 'Si', 'P', 'S', 'Cl', 'Ar', + 'K', 'Ca', 'Sc', 'Ti', 'V', 'Cr', 'Mn', 'Fe', 'Co', 'Ni', 'Cu', 'Zn', 'Ga', 'Ge', 'As', 'Se', 'Br', 'Kr', + 'Rb', 'Sr', 'Y', 'Zr', 'Nb', 'Mo', 'Tc', 'Ru', 'Rh', 'Pd', 'Ag', 'Cd'], + 'kXu': [ np.nan, np.nan, + 226.5, np.nan, np.nan, 43.68, 30.99, 23.32, np.nan, np.nan, + np.nan, 9.5117, 7.9511, 6.7446, 5.7866, 5.0182, 4.3969, 3.8707, + 3.43645, 3.07016, 2.7573, 2.49730, 2.26902, 2.07012, 1.89636, 1.74334, 1.60811, 1.48802, 1.38043, 1.2833, 1.19567, 1.11652, 1.04497, 0.97978, 0.91995, 0.86547, + 0.81549, 0.76969, 0.72762, 0.68877, 0.65291, 0.61977, 0.5891, 0.56047, 0.53378, 0.50915, 0.48582, 0.46409]} + + +k_edge = pd.DataFrame(k_edge) +k_edge['keV'] = np.round(h*c/(k_edge['kXu']*10**-10) * keV_per_J, 3) \ No newline at end of file From 6bbd6776b896bea21f4f6bfa846a6aecee7c0e59 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 29 Jun 2022 15:26:43 +0200 Subject: [PATCH 200/355] Tweaks based on workflow testing --- nafuma/auxillary.py | 14 +- nafuma/xanes/calib.py | 334 +++++++++++++++++++++++++----------------- 2 files changed, 214 insertions(+), 134 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 2b87479..0ccde1f 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -12,11 +12,21 @@ def update_options(options, required_options, default_options): return options -def save_options(options, path): +def save_options(options, path, ignore=None): ''' Saves any options dictionary to a JSON-file in the specified path''' + options_copy = options.copy() + + if ignore: + if not isinstance(ignore, list): + ignore = [ignore] + + for i in ignore: + options_copy[i] = 'Removed' + + with open(path, 'w') as f: - json.dump(options,f) + json.dump(options_copy,f, skipkeys=True, indent=4) def load_options(path): diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 9bb13f4..22d63eb 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -46,12 +46,12 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Add log-file - required_options = ['pre_edge_limit', 'masks', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive'] + required_options = ['pre_edge_limits', 'pre_edge_masks', 'pre_edge_polyorder', 'pre_edge_store_data', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive'] default_options = { 'pre_edge_limits': [None, None], 'pre_edge_masks': [], 'pre_edge_polyorder': 1, - 'pre_edge_save_data': False, + 'pre_edge_store_data': False, 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_fit.log', 'show_plots': False, @@ -159,7 +159,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: if options['log']: aux.write_log(message=f'Pre edge fitting done.', options=options) - if options['pre_edge_save_data']: + if options['pre_edge_store_data']: data['pre_edge_fit_data'] = pre_edge_fit_data return pre_edge_fit_data @@ -171,7 +171,8 @@ def pre_edge_fit_interactive(data: dict, options: dict) -> None: w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(pre_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), - pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) + pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001), + pre_edge_store_data=widgets.Checkbox(value=options['pre_edge_store_data']) ) options['widget'] = w @@ -183,12 +184,14 @@ def pre_edge_fit_interactive(data: dict, options: dict) -> None: def pre_edge_subtraction(data: dict, options={}): - required_options = ['log', 'logfile', 'save_plots', 'save_folder'] + required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'pre_edge_subtraction_store_data'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_pre_edge_subtraction.log', + 'show_plots': False, 'save_plots': False, - 'save_folder': './' + 'save_folder': './', + 'pre_edge_subtraction_store_data': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -203,19 +206,28 @@ def pre_edge_subtraction(data: dict, options={}): xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data_original'][filename] - data['pre_edge_fit_data'][filename]) - if options['save_plots']: - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) + if options['save_plots'] or options['show_plots']: - dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_subtraction.png' fig, ax = plt.subplots(figsize=(10,5)) - data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax) - xanes_data_bkgd_subtracted.plot(x='ZapEnergy', y=filename, color='red', ax=ax) + data['xanes_data_original'].plot(x='ZapEnergy', y=filename, color='black', ax=ax, label='Original data') + xanes_data_bkgd_subtracted.plot(x='ZapEnergy', y=filename, color='red', ax=ax, label='Pre edge subtracted') ax.set_title(f'{os.path.basename(filename)} - After subtraction', size=20) - plt.savefig(dst) - plt.close() + + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_subtraction.png' + + plt.savefig(dst) + + if not options['show_plots']: + plt.close() + + if options['pre_edge_subtraction_store_data']: + data['xanes_data'] = xanes_data_bkgd_subtracted return xanes_data_bkgd_subtracted @@ -230,14 +242,14 @@ def post_edge_fit(data: dict, options={}): ''' - required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'interactive', 'show_plots', 'save_plots', 'save_folder'] + required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'post_edge_store_data', 'interactive', 'show_plots', 'save_plots', 'save_folder'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', 'post_edge_limits': [None, None], 'post_edge_masks': [], 'post_edge_polyorder': 2, - 'post_edge_save_data': False, + 'post_edge_store_data': False, 'interactive': False, 'show_plots': False, 'save_plots': False, @@ -283,10 +295,10 @@ def post_edge_fit(data: dict, options={}): for i, filename in enumerate(data['path']): if options['log']: - aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["polyorder"]}', options=options) + aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["post_edge_polyorder"]}', options=options) #Fitting linear function to the background - params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], options['polyorder']) + params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], options['post_edge_polyorder']) fit_function = np.poly1d(params) if options['log']: @@ -340,7 +352,7 @@ def post_edge_fit(data: dict, options={}): if options['log']: aux.write_log(message='Post edge fitting done!', options=options) - if options['post_edge_save_data']: + if options['post_edge_store_data']: data['post_edge_fit_data'] = post_edge_fit_data @@ -352,7 +364,8 @@ def post_edge_fit_interactive(data: dict, options: dict) -> None: w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(post_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), - post_edge_limit=widgets.FloatRangeSlider(value=[options['post_edge.limits'][0], options['post_edge.limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001) + post_edge_limits=widgets.FloatRangeSlider(value=[options['post_edge_limits'][0], options['post_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001), + post_edge_store_data=widgets.Checkbox(value=options['post_edge_store_data']) ) options['widget'] = w @@ -364,16 +377,19 @@ def smoothing(data: dict, options={}): # FIXME Add logging # FIXME Add saving of files - required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'smooth_window_length', 'smooth_algorithm', 'smooth_polyorder', 'smooth_save_default'] + required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'interactive', 'smooth_window_length', 'smooth_algorithm', 'smooth_polyorder', 'smooth_save_default', 'smooth_store_data'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', + 'show_plots': False, 'save_plots': False, 'save_folder': './', + 'interactive': False, 'smooth_window_length': 3, 'smooth_polyorder': 2, 'smooth_algorithm': 'savgol', # At the present, only Savitzky-Golay filter is implemented. Add Gaussian and Boxcar later. 'smooth_save_default': False, + 'smooth_store_data': False, } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -387,6 +403,7 @@ def smoothing(data: dict, options={}): if options['interactive']: + data['xanes_data_backup'] = data['xanes_data'] options['interactive'] = False options['interactive_session_active'] = True options['show_plots'] = True @@ -405,7 +422,7 @@ def smoothing(data: dict, options={}): df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['smooth_window_length'], options['smooth_polyorder'])) if options['smooth_save_default']: - if options['smooth.algorithm'] == 'savgol': + if options['smooth_algorithm'] == 'savgol': if options['log']: aux.write_log(message=f'Smoothing {filename} using default parameters with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options) df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['smooth_window_length'], default_options['smooth_polyorder'])) @@ -413,27 +430,36 @@ def smoothing(data: dict, options={}): if options['save_plots'] or options['show_plots']: + + edge_pos = estimate_edge_position(data=data, options=options) intensity_midpoint = df_smooth[filename].iloc[np.where(df_smooth['ZapEnergy'] == find_nearest(df_smooth['ZapEnergy'], edge_pos))].values[0] + step_length = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0] + + if options['smooth_save_default']: fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5)) - data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter') - df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax1) + data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter') + df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax1) ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) - data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter') - df_smooth_default.loc[(df_smooth_default['ZapEnergy'] > edge_pos-0.0015) & (df_smooth_default['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) + data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter') + df_smooth_default.loc[(df_smooth_default['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth_default['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20) - elif not options['save_default']: - fig, ax = plt.subplots(figsize=(20,10)) - data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-0.0015) & (data['xanes_data']['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='black', ax=ax, kind='scatter') - df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-0.0015) & (df_smooth['ZapEnergy'] < edge_pos+0.0015)].plot(x='ZapEnergy', y=filename, color='red', ax=ax) - ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015]) - ax.set_ylim([intensity_midpoint*0.9, intensity_midpoint*1.1]) + elif not options['smooth_save_default']: + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter', c='black') + df_smooth.plot(x='ZapEnergy', y=filename, ax=ax1, c='red') + + data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter') + df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) + #ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015]) + #ax.set_ylim([intensity_midpoint*0.9, intensity_midpoint*1.1]) - ax.set_title(f'{os.path.basename(filename)} - Smooth', size=20) + ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) + ax2.set_title(f'{os.path.basename(filename)} - Smooth Edge Region', size=20) if options['save_plots']: if not os.path.isdir(options['save_folder']): @@ -445,8 +471,12 @@ def smoothing(data: dict, options={}): if not options['show_plots']: plt.close() - if not options['save_default']: + if not options['smooth_save_default']: df_smooth_default = None + + if options['smooth_store_data']: + data['xanes_data'] = df_smooth + options['smooth_store_data'] = False return df_smooth, df_smooth_default @@ -457,16 +487,21 @@ def smoothing_interactive(data: dict, options: dict) -> None: w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(smoothing), data=widgets.fixed(data), options=widgets.fixed(options), - smooth_window_length=widgets.IntSlider(value=options['smooth_window_length'], min=1, max=20, step=1), + smooth_window_length=widgets.IntSlider(value=options['smooth_window_length'], min=3, max=21, step=2), smooth_polyorder=widgets.IntSlider(value=options['smooth_polyorder'], min=1, max=5, step=1), + smooth_store_data=widgets.Checkbox(value=options['smooth_store_data']) ) - options['widget'] = w display(w) +def restore_from_backup(data): + if 'xanes_data_bakcup' in data.keys(): + data['xanes_data'] = data['xanes_data_backup'] + + def find_nearest(array, value): #function to find the value closes to "value" in an "array" array = np.asarray(array) @@ -480,7 +515,7 @@ def estimate_edge_position(data: dict, options={}, index=0): default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', - 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly + 'periods': 6, #Periods needs to be an even number for the shifting of values to work properly } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -509,7 +544,7 @@ def determine_edge_position(data: dict, options={}): Requires that XANES-data is already loaded in data['xanes_data']. This allows the user to choose when to determine the edge position - whether before or after normalisation, flattening etc.''' - required_options = ['save_values', 'log', 'logfile', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'fit_region'] + required_options = ['save_values', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'points_around_edge'] default_options = { 'save_values': True, # Whether the edge positions should be stored in a dictionary within the main data dictionary. 'log': False, # Toggles logging on/off @@ -521,9 +556,9 @@ def determine_edge_position(data: dict, options={}): 'diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the differential to 'diff.periods': 2, # Sets the number of data points between which the first order difference should be calculated. Needs to be even for subsequent shifting of data to function. 'double_diff': False, # Toggles calculation of the edge position based on double differential data - 'double_diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the double differential to + 'double_diff.polyorder': 1, # Sets the order of the polynomial to fit edge region of the double differential to 'double_diff.periods': 2, # Sets the number of data points between which the second order difference should be calculated. Needs to be even for subsequent shifting of data to function. - 'fit_region': None # The length of the region to find points to fit to a function + 'points_around_edge': 5 # The length of the region to find points to fit to a function } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -538,6 +573,16 @@ def determine_edge_position(data: dict, options={}): aux.write_log(message='Periods for double differentiation is not even. Ending run.', options=options) raise Exception("NB! Periods needs to be an even number for the shifting of values to work properly") + + if options['interactive']: + data['xanes_data_backup'] = data['xanes_data'] + options['interactive'] = False + options['interactive_session_active'] = True + options['show_plots'] = True + determine_edge_position_interactive(data=data, options=options) + return + + # Prepare dataframes for differential data @@ -554,8 +599,8 @@ def determine_edge_position(data: dict, options={}): for i, filename in enumerate(data['path']): estimated_edge_pos = estimate_edge_position(data, options=options, index=i) - if not options['fit_region']: - options['fit_region'] = (5)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]) + + fit_region = (options['points_around_edge']+1)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]) #========================== Fitting the first order derivative ========== @@ -565,7 +610,7 @@ def determine_edge_position(data: dict, options={}): df_diff[filename]=df_diff[filename].shift(-int(options['diff.periods']/2)) # Shifts the data back so that the difference between the points is located in the middle of the two points the caluclated difference is between # Picks out the points to be fitted - df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] <= estimated_edge_pos+options['fit_region']) & ((df_diff["ZapEnergy"] >= estimated_edge_pos-options['fit_region']))] + df_diff_edge = df_diff.loc[(df_diff["ZapEnergy"] <= estimated_edge_pos+fit_region) & ((df_diff["ZapEnergy"] >= estimated_edge_pos-fit_region))] # Fitting a function to the chosen interval @@ -594,7 +639,7 @@ def determine_edge_position(data: dict, options={}): df_double_diff[filename]=df_double_diff[filename].shift(-int(options['double_diff.periods'])) # Pick out region of interest - df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+options['fit_region']) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-options['fit_region']))] + df_double_diff_edge = df_double_diff.loc[(df_double_diff["ZapEnergy"] < estimated_edge_pos+fit_region) & ((df_double_diff["ZapEnergy"] > estimated_edge_pos-fit_region))] # Fitting a function to the chosen interval params = np.polyfit(df_double_diff_edge["ZapEnergy"], df_double_diff_edge[filename], options['double_diff.polyorder']) @@ -621,100 +666,100 @@ def determine_edge_position(data: dict, options={}): data['e0_double_diff'][filename] = edge_pos_double_diff - # Make and show / save plots - if options['save_plots'] or options['show_plots']: + # Make and show / save plots + if options['save_plots'] or options['show_plots']: - # If both are enabled - if options['diff'] and options['double_diff']: - - fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(ncols=3, nrows=2, figsize=(20,20)) - data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') - ax1.axvline(x=edge_pos_diff, ls='--', c='green') - - df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') - df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) - ax2.set_xlim([edge_pos_diff-0.0015, edge_pos_diff+0.0015]) - ax2.axvline(x=estimated_edge_pos-options['fit_region'], ls='--', c='black') - ax2.axvline(x=edge_pos_diff, ls='--', c='green') - ax2.axvline(x=estimated_edge_pos+options['fit_region'], ls='--', c='black') - ax2.set_title('Fit region of differentiated data') + # If both are enabled + if options['diff'] and options['double_diff']: - df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter') - df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) - ax3.axvline(x=edge_pos_diff, ls='--', c='green') - ax3.axvline(x=estimated_edge_pos, ls='--', c='red') - ax3.set_title('Fit of differentiated data') + fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(ncols=3, nrows=2, figsize=(20,20)) + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') + ax1.axvline(x=edge_pos_diff, ls='--', c='green') + + df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.set_xlim([edge_pos_diff-fit_region*1.5, edge_pos_diff+fit_region*1.5]) + ax2.axvline(x=estimated_edge_pos-fit_region, ls='--', c='black') + ax2.axvline(x=edge_pos_diff, ls='--', c='green') + ax2.axvline(x=estimated_edge_pos+fit_region, ls='--', c='black') + ax2.set_title('Fit region of differentiated data') + + df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.axvline(x=edge_pos_diff, ls='--', c='green') + ax3.axvline(x=estimated_edge_pos, ls='--', c='red') + ax3.set_title('Fit of differentiated data') - data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax4, c='black') - ax4.axvline(x=edge_pos_double_diff, ls='--', c='green') + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax4, c='black') + ax4.axvline(x=edge_pos_double_diff, ls='--', c='green') - df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax5, kind='scatter') - df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax5) - ax5.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015]) - ax5.axvline(x=estimated_edge_pos-options['fit_region'], ls='--', c='black') - ax5.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax5.axvline(x=estimated_edge_pos+options['fit_region'], ls='--', c='black') + df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax5, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax5) + ax5.set_xlim([edge_pos_double_diff-0.0015, edge_pos_double_diff+0.0015]) + ax5.axvline(x=estimated_edge_pos-fit_region, ls='--', c='black') + ax5.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax5.axvline(x=estimated_edge_pos+fit_region, ls='--', c='black') - df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax6, kind='scatter') - df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax6) - ax6.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax6.axvline(x=estimated_edge_pos, ls='--', c='red') - + df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax6, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax6) + ax6.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax6.axvline(x=estimated_edge_pos, ls='--', c='red') + - # If only first order differentials is enabled - elif options['diff']: - fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) - - data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') - ax1.axvline(x=edge_pos_diff, ls='--', c='green') + # If only first order differentials is enabled + elif options['diff']: + fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) + + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') + ax1.axvline(x=edge_pos_diff, ls='--', c='green') - df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') - df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) - ax2.set_xlim([edge_pos_diff-0.5, edge_pos_diff+0.5]) - ax2.axvline(x=edge_pos_diff-options['fit_region'], ls='--', c='black') - ax2.axvline(x=edge_pos_diff, ls='--', c='green') - ax2.axvline(x=edge_pos_diff+options['fit_region'], ls='--', c='black') + df_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.set_xlim([edge_pos_diff-fit_region*1.5, edge_pos_diff+fit_region*1.5]) + ax2.axvline(x=edge_pos_diff-fit_region, ls='--', c='black') + ax2.axvline(x=edge_pos_diff, ls='--', c='green') + ax2.axvline(x=edge_pos_diff+fit_region, ls='--', c='black') - df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) - df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) - ax3.axvline(x=edge_pos_diff, ls='--', c='green') - ax3.axvline(x=estimated_edge_pos, ls='--', c='red') + df_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) + df_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.axvline(x=edge_pos_diff, ls='--', c='green') + ax3.axvline(x=estimated_edge_pos, ls='--', c='red') - # If only second order differentials is enabled - elif options['double_diff']: - fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) - - data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') - ax1.axvline(x=edge_pos_double_diff, ls='--', c='green') + # If only second order differentials is enabled + elif options['double_diff']: + fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) + + data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') + ax1.axvline(x=edge_pos_double_diff, ls='--', c='green') - df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') - df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) - ax2.set_xlim([edge_pos_double_diff-0.5, edge_pos_double_diff+0.5]) - ax2.axvline(x=edge_pos_double_diff-options['fit_region'], ls='--', c='black') - ax2.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax2.axvline(x=edge_pos_double_diff+options['fit_region'], ls='--', c='black') + df_double_diff.plot(x='ZapEnergy', y=filename, ax=ax2, kind='scatter') + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax2) + ax2.set_xlim([edge_pos_double_diff-fit_region*1.5, edge_pos_double_diff+fit_region*1.5]) + ax2.axvline(x=edge_pos_double_diff-fit_region, ls='--', c='black') + ax2.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax2.axvline(x=edge_pos_double_diff+fit_region, ls='--', c='black') - df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) - df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) - ax3.axvline(x=edge_pos_double_diff, ls='--', c='green') - ax3.axvline(x=estimated_edge_pos, ls='--', c='red') + df_double_diff_edge.plot(x='ZapEnergy', y=filename, ax=ax3) + df_double_diff_fit_function.plot(x='x_diff', y='y_diff', ax=ax3) + ax3.axvline(x=edge_pos_double_diff, ls='--', c='green') + ax3.axvline(x=estimated_edge_pos, ls='--', c='red') - # Save plots if toggled - if options['save_plots']: - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) + # Save plots if toggled + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) - dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_edge_position.png' + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_edge_position.png' - plt.savefig(dst, transparent=False) + plt.savefig(dst, transparent=False) - # Close plots if show_plots not toggled - if not options['show_plots']: - plt.close() + # Close plots if show_plots not toggled + if not options['show_plots']: + plt.close() if not options['diff']: @@ -724,35 +769,59 @@ def determine_edge_position(data: dict, options={}): return edge_pos_diff, edge_pos_double_diff + + +def determine_edge_position_interactive(data: dict, options: dict) -> None: + ''' Defines the widgets to use with the ipywidgets interactive mode and calls the update function found in btp.ipywidgets. ''' + + + step_size = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0] + + w = widgets.interactive( + btp.ipywidgets_update, func=widgets.fixed(determine_edge_position), data=widgets.fixed(data), options=widgets.fixed(options), + points_around_edge=widgets.IntSlider(value=options['points_around_edge'], min=1, max=20, step=1), + ) + + options['widget'] = w + + display(w) + def normalise(data: dict, options={}): - required_options = ['log', 'logfile', 'save_values'] + required_options = ['log', 'logfile', 'normalisation_store_data'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log', - 'save_values': True + 'normalisation_store_data': False, } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) normalised_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) data['normalisation_constants'] = {} + if options['normalisation_store_data']: + pre_edge_fit_data_norm = pd.DataFrame(data['pre_edge_fit_data']['ZapEnergy']) + post_edge_fit_data_norm = pd.DataFrame(data['post_edge_fit_data']['ZapEnergy']) #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 for filename in data['path']: - e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0'][filename])].index.values[0] + e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0_diff'][filename])].index.values[0] #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant) + if options['normalisation_store_data']: + pre_edge_fit_data_norm.insert(1, filename, data['pre_edge_fit_data'][filename] / normalisation_constant) + post_edge_fit_data_norm.insert(1, filename, data['post_edge_fit_data'][filename] / normalisation_constant) + + + if options['normalisation_store_data']: + data['xanes_data'] = normalised_df # Normalise the pre-edge and post-edge fit function data - data['pre_edge_fit_data'][filename] = data['pre_edge_fit_data'][filename] / normalisation_constant - data['post_edge_fit_data'][filename] = data['post_edge_fit_data'][filename] / normalisation_constant + data['pre_edge_fit_data_norm'] = pre_edge_fit_data_norm + data['post_edge_fit_data_norm'] = post_edge_fit_data_norm data['normalisation_constants'][filename] = normalisation_constant - if options['save_values']: - data['xanes_data'] = normalised_df - return normalised_df @@ -760,11 +829,11 @@ def normalise(data: dict, options={}): def flatten(data:dict, options={}): #only picking out zapenergy-values higher than edge position (edge pos and below remains untouched) - required_options = ['log', 'logfile', 'save_values'] + required_options = ['log', 'logfile', 'flatten_store_data'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_flattening.log', - 'save_values': True + 'flatten_store_data': False, } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -772,13 +841,14 @@ def flatten(data:dict, options={}): flattened_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) for filename in data['path']: - fit_function_diff = -data['post_edge_fit_data'][filename] + data['pre_edge_params'][filename][0] - fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0'][filename]] = 0 + fit_function_diff = data['post_edge_fit_data_norm'][filename] - 1 + + fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0_diff'][filename]] = 0 flattened_df[filename] = data['xanes_data'][filename] - fit_function_diff - if options['save_values']: + if options['flatten_store_data']: data['xanes_data'] = flattened_df From faf41db41fcbad3720ddb4e804ef2ef2a8bb8bef Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 29 Jun 2022 15:26:51 +0200 Subject: [PATCH 201/355] Add tabulated K-edge values --- nafuma/xanes/edges.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nafuma/xanes/edges.py b/nafuma/xanes/edges.py index dc02601..abce1c6 100644 --- a/nafuma/xanes/edges.py +++ b/nafuma/xanes/edges.py @@ -6,7 +6,7 @@ from scipy.constants import c, h keV_per_J = (1 / 1.602176634e-19) / 1000 # kXu values taken from International Tables for Crystallography Volume , Kulwer Academic Publishers - Dordrect / Boston / London (1992) -k_edge = { 'Z': [ 1, 2, +K = { 'Z': [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, @@ -23,5 +23,8 @@ k_edge = { 'Z': [ 1, 2, 0.81549, 0.76969, 0.72762, 0.68877, 0.65291, 0.61977, 0.5891, 0.56047, 0.53378, 0.50915, 0.48582, 0.46409]} -k_edge = pd.DataFrame(k_edge) -k_edge['keV'] = np.round(h*c/(k_edge['kXu']*10**-10) * keV_per_J, 3) \ No newline at end of file +K = pd.DataFrame(K) +K['keV'] = np.round(h*c/(K['kXu']*10**-10) * keV_per_J, 3) + + +# FIXME If needed, add energies for L-edges as well. \ No newline at end of file From 254becff69cbf7bca4d4f5a0e93a01b10c8c29a1 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 29 Jun 2022 16:13:19 +0200 Subject: [PATCH 202/355] Make sure filenames is a list before reading --- nafuma/xanes/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 9676608..be7c9f3 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -183,12 +183,13 @@ def read_data(data: dict, options={}) -> pd.DataFrame: columns = ['ZapEnergy'] + if not isinstance(data['path'], list): + data['path'] = [data['path']] + # Initialise DataFrame with only ZapEnergy-column xanes_data = pd.read_csv(data['path'][0])[['ZapEnergy']] xanes_data['ZapEnergy'] += options['adjust'] - if not isinstance(data['path'], list): - data['path'] = [data['path']] for filename in data['path']: columns.append(filename) From b84cecaf84680e3e8cf3708efcba9cc6e842157a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 29 Jun 2022 16:40:36 +0200 Subject: [PATCH 203/355] Update documentation --- nafuma/xanes/calib.py | 118 ++++++++++++++++++++++++------------------ 1 file changed, 69 insertions(+), 49 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 22d63eb..71505f8 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -373,52 +373,53 @@ def post_edge_fit_interactive(data: dict, options: dict) -> None: display(w) def smoothing(data: dict, options={}): + ' Smoothes the data using the Savitzky-Golay filter. This is the only algorithm at this moment. ' - # FIXME Add logging - # FIXME Add saving of files required_options = ['log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'interactive', 'smooth_window_length', 'smooth_algorithm', 'smooth_polyorder', 'smooth_save_default', 'smooth_store_data'] default_options = { - 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', - 'show_plots': False, - 'save_plots': False, - 'save_folder': './', - 'interactive': False, - 'smooth_window_length': 3, - 'smooth_polyorder': 2, + 'log': False, # Toggles logging on / off + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_smoothing.log', # Sets path to log-file. Ignored if log == False + 'show_plots': False, # Toggles showing plots on / off. This is only recommended when working with a handful of scans. + 'save_plots': False, # Toggles saving plots on / off + 'save_folder': './', # Sets path to folder where plots should be saved. Ignored if save_plots == False + 'interactive': False, # Toggles interactive mode on / off. This is only recommended for a single scan to determine proper parameters for smoothing. + 'smooth_window_length': 3, # Determines the window length of smoothing that the savgol-filter uses for smoothing + 'smooth_polyorder': 2, # Determines the order of the polynomial used in the smoothing algorithm 'smooth_algorithm': 'savgol', # At the present, only Savitzky-Golay filter is implemented. Add Gaussian and Boxcar later. - 'smooth_save_default': False, - 'smooth_store_data': False, + 'smooth_save_default': False, # Toggles whether or not to run a separate smoothing using default values on / off + 'smooth_store_data': False, # Toggles storing data to data['xanes_data'] on / off } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + # Initialise new DataFrame with correct x-values df_smooth = pd.DataFrame(data['xanes_data']['ZapEnergy']) + # Do the same if smoothing with default values is toggled on if options['smooth_save_default']: df_smooth_default = pd.DataFrame(data['xanes_data']['ZapEnergy']) if options['log']: - aux.write_log(message='Starting smoothing.') + aux.write_log(message='Starting smoothing procedure.') + # Run in interactive mode if enabled if options['interactive']: - data['xanes_data_backup'] = data['xanes_data'] - options['interactive'] = False - options['interactive_session_active'] = True - options['show_plots'] = True - smoothing_interactive(data=data, options=options) + data['xanes_data_backup'] = data['xanes_data'] # Backup the data + options['interactive'] = False # Turn interactive mode off so that it is not called again within the interactive loop + options['show_plots'] = True # Force plotting on as interactive mode is useless without it + smoothing_interactive(data=data, options=options) # Call interactive version of the function return # FIXME Add other types of filters - # FIXME Instead of assigning values directly to the data dictionary, these should be made into an own DataFrame that you can decide later what to do with - these variables should - # then be returned for i, filename in enumerate(data['path']): if options['smooth_algorithm'] == 'savgol': if options['log']: aux.write_log(message=f'Smoothing {filename} with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options) + + # Apply savgol filter and add to DataFrame df_smooth.insert(1, filename, savgol_filter(data['xanes_data'][filename], options['smooth_window_length'], options['smooth_polyorder'])) if options['smooth_save_default']: @@ -428,16 +429,16 @@ def smoothing(data: dict, options={}): df_smooth_default.insert(1, filename, savgol_filter(data['xanes_data'][filename], default_options['smooth_window_length'], default_options['smooth_polyorder'])) + # Make plots ... if options['save_plots'] or options['show_plots']: edge_pos = estimate_edge_position(data=data, options=options) - intensity_midpoint = df_smooth[filename].iloc[np.where(df_smooth['ZapEnergy'] == find_nearest(df_smooth['ZapEnergy'], edge_pos))].values[0] step_length = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0] - + # ... if default smoothing is enabled. Only plotting +- 10 step sizes from the edge position if options['smooth_save_default']: fig, (ax1, ax2) = plt.subplots(1,2,figsize=(20,5)) data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax1, kind='scatter') @@ -448,6 +449,7 @@ def smoothing(data: dict, options={}): df_smooth_default.loc[(df_smooth_default['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth_default['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) ax2.set_title(f'{os.path.basename(filename)} - Smooth (default values)', size=20) + # ... if only smoothing with user defined variables is enabled. Only plotting +- 10 step sizes from the edge position elif not options['smooth_save_default']: fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10)) data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, kind='scatter', c='black') @@ -455,12 +457,11 @@ def smoothing(data: dict, options={}): data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > edge_pos-10*step_length) & (data['xanes_data']['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='black', ax=ax2, kind='scatter') df_smooth.loc[(df_smooth['ZapEnergy'] > edge_pos-10*step_length) & (df_smooth['ZapEnergy'] < edge_pos+10*step_length)].plot(x='ZapEnergy', y=filename, color='red', ax=ax2) - #ax.set_xlim([edge_pos-0.0015, edge_pos+0.0015]) - #ax.set_ylim([intensity_midpoint*0.9, intensity_midpoint*1.1]) ax1.set_title(f'{os.path.basename(filename)} - Smooth', size=20) ax2.set_title(f'{os.path.basename(filename)} - Smooth Edge Region', size=20) + # Save plots if options['save_plots']: if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) @@ -468,6 +469,7 @@ def smoothing(data: dict, options={}): dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_smooth.png' plt.savefig(dst, transparent=False) + # Close plots if not options['show_plots']: plt.close() @@ -498,23 +500,29 @@ def smoothing_interactive(data: dict, options: dict) -> None: def restore_from_backup(data): + ''' Restores DataFrame from data['xanes_data_backup'] to data['xanes_data']. This can be useful e.g. when smoothing and you want to re-do the smoothing with different parameters. + + If there is no DataFrame stored in data['xanes_data_backup'], this function does nothing. ''' + if 'xanes_data_bakcup' in data.keys(): data['xanes_data'] = data['xanes_data_backup'] def find_nearest(array, value): - #function to find the value closes to "value" in an "array" + ''' Finds the value closest to value in array''' array = np.asarray(array) idx = (np.abs(array - value)).argmin() return array[idx] def estimate_edge_position(data: dict, options={}, index=0): - #a dataset is differentiated to find a first estimate of the edge shift to use as starting point. + ''' Gets an estimation of the edge position. This is very similar to determine_edge_position, but provides instead a quick and dirty way where the actual data point closest to the maximum of the differentiated data + is located. ''' + required_options = ['log','logfile', 'periods'] default_options = { - 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', + 'log': False, # Toggles logging on/off + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', # Sets path to log-file 'periods': 6, #Periods needs to be an even number for the shifting of values to work properly } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -528,10 +536,8 @@ def estimate_edge_position(data: dict, options={}, index=0): df_diff_max = df_diff[data['path'][index]].dropna().max() estimated_edge_shift =df_diff.loc[df_diff[data['path'][index]] == df_diff_max,'ZapEnergy'].values[0] - # FIXME Add logging option to see the result - if options['log']: - aux.write_log(message=f'Estimated edge shift for determination of pre-edge area is: {estimated_edge_shift} keV', options=options) + aux.write_log(message=f'Estimated edge shift is: {estimated_edge_shift} keV', options=options) return estimated_edge_shift @@ -666,14 +672,14 @@ def determine_edge_position(data: dict, options={}): data['e0_double_diff'][filename] = edge_pos_double_diff - # Make and show / save plots + # Make and show / save plots ... if options['save_plots'] or options['show_plots']: - # If both are enabled + # ... if both are enabled if options['diff'] and options['double_diff']: - fig, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(ncols=3, nrows=2, figsize=(20,20)) + _, ((ax1, ax2, ax3), (ax4, ax5, ax6)) = plt.subplots(ncols=3, nrows=2, figsize=(20,20)) data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') ax1.axvline(x=edge_pos_diff, ls='--', c='green') @@ -708,9 +714,9 @@ def determine_edge_position(data: dict, options={}): ax6.axvline(x=estimated_edge_pos, ls='--', c='red') - # If only first order differentials is enabled + # ... if only first order differentials is enabled elif options['diff']: - fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) + _, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') ax1.axvline(x=edge_pos_diff, ls='--', c='green') @@ -727,9 +733,9 @@ def determine_edge_position(data: dict, options={}): ax3.axvline(x=edge_pos_diff, ls='--', c='green') ax3.axvline(x=estimated_edge_pos, ls='--', c='red') - # If only second order differentials is enabled + # ... if only second order differentials is enabled elif options['double_diff']: - fig, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) + _, (ax1, ax2, ax3) = plt.subplots(ncols=3,nrows=1, figsize=(20, 10)) data['xanes_data'].plot(x='ZapEnergy', y=filename, ax=ax1, c='black') ax1.axvline(x=edge_pos_double_diff, ls='--', c='green') @@ -774,9 +780,6 @@ def determine_edge_position(data: dict, options={}): def determine_edge_position_interactive(data: dict, options: dict) -> None: ''' Defines the widgets to use with the ipywidgets interactive mode and calls the update function found in btp.ipywidgets. ''' - - step_size = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0] - w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(determine_edge_position), data=widgets.fixed(data), options=widgets.fixed(options), points_around_edge=widgets.IntSlider(value=options['points_around_edge'], min=1, max=20, step=1), @@ -786,12 +789,18 @@ def determine_edge_position_interactive(data: dict, options: dict) -> None: display(w) + def normalise(data: dict, options={}): + ''' Normalises the data so that the difference between the fitted pre- and post-edge functions is 1 at the edge position. + + Requires that edge positions have already been determined with determine_edge_position() and stored in data['e0_diff']. ''' + + required_options = ['log', 'logfile', 'normalisation_store_data'] default_options = { - 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log', - 'normalisation_store_data': False, + 'log': False, # Toggles logging on/off + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log', # Sets path to log-file + 'normalisation_store_data': False, # Toggles storing of the flattened data in data['xanes_data'] on/off } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -827,33 +836,44 @@ def normalise(data: dict, options={}): def flatten(data:dict, options={}): - #only picking out zapenergy-values higher than edge position (edge pos and below remains untouched) + ''' Flattens the post-edge region (from edge position and up). Only for visual purposes. + + Requires data['xanes_data'] that is normalised through normalise() and that normalised versions of the post_edge_fit_data is stored in data['post_edge_fit_data_norm']. + Also assumes that the pre edge-fit data is already subtracted from the data''' + required_options = ['log', 'logfile', 'flatten_store_data'] default_options = { - 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_flattening.log', - 'flatten_store_data': False, + 'log': False, # Toggles logging on/off + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_flattening.log', # Sets path to log-file + 'flatten_store_data': False, # Toggles storing of the flattened data in data['xanes_data'] on/off } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + # Initialise DataFrame with x-values flattened_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) + # Loop through all files for filename in data['path']: + + # Subtract 1 from the _normalised_ post edge fit function fit_function_diff = data['post_edge_fit_data_norm'][filename] - 1 + # Set all values from edge position and downwards to 0 so that only data above the edge position will be adjusted fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0_diff'][filename]] = 0 + # Subtract the difference between 1 and the post edge fit function from the normalised data. flattened_df[filename] = data['xanes_data'][filename] - fit_function_diff + # Saves the flattened DataFrame if options['flatten_store_data']: data['xanes_data'] = flattened_df return flattened_df, fit_function_diff - #make a new dataframe with flattened values + From 8702cdfa0025f6e53594ccc7aa0ec0700f1893ec Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 30 Jun 2022 17:07:31 +0200 Subject: [PATCH 204/355] Make logfile-directory if not already exists --- nafuma/auxillary.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 0ccde1f..bf06f8e 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -77,6 +77,10 @@ def write_log(message, options={}): options = update_options(options=options, required_options=required_options, default_options=default_options) + if not os.path.isdir(os.path.dirname(options['logfile'])): + os.makedirs(os.path.dirname(options['logfile'])) + + now = datetime.now().strftime('%Y/%m/%d %H:%M:%S') message = f'[{now}] {message} \n' From 1757445f89892fe41e386edf9442624aa87daa5f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 30 Jun 2022 17:08:14 +0200 Subject: [PATCH 205/355] Add new functions and fix certain bugs --- nafuma/xanes/calib.py | 78 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 13 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 71505f8..7d567b7 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -115,6 +115,11 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: fit_function = np.poly1d(params) data['pre_edge_params'][filename] = params + + if options['log']: + aux.write_log(message=f'Pre edge fitted between {options["pre_edge_limits"][0]} and {options["pre_edge_limits"][1]} with polynomial of order {options["pre_edge_polyorder"]} with parmameters {params}.', options=options) + if options['pre_edge_masks']: + aux.write_log(message=f'Excluded regions: {options["pre_edge_masks"]}', options=options) #making a list, y_pre,so the background will be applied to all ZapEnergy-values background=fit_function(pre_edge_fit_data["ZapEnergy"]) @@ -171,7 +176,7 @@ def pre_edge_fit_interactive(data: dict, options: dict) -> None: w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(pre_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), - pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001), + pre_edge_limits=widgets.FloatRangeSlider(value=[options['pre_edge_limits'][0], options['pre_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.0001), pre_edge_store_data=widgets.Checkbox(value=options['pre_edge_store_data']) ) @@ -267,10 +272,13 @@ def post_edge_fit(data: dict, options={}): edge_position = estimate_edge_position(data, options, index=0) options['post_edge_limits'][0] = edge_position + post_edge_limit_offset - if not options['post_edge_limits'][1]: options['post_edge_limits'][1] = data['xanes_data_original']['ZapEnergy'].max() + if options['post_edge_limits'][0] > options['post_edge_limits'][1]: + options['post_edge_limits'][0] = options['post_edge_limits'][1] - 0.1 + + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session if options['interactive']: options['interactive'] = False @@ -288,6 +296,8 @@ def post_edge_fit(data: dict, options={}): post_edge_data = post_edge_data.dropna() #Removing all indexes without any value, as some of the data sets misses the few last data points and fucks up the fit + + # Making a new dataframe, with only the ZapEnergies as the first column -> will be filled to include the background data post_edge_fit_data = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) @@ -302,7 +312,9 @@ def post_edge_fit(data: dict, options={}): fit_function = np.poly1d(params) if options['log']: - aux.write_log(message=f'Post edge fitted with parameters: {params}') + aux.write_log(message=f'Post edge fitted between {options["post_edge_limits"][0]} and {options["post_edge_limits"][1]} with polynomial of order {options["post_edge_polyorder"]} with parmameters {params}.', options=options) + if options['post_edge_masks']: + aux.write_log(message=f'Excluded regions: {options["post_edge_masks"]}', options=options) data['post_edge_params'][filename] = params @@ -364,7 +376,7 @@ def post_edge_fit_interactive(data: dict, options: dict) -> None: w = widgets.interactive( btp.ipywidgets_update, func=widgets.fixed(post_edge_fit), data=widgets.fixed(data), options=widgets.fixed(options), - post_edge_limits=widgets.FloatRangeSlider(value=[options['post_edge_limits'][0], options['post_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.001), + post_edge_limits=widgets.FloatRangeSlider(value=[options['post_edge_limits'][0], options['post_edge_limits'][1]], min=data['xanes_data_original']['ZapEnergy'].min(), max=data['xanes_data_original']['ZapEnergy'].max(), step=0.0001), post_edge_store_data=widgets.Checkbox(value=options['post_edge_store_data']) ) @@ -400,7 +412,7 @@ def smoothing(data: dict, options={}): df_smooth_default = pd.DataFrame(data['xanes_data']['ZapEnergy']) if options['log']: - aux.write_log(message='Starting smoothing procedure.') + aux.write_log(message='Starting smoothing procedure.', options=options) # Run in interactive mode if enabled @@ -498,6 +510,10 @@ def smoothing_interactive(data: dict, options: dict) -> None: display(w) +def backup(data): + + data['xanes_data_backup'] = data['xanes_data'].copy() + def restore_from_backup(data): ''' Restores DataFrame from data['xanes_data_backup'] to data['xanes_data']. This can be useful e.g. when smoothing and you want to re-do the smoothing with different parameters. @@ -505,7 +521,7 @@ def restore_from_backup(data): If there is no DataFrame stored in data['xanes_data_backup'], this function does nothing. ''' if 'xanes_data_bakcup' in data.keys(): - data['xanes_data'] = data['xanes_data_backup'] + data['xanes_data'] = data['xanes_data_backup'].copy() def find_nearest(array, value): @@ -523,7 +539,7 @@ def estimate_edge_position(data: dict, options={}, index=0): default_options = { 'log': False, # Toggles logging on/off 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_edge_position_estimation.log', # Sets path to log-file - 'periods': 6, #Periods needs to be an even number for the shifting of values to work properly + 'periods': 2, #Periods needs to be an even number for the shifting of values to work properly } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -533,13 +549,25 @@ def estimate_edge_position(data: dict, options={}, index=0): #shifting column values up so that average differential fits right between the points used in the calculation df_diff[data['path'][index]]=df_diff[data['path'][index]].shift(-int(options['periods']/2)) + + + if 'pre_edge_masks' in options.keys(): + for mask in options['pre_edge_masks']: + df_diff[data['path'][index]].loc[(df_diff['ZapEnergy'] > mask[0]) & (df_diff['ZapEnergy'] < mask[1])] = 0 + + if 'post_edge_masks' in options.keys(): + for mask in options['post_edge_masks']: + df_diff[data['path'][index]].loc[(df_diff['ZapEnergy'] > mask[0]) & (df_diff['ZapEnergy'] < mask[1])] = 0 + + df_diff_max = df_diff[data['path'][index]].dropna().max() - estimated_edge_shift =df_diff.loc[df_diff[data['path'][index]] == df_diff_max,'ZapEnergy'].values[0] + + estimated_edge_pos = df_diff.loc[df_diff[data['path'][index]] == df_diff_max,'ZapEnergy'].values[0] if options['log']: - aux.write_log(message=f'Estimated edge shift is: {estimated_edge_shift} keV', options=options) + aux.write_log(message=f'Estimated edge position is: {estimated_edge_pos} keV', options=options) - return estimated_edge_shift + return estimated_edge_pos def determine_edge_position(data: dict, options={}): ''' Determines the edge position by 1) first differential maximum and/or 2) second differential zero-point. Calculates differential and/or double differential by diff.periods and double_diff.periods respectively. @@ -550,7 +578,7 @@ def determine_edge_position(data: dict, options={}): Requires that XANES-data is already loaded in data['xanes_data']. This allows the user to choose when to determine the edge position - whether before or after normalisation, flattening etc.''' - required_options = ['save_values', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'points_around_edge'] + required_options = ['save_values', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'diff', 'diff.polyorder', 'diff.periods', 'double_diff', 'double_diff.polyorder', 'double_diff.periods', 'points_around_edge', 'save_diff_data'] default_options = { 'save_values': True, # Whether the edge positions should be stored in a dictionary within the main data dictionary. 'log': False, # Toggles logging on/off @@ -564,7 +592,8 @@ def determine_edge_position(data: dict, options={}): 'double_diff': False, # Toggles calculation of the edge position based on double differential data 'double_diff.polyorder': 1, # Sets the order of the polynomial to fit edge region of the double differential to 'double_diff.periods': 2, # Sets the number of data points between which the second order difference should be calculated. Needs to be even for subsequent shifting of data to function. - 'points_around_edge': 5 # The length of the region to find points to fit to a function + 'points_around_edge': 1, # The length of the region to find points to fit to a function + 'save_diff_data': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -608,6 +637,9 @@ def determine_edge_position(data: dict, options={}): fit_region = (options['points_around_edge']+1)*(data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0]) + if fit_region < 0: + fit_region = (options['points_around_edge']+1)*(data['xanes_data']['ZapEnergy'].iloc[10] - data['xanes_data']['ZapEnergy'].iloc[9]) + #========================== Fitting the first order derivative ========== @@ -666,7 +698,7 @@ def determine_edge_position(data: dict, options={}): aux.write_log(message=f"Edge position estimated by the double differential zero-point is {str(round(edge_pos_double_diff,5))} keV", options=options) if options['diff']: - aux.write_log(message=f"Difference between edge position estimated from differential maximum and double differential zero-point is {(edge_pos_diff-edge_pos_double_diff)*1000} eV.") + aux.write_log(message=f"Difference between edge position estimated from differential maximum and double differential zero-point is {(edge_pos_diff-edge_pos_double_diff)*1000} eV.", options=options) if options['save_values']: data['e0_double_diff'][filename] = edge_pos_double_diff @@ -773,6 +805,11 @@ def determine_edge_position(data: dict, options={}): if not options['double_diff']: edge_pos_double_diff = None + + if options['save_diff_data']: + data['diff_data'] = df_diff if options['diff'] else None + data['double_diff_data'] = df_double_diff if options['double_diff'] else None + return edge_pos_diff, edge_pos_double_diff @@ -789,6 +826,20 @@ def determine_edge_position_interactive(data: dict, options: dict) -> None: display(w) +def determine_edge_shift(data: dict, options: dict, edge_pos: float) -> None: + + if 'edge' not in data.keys(): + data['edge'] = find_element(data) + + + reference_energy = xas.edges.K['keV'].loc[xas.edges.K['Atom'] == data['edge']].values[0] + + edge_shift = reference_energy - edge_pos + + if options['log']: + aux.write_log(message=f'Edge shift vs. reference value for {data["edge"]} is {edge_shift*1000} eV', options=options) + + return edge_shift def normalise(data: dict, options={}): ''' Normalises the data so that the difference between the fitted pre- and post-edge functions is 1 at the edge position. @@ -806,6 +857,7 @@ def normalise(data: dict, options={}): normalised_df = pd.DataFrame(data['xanes_data']['ZapEnergy']) data['normalisation_constants'] = {} + if options['normalisation_store_data']: pre_edge_fit_data_norm = pd.DataFrame(data['pre_edge_fit_data']['ZapEnergy']) post_edge_fit_data_norm = pd.DataFrame(data['post_edge_fit_data']['ZapEnergy']) From eb8660d71df6ad2ce3fb7a3807b7f41a032b1e01 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 30 Jun 2022 17:08:28 +0200 Subject: [PATCH 206/355] Fix determination of active roi if only one exists --- nafuma/xanes/io.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index be7c9f3..25e792f 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -12,7 +12,7 @@ def split_scan_data(data: dict, options={}) -> list: As of now only picks out xmap_rois (fluoresence mode) and for Mn, Fe, Co and Ni K-edges.''' - required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'add_rois'] + required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'add_rois', 'return'] default_options = { 'log': False, @@ -20,7 +20,8 @@ def split_scan_data(data: dict, options={}) -> list: 'save': False, # whether to save the files or not 'save_folder': '.', # root folder of where to save the files 'replace': False, # whether to replace the files if they already exist - 'add_rois': False # Whether to add the rois of individual scans of the same edge together + 'add_rois': False, # Whether to add the rois of individual scans of the same edge together + 'return': True } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -165,7 +166,10 @@ def split_scan_data(data: dict, options={}) -> list: aux.write_log(message=f'All done!', options=options) - return all_scans + if options['return']: + return all_scans + else: + return @@ -219,8 +223,15 @@ def determine_active_roi(scan_data): # active_roi = 'xmap_roi00' # else: # active_roi = 'xmap_roi01' + + + if not ('xmap_roi00' in scan_data.columns) or not ('xmap_roi01' in scan_data.columns): + if 'xmap_roi00' in scan_data.columns: + active_roi = 'xmap_roi00' + elif 'xmap_roi01' in scan_data.columns: + active_roi = 'xmap_roi01' - if (scan_data['xmap_roi00'].iloc[0:100].mean() < scan_data['xmap_roi00'].iloc[-100:].mean()) and (scan_data['xmap_roi01'].iloc[0:100].mean() < scan_data['xmap_roi01'].iloc[-100:].mean()): + elif (scan_data['xmap_roi00'].iloc[0:100].mean() < scan_data['xmap_roi00'].iloc[-100:].mean()) and (scan_data['xmap_roi01'].iloc[0:100].mean() < scan_data['xmap_roi01'].iloc[-100:].mean()): if (scan_data['xmap_roi00'].max()-scan_data['xmap_roi00'].min()) > (scan_data['xmap_roi01'].max() - scan_data['xmap_roi01'].min()): active_roi = 'xmap_roi00' else: From 7336af061ffe5209fb6e488dff7c8dbe4f57b599 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 1 Jul 2022 14:29:13 +0200 Subject: [PATCH 207/355] Only plot one dataset if in interactive mode --- nafuma/xanes/calib.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 7d567b7..617cc83 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -46,7 +46,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: # FIXME Add log-file - required_options = ['pre_edge_limits', 'pre_edge_masks', 'pre_edge_polyorder', 'pre_edge_store_data', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive'] + required_options = ['pre_edge_limits', 'pre_edge_masks', 'pre_edge_polyorder', 'pre_edge_store_data', 'log', 'logfile', 'show_plots', 'save_plots', 'save_folder', 'ylim', 'interactive', 'interactive_session_active'] default_options = { 'pre_edge_limits': [None, None], 'pre_edge_masks': [], @@ -58,7 +58,8 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: 'save_plots': False, 'save_folder': './', 'ylim': [None, None], - 'interactive': False + 'interactive': False, + 'interactive_session_active': False } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -107,6 +108,10 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: data['pre_edge_params'] = {} for i, filename in enumerate(data['path']): + + if options['interactive_session_active'] and i > 0: + continue + if options['log']: aux.write_log(message=f'Fitting background on {os.path.basename(filename)} ({i+1}/{len(data["path"])})', options=options) @@ -247,7 +252,7 @@ def post_edge_fit(data: dict, options={}): ''' - required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'post_edge_store_data', 'interactive', 'show_plots', 'save_plots', 'save_folder'] + required_options = ['log', 'logfile', 'post_edge_masks', 'post_edge_limits', 'post_edge_polyorder', 'post_edge_store_data', 'interactive', 'interactive_session_active', 'show_plots', 'save_plots', 'save_folder'] default_options = { 'log': False, 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_post_edge_fit.log', @@ -256,6 +261,7 @@ def post_edge_fit(data: dict, options={}): 'post_edge_polyorder': 2, 'post_edge_store_data': False, 'interactive': False, + 'interactive_session_active': False, 'show_plots': False, 'save_plots': False, 'save_folder': './', @@ -304,6 +310,10 @@ def post_edge_fit(data: dict, options={}): data['post_edge_params'] = {} for i, filename in enumerate(data['path']): + + if options['interactive_session_active'] and i > 0: + continue + if options['log']: aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["post_edge_polyorder"]}', options=options) From 1a06e7b4fcace31e3609b36ce10d08517a1d870a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 1 Jul 2022 16:09:43 +0200 Subject: [PATCH 208/355] Poor attempts at fixing bug in roi choice --- nafuma/xanes/calib.py | 2 ++ nafuma/xanes/io.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 617cc83..e16f5a3 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -536,6 +536,7 @@ def restore_from_backup(data): def find_nearest(array, value): ''' Finds the value closest to value in array''' + array = np.asarray(array) idx = (np.abs(array - value)).argmin() return array[idx] @@ -874,6 +875,7 @@ def normalise(data: dict, options={}): #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 for filename in data['path']: + print(filename) e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0_diff'][filename])].index.values[0] #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 25e792f..7a554db 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -225,6 +225,9 @@ def determine_active_roi(scan_data): # active_roi = 'xmap_roi01' + # FIXME This is broken now - + + if not ('xmap_roi00' in scan_data.columns) or not ('xmap_roi01' in scan_data.columns): if 'xmap_roi00' in scan_data.columns: active_roi = 'xmap_roi00' @@ -232,7 +235,7 @@ def determine_active_roi(scan_data): active_roi = 'xmap_roi01' elif (scan_data['xmap_roi00'].iloc[0:100].mean() < scan_data['xmap_roi00'].iloc[-100:].mean()) and (scan_data['xmap_roi01'].iloc[0:100].mean() < scan_data['xmap_roi01'].iloc[-100:].mean()): - if (scan_data['xmap_roi00'].max()-scan_data['xmap_roi00'].min()) > (scan_data['xmap_roi01'].max() - scan_data['xmap_roi01'].min()): + if ((scan_data['xmap_roi00'].max()-scan_data['xmap_roi00'].min())) > ((scan_data['xmap_roi01'].max() - scan_data['xmap_roi01'].min())): active_roi = 'xmap_roi00' else: active_roi = 'xmap_roi01' From b5cac158a4697c23e9f2b90bc14686d326c16202 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 4 Jul 2022 18:18:31 +0200 Subject: [PATCH 209/355] Fix active_roi determination --- nafuma/xanes/calib.py | 3 +-- nafuma/xanes/io.py | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index e16f5a3..a5e4455 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -375,7 +375,7 @@ def post_edge_fit(data: dict, options={}): aux.write_log(message='Post edge fitting done!', options=options) if options['post_edge_store_data']: - data['post_edge_fit_data'] = post_edge_fit_data + data['post_edge_fit_data'] = post_edge_fit_data.dropna(axis=0) return post_edge_fit_data @@ -875,7 +875,6 @@ def normalise(data: dict, options={}): #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 for filename in data['path']: - print(filename) e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0_diff'][filename])].index.values[0] #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 7a554db..217a529 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -225,9 +225,6 @@ def determine_active_roi(scan_data): # active_roi = 'xmap_roi01' - # FIXME This is broken now - - - if not ('xmap_roi00' in scan_data.columns) or not ('xmap_roi01' in scan_data.columns): if 'xmap_roi00' in scan_data.columns: active_roi = 'xmap_roi00' @@ -235,7 +232,7 @@ def determine_active_roi(scan_data): active_roi = 'xmap_roi01' elif (scan_data['xmap_roi00'].iloc[0:100].mean() < scan_data['xmap_roi00'].iloc[-100:].mean()) and (scan_data['xmap_roi01'].iloc[0:100].mean() < scan_data['xmap_roi01'].iloc[-100:].mean()): - if ((scan_data['xmap_roi00'].max()-scan_data['xmap_roi00'].min())) > ((scan_data['xmap_roi01'].max() - scan_data['xmap_roi01'].min())): + if (scan_data['xmap_roi00'].iloc[:int(scan_data.shape[0]/2)].max() - scan_data['xmap_roi00'].iloc[0])/scan_data['xmap_roi00'].max() > (scan_data['xmap_roi01'].iloc[:int(scan_data.shape[0]/2)].max() - scan_data['xmap_roi01'].iloc[0])/scan_data['xmap_roi01'].max(): active_roi = 'xmap_roi00' else: active_roi = 'xmap_roi01' From da7099a9248b0d5a37f334cb6f766a5fe529a5b1 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Jul 2022 10:55:39 +0200 Subject: [PATCH 210/355] Add manual choice of active roi --- nafuma/xanes/io.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 217a529..a590255 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -12,7 +12,7 @@ def split_scan_data(data: dict, options={}) -> list: As of now only picks out xmap_rois (fluoresence mode) and for Mn, Fe, Co and Ni K-edges.''' - required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'add_rois', 'return'] + required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'active_roi', 'add_rois', 'return'] default_options = { 'log': False, @@ -20,6 +20,7 @@ def split_scan_data(data: dict, options={}) -> list: 'save': False, # whether to save the files or not 'save_folder': '.', # root folder of where to save the files 'replace': False, # whether to replace the files if they already exist + 'active_roi': None, 'add_rois': False, # Whether to add the rois of individual scans of the same edge together 'return': True } @@ -200,7 +201,11 @@ def read_data(data: dict, options={}) -> pd.DataFrame: scan_data = pd.read_csv(filename) - scan_data = scan_data[[determine_active_roi(scan_data)]] + if not options['active_roi']: + scan_data = scan_data[[determine_active_roi(scan_data)]] + else: + scan_data = scan_data[options['active_roi']] + xanes_data = pd.concat([xanes_data, scan_data], axis=1) From 5e2cef2cdea3dfcf7f3d0c68d9ce6b6943c62186 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Jul 2022 16:37:31 +0200 Subject: [PATCH 211/355] Let save_options create folder if it doesn't exist --- nafuma/auxillary.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index bf06f8e..45633c5 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -25,6 +25,10 @@ def save_options(options, path, ignore=None): options_copy[i] = 'Removed' + if not os.path.isdir(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + + with open(path, 'w') as f: json.dump(options_copy,f, skipkeys=True, indent=4) From 327cef5b51e8760c21bd6fbd00f9659940d662d5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 5 Jul 2022 16:37:42 +0200 Subject: [PATCH 212/355] Log updates and let normalise and flatten save plots --- nafuma/xanes/calib.py | 94 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 83 insertions(+), 11 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index a5e4455..84ed070 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -82,6 +82,11 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: edge_position = estimate_edge_position(data, options, index=0) options['pre_edge_limits'][1] = edge_position - pre_edge_limit_offset + print(edge_position) + + if options['pre_edge_limits'][0] >= options['pre_edge_limits'][1]: + options['pre_edge_limits'][1] = options['pre_edge_limits'][0] + 0.03 + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session if options['interactive']: options['interactive'] = False @@ -113,7 +118,7 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: continue if options['log']: - aux.write_log(message=f'Fitting background on {os.path.basename(filename)} ({i+1}/{len(data["path"])})', options=options) + aux.write_log(message=f'... Fitting pre edge on {os.path.basename(filename)} ({i+1}/{len(data["path"])})', options=options) #Fitting linear function to the background params = np.polyfit(pre_edge_data["ZapEnergy"],pre_edge_data[filename],options['pre_edge_polyorder']) @@ -122,9 +127,9 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: data['pre_edge_params'][filename] = params if options['log']: - aux.write_log(message=f'Pre edge fitted between {options["pre_edge_limits"][0]} and {options["pre_edge_limits"][1]} with polynomial of order {options["pre_edge_polyorder"]} with parmameters {params}.', options=options) + aux.write_log(message=f'...... Pre edge fitted between {options["pre_edge_limits"][0]} and {options["pre_edge_limits"][1]} with polynomial of order {options["pre_edge_polyorder"]} with parmameters {params}.', options=options) if options['pre_edge_masks']: - aux.write_log(message=f'Excluded regions: {options["pre_edge_masks"]}', options=options) + aux.write_log(message=f'...... Excluded regions: {options["pre_edge_masks"]}', options=options) #making a list, y_pre,so the background will be applied to all ZapEnergy-values background=fit_function(pre_edge_fit_data["ZapEnergy"]) @@ -212,7 +217,7 @@ def pre_edge_subtraction(data: dict, options={}): for i, filename in enumerate(data['path']): if options['log']: - aux.write_log(message=f'Subtracting background on {filename} ({i} / {len(data["path"])}', options=options) + aux.write_log(message=f'... Subtracting background on {os.path.basename(filename)} ({i}/{len(data["path"])})', options=options) xanes_data_bkgd_subtracted.insert(1, filename, data['xanes_data_original'][filename] - data['pre_edge_fit_data'][filename]) @@ -315,16 +320,16 @@ def post_edge_fit(data: dict, options={}): continue if options['log']: - aux.write_log(message=f'Fitting post edge on {os.path.basename(filename)} ({i+1} / {len(data["path"])}) with polynomial order {options["post_edge_polyorder"]}', options=options) + aux.write_log(message=f'... Fitting post edge on {os.path.basename(filename)} ({i+1}/{len(data["path"])}) with polynomial order {options["post_edge_polyorder"]}', options=options) #Fitting linear function to the background params = np.polyfit(post_edge_data["ZapEnergy"], post_edge_data[filename], options['post_edge_polyorder']) fit_function = np.poly1d(params) if options['log']: - aux.write_log(message=f'Post edge fitted between {options["post_edge_limits"][0]} and {options["post_edge_limits"][1]} with polynomial of order {options["post_edge_polyorder"]} with parmameters {params}.', options=options) + aux.write_log(message=f'...... Post edge fitted between {options["post_edge_limits"][0]} and {options["post_edge_limits"][1]} with polynomial of order {options["post_edge_polyorder"]} with parmameters {params}.', options=options) if options['post_edge_masks']: - aux.write_log(message=f'Excluded regions: {options["post_edge_masks"]}', options=options) + aux.write_log(message=f'...... Excluded regions: {options["post_edge_masks"]}', options=options) data['post_edge_params'][filename] = params @@ -570,6 +575,9 @@ def estimate_edge_position(data: dict, options={}, index=0): for mask in options['post_edge_masks']: df_diff[data['path'][index]].loc[(df_diff['ZapEnergy'] > mask[0]) & (df_diff['ZapEnergy'] < mask[1])] = 0 + if 'edge_masks' in options.keys(): + for mask in options['edge_masks']: + df_diff[data['path'][index]].loc[(df_diff['ZapEnergy'] > mask[0]) & (df_diff['ZapEnergy'] < mask[1])] = 0 df_diff_max = df_diff[data['path'][index]].dropna().max() @@ -597,6 +605,7 @@ def determine_edge_position(data: dict, options={}): 'show_plots': False, # Toggles on/off whether plots should be shown. For sequential data, saving the plots and inspecting them there is probably better. 'save_plots': False, # Toggles on/off whether plots should be saved. 'save_folder': './', # Sets the path to where the plots should be saved. Creates folder if doesn't exist. Ignored if save_plots == False + 'edge_masks': [], 'diff': True, # Toggles calculation of the edge position based on differential data 'diff.polyorder': 2, # Sets the order of the polynomial to fit edge region of the differential to 'diff.periods': 2, # Sets the number of data points between which the first order difference should be calculated. Needs to be even for subsequent shifting of data to function. @@ -641,8 +650,13 @@ def determine_edge_position(data: dict, options={}): data['e0_double_diff'] = {} + if options['log']: + aux.write_log(message='Starting edge position determination', options=options) + + # Get rough estimate of edge position for i, filename in enumerate(data['path']): + estimated_edge_pos = estimate_edge_position(data, options=options, index=i) @@ -677,7 +691,7 @@ def determine_edge_position(data: dict, options={}): edge_pos_diff=x_diff[np.where(y_diff == np.amax(y_diff))][0] if options['log']: - aux.write_log(message=f"Edge position estimated by the differential maximum is: {str(round(edge_pos_diff,5))} keV", options=options) + aux.write_log(message=f"... Edge position of {os.path.basename(filename)} determined by the differential maximum is: {str(round(edge_pos_diff,5))} keV", options=options) if options['save_values']: data['e0_diff'][filename] = edge_pos_diff @@ -706,10 +720,10 @@ def determine_edge_position(data: dict, options={}): edge_pos_double_diff=x_double_diff[np.where(y_double_diff == find_nearest(y_double_diff,0))][0] if options['log']: - aux.write_log(message=f"Edge position estimated by the double differential zero-point is {str(round(edge_pos_double_diff,5))} keV", options=options) + aux.write_log(message=f"... Edge position of {os.path.basename(filename)} determined by the double differential zero-point is {str(round(edge_pos_double_diff,5))} keV", options=options) if options['diff']: - aux.write_log(message=f"Difference between edge position estimated from differential maximum and double differential zero-point is {(edge_pos_diff-edge_pos_double_diff)*1000} eV.", options=options) + aux.write_log(message=f"... Difference between edge position estimated from differential maximum and double differential zero-point is {(edge_pos_diff-edge_pos_double_diff)*1000} eV.", options=options) if options['save_values']: data['e0_double_diff'][filename] = edge_pos_double_diff @@ -862,6 +876,9 @@ def normalise(data: dict, options={}): default_options = { 'log': False, # Toggles logging on/off 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_normalisation.log', # Sets path to log-file + 'show_plots': False, # Toggles on/off whether plots should be shown. For sequential data, saving the plots and inspecting them there is probably better. + 'save_plots': False, # Toggles on/off whether plots should be saved. + 'save_folder': './', # Sets the path to where the plots should be saved. Creates folder if doesn't exist. Ignored if save_plots == False 'normalisation_store_data': False, # Toggles storing of the flattened data in data['xanes_data'] on/off } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -877,15 +894,44 @@ def normalise(data: dict, options={}): for filename in data['path']: e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0_diff'][filename])].index.values[0] #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] - normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] + normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] #- data['pre_edge_fit_data'][filename].iloc[e0_ind] normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant) + + if options['show_plots'] or options['save_plots']: + + fig, ax = plt.subplots(figsize=(10,5)) + + normalised_df.plot(x='ZapEnergy', y=filename, ax=ax, color='red', label='Normalised data') + ax.set_title(f'{os.path.basename(filename)} - After normalisation', size=20) + ax.set_ylabel('Normalised x$\mu$(E)', size=20) + ax.set_xlabel('Energy (keV)', size=20) + ax.axhline(y=1, ls='--', c='black') + + + # Save plots if toggled + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_normalisation.png' + + plt.savefig(dst, transparent=False) + + + # Close plots if show_plots not toggled + if not options['show_plots']: + plt.close() + + if options['normalisation_store_data']: pre_edge_fit_data_norm.insert(1, filename, data['pre_edge_fit_data'][filename] / normalisation_constant) post_edge_fit_data_norm.insert(1, filename, data['post_edge_fit_data'][filename] / normalisation_constant) + + if options['normalisation_store_data']: data['xanes_data'] = normalised_df # Normalise the pre-edge and post-edge fit function data @@ -930,6 +976,32 @@ def flatten(data:dict, options={}): flattened_df[filename] = data['xanes_data'][filename] - fit_function_diff + if options['show_plots'] or options['save_plots']: + + fig, ax = plt.subplots(figsize=(10,5)) + + flattened_df.plot(x='ZapEnergy', y=filename, ax=ax, color='red', label='Flattened data') + ax.set_title(f'{os.path.basename(filename)} - After flattening', size=20) + ax.set_ylabel('Normalised x$\mu$(E)', size=20) + ax.set_xlabel('Energy (keV)', size=20) + ax.axhline(y=1, ls='--', c='black') + + + # Save plots if toggled + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_flattened.png' + + plt.savefig(dst, transparent=False) + + + # Close plots if show_plots not toggled + if not options['show_plots']: + plt.close() + + # Saves the flattened DataFrame if options['flatten_store_data']: data['xanes_data'] = flattened_df From 6eb45772d1947aee7b467cdcfc6e7f8551c1f376 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 6 Jul 2022 17:33:38 +0200 Subject: [PATCH 213/355] Fix normalisation issues --- nafuma/xanes/calib.py | 8 +++++--- nafuma/xanes/io.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 84ed070..2c8881e 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -206,7 +206,7 @@ def pre_edge_subtraction(data: dict, options={}): 'show_plots': False, 'save_plots': False, 'save_folder': './', - 'pre_edge_subtraction_store_data': False + 'pre_edge_subtraction_store_data': True } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -893,8 +893,10 @@ def normalise(data: dict, options={}): #Finding the normalisation constant µ_0(E_0), by subtracting the value of the pre-edge-line from the value of the post-edge line at e0 for filename in data['path']: e0_ind = data['post_edge_fit_data'].loc[data['post_edge_fit_data']['ZapEnergy'] == find_nearest(data['post_edge_fit_data']['ZapEnergy'], data['e0_diff'][filename])].index.values[0] + #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] - normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] #- data['pre_edge_fit_data'][filename].iloc[e0_ind] + normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] + print(normalisation_constant) normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant) @@ -967,7 +969,7 @@ def flatten(data:dict, options={}): for filename in data['path']: # Subtract 1 from the _normalised_ post edge fit function - fit_function_diff = data['post_edge_fit_data_norm'][filename] - 1 + fit_function_diff = data['post_edge_fit_data_norm'][filename] - 1 - data['pre_edge_fit_data_norm'][filename] # Set all values from edge position and downwards to 0 so that only data above the edge position will be adjusted fit_function_diff.loc[flattened_df['ZapEnergy'] <= data['e0_diff'][filename]] = 0 diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index a590255..6c4ac1a 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -57,7 +57,7 @@ def split_scan_data(data: dict, options={}) -> list: continue # First line after data started with #C - stops data read-in - elif line[0:2] == "#C": + elif line[0:2] == "#C" or line[0:2] == '#S': read_data = False if scan_data: From 4cec8d275663c4cdc3b98f693a0658dee43c6d3e Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Jul 2022 10:24:04 +0200 Subject: [PATCH 214/355] added FIXMEs --- nafuma/xanes/io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 6c4ac1a..70f6891 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -178,7 +178,8 @@ def read_data(data: dict, options={}) -> pd.DataFrame: # FIXME Handle the case when dataseries are not the same size - + # FIXME Add possibility to extract TIME (for operando runs) and Blower Temp (for variable temperature runs) + # FIXME Add possibility to iport transmission data required_options = ['adjust'] default_options = { 'adjust': 0 From 8939bb8479cc1ba6c7813734102b20c308dd71f9 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Jul 2022 11:49:58 +0200 Subject: [PATCH 215/355] Remove annoying print in normalise() --- nafuma/xanes/calib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 2c8881e..77e3a5c 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -896,7 +896,6 @@ def normalise(data: dict, options={}): #norm = data['post_edge_fit_data'][filename].iloc[find_nearest(data['post_edge_fit_data'][filename], data['e0'][filename])] normalisation_constant = data['post_edge_fit_data'][filename].iloc[e0_ind] - data['pre_edge_fit_data'][filename].iloc[e0_ind] - print(normalisation_constant) normalised_df.insert(1, filename, data['xanes_data'][filename] / normalisation_constant) From 0ed85e1398949625e02b25f6c6f3e124f1aa1f54 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Jul 2022 11:50:15 +0200 Subject: [PATCH 216/355] Write timestamp during split and add reading of metadata --- nafuma/xanes/io.py | 70 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 70f6891..f62554b 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -4,7 +4,7 @@ import os import numpy as np import nafuma.auxillary as aux from nafuma.xanes.calib import find_element -from datetime import datetime +import datetime def split_scan_data(data: dict, options={}) -> list: @@ -16,7 +16,7 @@ def split_scan_data(data: dict, options={}) -> list: default_options = { 'log': False, - 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_split_edges.log', + 'logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_split_edges.log', 'save': False, # whether to save the files or not 'save_folder': '.', # root folder of where to save the files 'replace': False, # whether to replace the files if they already exist @@ -43,13 +43,17 @@ def split_scan_data(data: dict, options={}) -> list: with open(filename, 'r') as f: lines = f.readlines() + timestamps = [] scan_datas, scan_data = [], [] headers, header = [], '' read_data = False - for line in lines: + for i, line in enumerate(lines): # Header line starts with #L - reads headers, and toggles data read-in on - if line[0:2] == "#L": + if 'zapline mono' in line: + timestamps.append(lines[i+1].strip('#D')) + + elif line[0:2] == "#L": header, read_data = line[2:].split(), True if options['log']: @@ -148,12 +152,19 @@ def split_scan_data(data: dict, options={}) -> list: path = os.path.join(options['save_folder'], f'{filename}_{edge}{count}.dat') if not os.path.isfile(path): - scan.to_csv(path) + + with open(path, 'w', newline = '\n') as f: + + f.write(f'# Time: {timestamps[i]}') + scan.to_csv(f) + if options['log']: aux.write_log(message=f'... ... Scan saved to {path}', options=options) elif options['replace'] and os.path.isfile(path): - scan.to_csv(path) + with open(path, 'w', newline = '\n') as f: + scan.to_csv(f) + if options['log']: aux.write_log(message=f'... ... File already exists. Overwriting to {path}', options=options) @@ -193,14 +204,14 @@ def read_data(data: dict, options={}) -> pd.DataFrame: data['path'] = [data['path']] # Initialise DataFrame with only ZapEnergy-column - xanes_data = pd.read_csv(data['path'][0])[['ZapEnergy']] + xanes_data = pd.read_csv(data['path'][0], skiprows=1)[['ZapEnergy']] xanes_data['ZapEnergy'] += options['adjust'] for filename in data['path']: columns.append(filename) - scan_data = pd.read_csv(filename) + scan_data = pd.read_csv(filename, skiprows=1) if not options['active_roi']: scan_data = scan_data[[determine_active_roi(scan_data)]] @@ -216,6 +227,49 @@ def read_data(data: dict, options={}) -> pd.DataFrame: return xanes_data +def read_metadata(data: dict, options={}) -> dict: + + required_options = ['get_temperature', 'get_timestamp', 'adjust_time'] + + default_options = { + 'get_temperature': True, + 'get_timestamp': True, + 'adjust_time': False + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + temperatures = [] + timestamps = [] + + for filename in data['path']: + scan_data = pd.read_csv(filename, skiprows=1) + + if options['get_temperature']: + temperatures.append(scan_data['ZBlower2'].mean()) + + if options['get_timestamp']: + + with open(filename, 'r') as f: + time = f.readline().strip('# Time: ') + time = datetime.datetime.strptime(time, "%a %b %d %H:%M:%S %Y ") + + if options['adjust_time']: + time_elapsed = scan_data['Htime'].iloc[-1] - scan_data['Htime'].iloc[0] + + time += datetime.timedelta(microseconds=time_elapsed)/2 + + + timestamps.append(time) + + + metadata = {'time': timestamps, 'temperature': temperatures} + + return metadata + + + From b1b705f28f7d3b888f6e8bf1995ec429bef38ebb Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Jul 2022 13:07:22 +0200 Subject: [PATCH 217/355] Ignore makedirs-command if file in base folder --- nafuma/auxillary.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 45633c5..a63c8fc 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -26,11 +26,12 @@ def save_options(options, path, ignore=None): if not os.path.isdir(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) + if os.path.dirname(path): + os.makedirs(os.path.dirname(path)) with open(path, 'w') as f: - json.dump(options_copy,f, skipkeys=True, indent=4) + json.dump(options_copy, f, skipkeys=True, indent=4) def load_options(path): From f0c547f889be17454f75828aeb324aa1d4c51f20 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Jul 2022 14:49:27 +0200 Subject: [PATCH 218/355] Adding function to rearrange filenames-array --- nafuma/auxillary.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 45633c5..bd30713 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -103,4 +103,11 @@ def get_filenames(path, ext, filter=''): filenames = [os.path.join(path, filename) for filename in os.listdir(path) if os.path.isfile(os.path.join(path, filename)) and filename.endswith(ext) and filter in filename] + return filenames + +def move_list_element_last(filenames,string): + for i,file in enumerate(filenames): + if string in file: + del filenames[i] + filenames.append(file) return filenames \ No newline at end of file From 8096cf3bd540a8cb142e4439ff51461c2aa79b83 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 7 Jul 2022 14:51:57 +0200 Subject: [PATCH 219/355] no change --- nafuma/xanes/io.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 70f6891..98a3943 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -253,3 +253,4 @@ def determine_active_roi(scan_data): active_roi = None return active_roi + \ No newline at end of file From 7f61617d9af9f801a7d80d7c361e401754693411 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Jul 2022 14:52:56 +0200 Subject: [PATCH 220/355] Add HTXRD-reading with saving in individual files --- nafuma/xrd/io.py | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index ef77b0e..d7a3b51 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -5,6 +5,7 @@ import numpy as np import os import shutil import sys +import datetime import zipfile import xml.etree.ElementTree as ET @@ -320,13 +321,20 @@ def read_brml(data, options={}, index=0): def read_htxrd(data, options={}, index=0): - required_options = ['extract_folder', 'save_folder', 'save_filename'] + required_options = ['extract_folder', 'save_folder', 'save_filename', 'adjust_time'] default_options = { 'extract_folder': 'tmp', 'save_folder': None, - 'save_filename': None + 'save_filename': None, + 'adjust_time': True } + if not isinstance(data['path'], list): + data['path'] = [data['path']] + + if 'wavelength' not in data.keys(): + data['wavelength'] = [None for i in range(len(data['path']))] + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -343,6 +351,9 @@ def read_htxrd(data, options={}, index=0): # initalise empty list to store all DataFrames diffractograms = [] wavelengths = [] + + active_scan = False + timestamps = [] # Loop through all RawData-files and extract all data and temperatures for i, file in enumerate(files): @@ -357,7 +368,6 @@ def read_htxrd(data, options={}, index=0): # initalise empty list to store data from this particular scan diffractogram = [] - for chain in root.findall('./DataRoutes/DataRoute'): scantypes = chain.findall('ScanInformation') @@ -367,6 +377,7 @@ def read_htxrd(data, options={}, index=0): continue else: + active_scan = True if chain.get('RouteFlag') == 'Final': for scandata in chain.findall('Datum'): scandata = scandata.text.split(',') @@ -387,8 +398,22 @@ def read_htxrd(data, options={}, index=0): wavelengths.append(wavelength) + if active_scan: + for chain in root.findall('./TimeStampStarted'): + time_start = datetime.datetime.strptime(chain.text[:-7], "%Y-%m-%dT%H:%M:%S.%f") + for chain in root.findall('./TimeStampFinished'): + time_end = datetime.datetime.strptime(chain.text[:-7], "%Y-%m-%dT%H:%M:%S.%f") + + + time_diff = time_end - time_start + + if options['adjust_time']: + timestamps.append(time_start + time_diff/2) + + if options['save_folder']: - for i, (diffractogram, wavelength) in enumerate(zip(diffractograms, wavelengths)): + print(options['save_folder']) + for i, (diffractogram, wavelength, timestamp) in enumerate(zip(diffractograms, wavelengths, timestamps)): if not options['save_filename']: filename = os.path.basename(data['path'][index]).split('.')[0] + '_' + str(i).zfill(4) +'.xy' else: @@ -398,7 +423,7 @@ def read_htxrd(data, options={}, index=0): if not os.path.isdir(options['save_folder']): os.makedirs(options['save_folder']) - save_htxrd_as_xy(diffractogram, wavelength, filename, options['save_folder']) + save_htxrd_as_xy(diffractogram, wavelength, timestamp, filename, options['save_folder']) @@ -408,19 +433,20 @@ def read_htxrd(data, options={}, index=0): return diffractograms, wavelengths -def save_htxrd_as_xy(diffractogram, wavelength, filename, save_path): +def save_htxrd_as_xy(diffractogram, wavelength, timestamp, filename, save_path): headers = '\n'.join( [line for line in [f'# Temperature {np.round(diffractogram["T"].mean())}', f'# Wavelength {wavelength}', + f'# Time {timestamp}' ] ] ) diffractogram = diffractogram.drop('T', axis=1) - with open(os.path.join(save_path, filename), 'w') as f: + with open(os.path.join(save_path, filename), 'w', newline='\n') as f: for line in headers: f.write(line) From c6b5dc4627929c58753ef885cedf4c864a2034cf Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 7 Jul 2022 15:15:07 +0200 Subject: [PATCH 221/355] Update with new version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b3f9f34..59274fe 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='nafuma', - version='0.2', + version='0.3', description='Analysis tools for inorganic materials chemistry at the NAFUMA-group at the University of Oslo', url='https://github.com/rasmusthog/nafuma', author='Rasmus Vester Thøgersen, Halvor Høen Hval', From c9b620f16649da80caabb8fcf361c5034ddfbfe5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Jul 2022 17:41:13 +0200 Subject: [PATCH 222/355] Get HT-XRD plotting to a working state --- nafuma/plotting.py | 13 ++- nafuma/xrd/io.py | 112 +++++++++++-------- nafuma/xrd/plot.py | 262 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 292 insertions(+), 95 deletions(-) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 416c1cb..4bf0ed9 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -55,6 +55,8 @@ def prepare_plot(options={}): format_params = aux.update_options(format_params, required_format_params, default_format_params) + + # Reset run commands plt.rcdefaults() @@ -125,12 +127,15 @@ def adjust_plot(fig, ax, options): } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) # Set labels on x- and y-axes if not options['hide_y_labels']: - ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + if not options['yunit']: + ax.set_ylabel(f'{options["ylabel"]}') + else: + ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + else: ax.set_ylabel('') @@ -141,8 +146,7 @@ def adjust_plot(fig, ax, options): ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]') else: ax.set_xlabel('') - - + # Set multiple locators if options['y_tick_locators']: ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) @@ -269,7 +273,6 @@ def ipywidgets_update(func, data, options={}, **kwargs): where key1, key2, key3 etc. are the values in the options-dictionary you want widget control of, and widget1, widget2, widget3 etc. are widgets to control these values, e.g. widgets.IntSlider(value=1, min=0, max=10) ''' - # Update the options-dictionary with the values from the widgets for key in kwargs: options[key] = kwargs[key] diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index d7a3b51..41821ca 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -412,7 +412,6 @@ def read_htxrd(data, options={}, index=0): if options['save_folder']: - print(options['save_folder']) for i, (diffractogram, wavelength, timestamp) in enumerate(zip(diffractograms, wavelengths, timestamps)): if not options['save_filename']: filename = os.path.basename(data['path'][index]).split('.')[0] + '_' + str(i).zfill(4) +'.xy' @@ -439,7 +438,8 @@ def save_htxrd_as_xy(diffractogram, wavelength, timestamp, filename, save_path): [line for line in [f'# Temperature {np.round(diffractogram["T"].mean())}', f'# Wavelength {wavelength}', - f'# Time {timestamp}' + f'# Time {timestamp}', + '# 2th \t I' ] ] ) @@ -452,7 +452,7 @@ def save_htxrd_as_xy(diffractogram, wavelength, timestamp, filename, save_path): f.write('\n') - diffractogram.to_csv(f, index=False, sep='\t') + diffractogram.to_csv(f, index=False, header=False, sep='\t') def read_xy(data, options={}, index=0): @@ -460,8 +460,10 @@ def read_xy(data, options={}, index=0): #if 'wavelength' not in data.keys(): # Get wavelength from scan - if not 'wavelength' in data.keys() or data['wavelength'][index]: - wavelength = find_wavelength_from_xy(path=data['path'][index]) + + + if not 'wavelength' in data.keys() or not data['wavelength'][index]: + wavelength = read_metadata_from_xy(path=data['path'][index])['wavelength'] else: wavelength = data['wavelength'][index] @@ -478,6 +480,7 @@ def read_xy(data, options={}, index=0): diffractogram = pd.read_csv(f, header=None, delim_whitespace=True) + if diffractogram.shape[1] == 2: diffractogram.columns = ['2th', 'I'] elif diffractogram.shape[1] == 3: @@ -488,6 +491,65 @@ def read_xy(data, options={}, index=0): +def read_metadata_from_xy(path): + + metadata = {} + wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} + + with open(path, 'r') as f: + lines = f.readlines() + + for line in lines: + # For .xy-files output from EVA + if 'Anode' in line: + anode = line.split()[8].strip('"') + metadata['wavelength'] = wavelength_dict[anode] + + + elif 'Wavelength' in line: + # For .xy-files output from pyFAI integration + if line.split()[-1] == 'm': + metadata['wavelength'] = float(line.split()[2])*10**10 + + else: + metadata['wavelength'] = float(line.split()[-1]) + + + # Get temperature - exists in .xy-files saved from HTXRD-runs in .brml-files + if 'Temperature' in line: + metadata['temperature'] = line.split()[-1] + + # Get timestamp - exists in .xy-files saved from .brml-files + if 'Time' in line: + metadata['time'] = " ".join(line.split()[2:]) + + + + + if 'wavelength' not in metadata.keys(): + metadata['wavelength'] = None + if 'temperature' not in metadata.keys(): + metadata['temperature'] = None + if 'time' not in metadata.keys(): + metadata['time'] = None + + return metadata + + +def find_wavelength_from_poni(path): + + with open(path, 'r') as f: + lines = f.readlines() + + for line in lines: + if 'Wavelength' in line: + wavelength = float(line.split()[-1])*10**10 + + + return wavelength + + + def strip_headers_from_xy(path: str, filename=None) -> None: ''' Strips headers from a .xy-file''' @@ -533,9 +595,7 @@ def read_data(data, options={}, index=0): elif file_extension in['xy', 'xye']: diffractogram, wavelength = read_xy(data=data, options=options, index=index) - - - + if options['offset'] or options['normalise']: # Make copy of the original intensities before any changes are made through normalisation or offset, to easily revert back if need to update. diffractogram['I_org'] = diffractogram['I'] @@ -704,40 +764,4 @@ def translate_wavelengths(data: pd.DataFrame, wavelength: float, to_wavelength=N -def find_wavelength_from_xy(path): - - wavelength_dict = {'Cu': 1.54059, 'Mo': 0.71073} - - with open(path, 'r') as f: - lines = f.readlines() - - for line in lines: - # For .xy-files output from EVA - if 'Anode' in line: - anode = line.split()[8].strip('"') - wavelength = wavelength_dict[anode] - - # For .xy-files output from pyFAI integration - elif 'Wavelength' in line: - wavelength = float(line.split()[2])*10**10 - - else: - wavelength = None - - - - return wavelength - - -def find_wavelength_from_poni(path): - - with open(path, 'r') as f: - lines = f.readlines() - - for line in lines: - if 'Wavelength' in line: - wavelength = float(line.split()[-1])*10**10 - - - return wavelength diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index b562eeb..4253c9c 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -1,6 +1,6 @@ import seaborn as sns import matplotlib.pyplot as plt -from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) +from matplotlib.ticker import MultipleLocator import pandas as pd import numpy as np @@ -21,13 +21,12 @@ def plot_diffractogram(data, options={}): # Update options required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', 'offset_change', - 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] + 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'highlight', 'highlight_colours', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] default_options = { - 'x_vals': '2th', - 'y_vals': 'I', - 'ylabel': 'Intensity', 'xlabel': '2theta', - 'xunit': 'deg', 'yunit': 'a.u.', + 'x_vals': '2th', 'y_vals': 'I', + 'xlabel': '2$\\theta$', 'ylabel': None, + 'xunit': '$^{\circ}$', 'yunit': None, 'xlim': None, 'ylim': None, 'normalise': True, 'offset': True, @@ -43,6 +42,8 @@ def plot_diffractogram(data, options={}): 'cmap': 'viridis', 'plot_kind': None, 'palettes': [('qualitative', 'Dark2_8')], + 'highlight': None, + 'highlight_colours': ['red'], 'interactive': False, 'interactive_session_active': False, 'rc_params': {}, @@ -60,14 +61,21 @@ def plot_diffractogram(data, options={}): if not isinstance(data['path'], list): data['path'] = [data['path']] - - + + ############################################################################################################################################################ + ##### LOADING DATA ######################################################################################################################################### + ############################################################################################################################################################ + # Check if there is some data stored already, load in data if not. This speeds up replotting in interactive mode. if not 'diffractogram' in data.keys(): - # Initialise empty list for diffractograms and wavelengths + + # This is to set the default values of the diffractogram y-label and -unit so that the actual yunit and ylable can switch back and forth between these and the heatmap values + options['diff.yunit'] = 'a.u.' + options['diff.ylabel'] = 'Intensity' + + # Initialise empty list for diffractograms and wavelengths. If wavelength is not manually passed it should be automatically gathered from the .xy-file data['diffractogram'] = [None for _ in range(len(data['path']))] - - # If wavelength is not manually passed it should be automatically gathered from the .xy-file + if 'wavelength' not in data.keys(): data['wavelength'] = [None for _ in range(len(data['path']))] else: @@ -75,41 +83,42 @@ def plot_diffractogram(data, options={}): if not isinstance(data['wavelength'], list): data['wavelength'] = [data['wavelength'] for _ in range(len(data['path']))] + # LOAD DIFFRACTOGRAMS for index in range(len(data['path'])): diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) data['diffractogram'][index] = diffractogram data['wavelength'][index] = wavelength - # FIXME This is a quick fix as the image is not reloaded when passing multiple beamline datasets + # FIXME This is a quick fix as the image is not reloaded when passing multiple beamline datasets. Should probably be handled in io? data['image'] = None - # Sets the xlim if this has not bee specified + + # Sets the xlim if this has not been specified if not options['xlim']: options['xlim'] = [data['diffractogram'][0][options['x_vals']].min(), data['diffractogram'][0][options['x_vals']].max()] - - # Generate heatmap data - data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'] = generate_heatmap(data=data, options=options) + + # GENERATE HEATMAP DATA + data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'], data['heatmap_yticks'], data['heatmap_yticklabels'] = generate_heatmap(data=data, options=options) options['heatmap_loaded'] = True if options['heatmap']: - options['xlim'] = options['heatmap_xlim'] + options['xlim'] = [options['heatmap_xlim'][0], options['heatmap_xlim'][1]] + # If data was already loaded, only do a check to see if the data is in a list or not, and if not, put it in one. This is because it will be looped over later. else: if not isinstance(data['diffractogram'], list): data['diffractogram'] = [data['diffractogram']] data['wavelength'] = [data['wavelength']] + ############################################################################################################################################################ + ##### INTERACTIVE SESSION ################################################################################################################################## + ############################################################################################################################################################ - if options['interactive_session_active']: - if options['offset']: - if (options['offset_x'] != options['current_offset_x']) or (options['offset_y'] != options['current_offset_y']): - for i, (diff, wl) in enumerate(zip(data['diffractogram'], data['wavelength'])): - xrd.io.apply_offset(diff, wl, i, options) - - # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not start another interactive session + # START INTERACTIVE SESSION + # Start inteactive session with ipywidgets. Disables options['interactive'] in order for the interactive loop to not recursively start new interactive sessions if options['interactive']: options['interactive'] = False options['interactive_session_active'] = True @@ -117,21 +126,36 @@ def plot_diffractogram(data, options={}): return + # If interactive mode is already enabled, update the offsets. + if options['interactive_session_active']: + if options['offset']: + if (options['offset_x'] != options['current_offset_x']) or (options['offset_y'] != options['current_offset_y']): + for i, (diff, wl) in enumerate(zip(data['diffractogram'], data['wavelength'])): + xrd.io.apply_offset(diff, wl, i, options) + + + + + ############################################################################################################################################################ + ##### PREPARE THE PLOT AND COLOURS ######################################################################################################################### + ############################################################################################################################################################ + + # CREATE AND ASSIGN AXES + # Makes a list out of reflections_data if it only passed as a dict, as it will be looped through later if options['reflections_data']: if not isinstance(options['reflections_data'], list): options['reflections_data'] = [options['reflections_data']] - - # Determine number of subplots and height ratios between them + + + # Determine the grid layout based on how many sets of reflections data has been passed if options['reflections_data'] and len(options['reflections_data']) >= 1: options = determine_grid_layout(options=options) - - # Prepare plot, and read and process data + # Create the Figure and Axes objects fig, ax = btp.prepare_plot(options=options) - - # Assign the correct axes + # Assign the correct axes to the indicies, reflections and figure itself if options['reflections_plot'] or options['reflections_indices']: if options['reflections_indices']: @@ -145,48 +169,154 @@ def plot_diffractogram(data, options={}): ax = ax[-1] - if len(data['path']) < 10: + + # GENERATE COLOURS + + # Limit for when it is assumed that each diffractogram should have its own colour - after 8, the default colour palette is used up and starts a new. + # FIXME Should probably allow for more than 8 if wanted - not a priority now + if len(data['path']) < 8: colours = btp.generate_colours(options['palettes']) + + + # Generates the colours of a list of scans to highlight is passed. options['highlight'] and options['highlight_colour'] must be of equal length. Entries in highlight can either be a list or a single number, + # if the latter it will be turned into a list with the same number as element 1 and 2. + elif options['highlight']: + # Make sure that options['highlight'] is a list + if not isinstance(options['highlight'], list): + options['highlight'] = [[options['highlight'], options['highlight']]] + + # Make sure that options['highlight_colours] is a list + if not isinstance(options['highlight_colours'], list): + options['highlight_colours'] = [options['highlight_colours']] + + colours = [] + + # Loop through each scan - assign the correct colour to each of the scan intervals in options['highlight'] + for i in range(len(data['path'])): + assigned = False + for j, highlight in enumerate(options['highlight']): + + # If one of the elements in options['highlight'] is a single number (i.e. only one scan should be highlighted), this is converted into the suitable format to be handled below + if not isinstance(highlight, list): + highlight = [highlight, highlight] + + # Assigns the j-th colour if scan number (i) is within the j-th highlight-interval + if i >= highlight[0] and i <= highlight[1]: + colours.append(options['highlight_colours'][j]) + assigned = True + + # Only assign black to i if not already been given a colour + if not assigned: + colours.append('black') + + # Reset the 'assigned' value for the next iteration + assigned = False + + # Make a itertools cycle out of the colours + colours = btp.generate_colours(colours, kind='single') + + + # If there are many scans and no highlight-options have been passed, all scans will be black else: colours = btp.generate_colours(['black'], kind='single') + + + ############################################################################################################################################################ + ##### PLOT THE DATA ######################################################################################################################################## + ############################################################################################################################################################ + + + # PLOT HEATMAP if options['heatmap']: + + ax.yaxis.set_major_locator(MultipleLocator(100)) + ax.yaxis.set_minor_locator(MultipleLocator(50)) + + # Call Seaborn to plot the data sns.heatmap(data['heatmap'], cmap=options['cmap'], cbar=False, ax=ax) + + + # Set the ticks and ticklabels to match the data point number with 2th values ax.set_xticks(data['heatmap_xticks'][options['x_vals']]) ax.set_xticklabels(data['heatmap_xticklabels'][options['x_vals']]) - ax.tick_params(axis='x', which='minor', bottom=False, top=False) + ax.set_yticks(data['heatmap_yticks']) + ax.set_yticklabels(data['heatmap_yticklabels']) + # Set the labels to the relevant values for heatmap plot + if not options['ylabel'] or options['ylabel'] == options['diff.ylabel']: + options['ylabel'] = options['heatmap.ylabel'] + if not options['yunit'] or options['yunit'] == options['diff.yunit']: + options['yunit'] = options['heatmap.yunit'] + + ax.tick_params(axis='x', which='minor', bottom=False, top=False) + ax.tick_params(axis='y', which='minor', left=False, right=False) + + options['hide_y_ticklabels'] = False + options['hide_y_ticks'] = False + + + # Toggle on the frame around the heatmap - this makes it look better together with axes ticks + for _, spine in ax.spines.items(): + spine.set_visible(True) + + + if options['highlight']: + for i, highlight in enumerate(options['highlight']): + if i < len(options['highlight'])-1 or len(options['highlight']) == 1: + ax.axhline(y=highlight[1], c=options['highlight_colours'][i], ls='--', lw=0.5) + + + # PLOT DIFFRACTOGRAM else: for diffractogram in data['diffractogram']: + + # Plot data as line plot if options['line']: diffractogram.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours), zorder=1) + # Plot data as scatter plot if options['scatter']: ax.scatter(x=diffractogram[options['x_vals']], y = diffractogram[options['y_vals']], c=[(1,1,1,0)], edgecolors=[next(colours)], linewidths=plt.rcParams['lines.markeredgewidth'], zorder=2) #, edgecolors=np.array([next(colours)])) + # Set the labels to the relevant values for diffractogram plot + if not options['ylabel'] or options['ylabel'] == options['heatmap.ylabel']: + options['ylabel'] = options['diff.ylabel'] + if not options['yunit'] or options['yunit'] == options['heatmap.yunit']: + options['yunit'] = options['diff.yunit'] + + options['hide_y_ticklabels'] = True + options['hide_y_ticks'] = True + + + # Adjust the plot to make it prettier fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + - - - # Make the reflection plots. By default, the wavelength of the first diffractogram will be used for these. + # PLOT REFLECTION TABLES if options['reflections_plot'] and options['reflections_data']: options['xlim'] = ax.get_xlim() - options['to_wavelength'] = data['wavelength'][0] + options['to_wavelength'] = data['wavelength'][0] # By default, the wavelength of the first diffractogram will be used for these. + # Plot each reflection table in the relevant axis for reflections_params, axis in zip(options['reflections_data'], ref_axes): plot_reflection_table(data=data, reflections_params=reflections_params, ax=axis, options=options) - # Print the reflection indices. By default, the wavelength of the first diffractogram will be used for this. + # Print the reflection indices. if options['reflections_indices'] and options['reflections_data']: options['xlim'] = ax.get_xlim() - options['to_wavelength'] = data['wavelength'][0] + options['to_wavelength'] = data['wavelength'][0] # By default, the wavelength of the first diffractogram will be used for this. for reflections_params in options['reflections_data']: plot_reflection_indices(data=data, reflections_params=reflections_params, ax=indices_ax, options=options) + ############################################################################################################################################################ + ##### UPDATE WIDGET ######################################################################################################################################## + ############################################################################################################################################################ + if options['interactive_session_active']: options['current_y_offset'] = options['widget'].kwargs['offset_y'] update_widgets(data=data, options=options) @@ -199,10 +329,15 @@ def plot_diffractogram(data, options={}): def generate_heatmap(data, options={}): - required_options = ['x_tick_locators'] + required_options = ['x_tick_locators', 'heatmap_y_tick_locators', 'heatmap_normalise', 'normalisation_range', 'increase_contrast'] default_options = { - 'x_tick_locators': [0.5, 0.1] + 'x_tick_locators': [0.5, 0.1], + 'heatmap_y_tick_locators': [10, 5], # Major ticks for every 10 scans, minor for every 5 + 'heatmap_normalise': False, + 'normalisation_range': None, + 'increase_contrast': False, + 'contrast_factor': 100 } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -212,13 +347,25 @@ def generate_heatmap(data, options={}): scans = [] for i, d in enumerate(data['diffractogram']): + + # Find normalisation factor + if options['heatmap_normalise'] and options['normalisation_range']: + mean_background = d['I'].loc[(d['2th'] > options['normalisation_range'][0]) & (d['2th'] < options['normalisation_range'][1])].mean() + + d['I'] = d['I'] / mean_background + + + if options['increase_contrast']: + d['I'] = d['I']**(1/options['contrast_factor']) + twotheta = np.append(twotheta, d['2th'].to_numpy()) intensities = np.append(intensities, d['I'].to_numpy()) scans = np.append(scans, np.full(len(d['2th'].to_numpy()), int(i))) - + heatmap = pd.DataFrame({'2th': twotheta, 'scan': scans, 'I': intensities}) xrd.io.translate_wavelengths(data=heatmap, wavelength=data['wavelength'][0]) + min_dict = {'2th': heatmap['2th'].min(), '2th_cuka': heatmap['2th_cuka'].min(), '2th_moka': heatmap['2th_moka'].min(), 'q': heatmap['q'].min(), 'q2': heatmap['q2'].min(), 'q4': heatmap['q4'].min(), '1/d': heatmap['1/d'].min()} @@ -235,7 +382,6 @@ def generate_heatmap(data, options={}): for xval in min_dict.keys(): # Add xticks labels - label_max = aux.floor(max_dict[xval], roundto=options['x_tick_locators'][0]) label_min = aux.ceil(min_dict[xval], roundto=options['x_tick_locators'][0]) label_steps = (label_max - label_min)/options['x_tick_locators'][0] @@ -262,7 +408,32 @@ def generate_heatmap(data, options={}): options['heatmap_xlim'] = xlims - return heatmap, xticks, xticklabels + # Get temperatures if HTXRD-scans + scan_numbers = [] + + temperatures = [] + for i, filename in enumerate(data['path']): + scan_numbers.append(i) + temperatures.append(xrd.io.read_metadata_from_xy(filename)['temperature']) + + yticks = scan_numbers[0::options['heatmap_y_tick_locators'][0]] + yticks.append(scan_numbers[-1]) + + if not temperatures[0]: + yticklabels = yticks + options['heatmap.ylabel'] = 'Scan number' + options['heatmap.yunit'] = None + + else: + yticklabels = temperatures[0::options['heatmap_y_tick_locators'][0]] + yticklabels.append(temperatures[-1]) + options['heatmap.ylabel'] = 'Temperature' + options['heatmap.yunit'] = '$^\circ$C' + + + + + return heatmap, xticks, xticklabels, yticks, yticklabels @@ -349,12 +520,12 @@ def plot_diffractogram_interactive(data, options): if options['reflections_data']: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), + x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), reflections_plot=widgets.ToggleButton(value=True), reflections_indices=widgets.ToggleButton(value=False), heatmap=widgets.ToggleButton(value=options['heatmap']), - x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], ylim=options['widgets']['ylim']['w'], offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01, description='offset_y'), @@ -363,10 +534,10 @@ def plot_diffractogram_interactive(data, options): else: w = widgets.interactive(btp.ipywidgets_update, func=widgets.fixed(plot_diffractogram), data=widgets.fixed(data), options=widgets.fixed(options), + x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), scatter=widgets.ToggleButton(value=False), line=widgets.ToggleButton(value=True), heatmap=widgets.ToggleButton(value=options['heatmap']), - x_vals=widgets.Dropdown(options=['2th', 'd', '1/d', 'q', 'q2', 'q4', '2th_cuka', '2th_moka'], value='2th', description='X-values'), xlim=options['widgets']['xlim']['w'], ylim=options['widgets']['ylim']['w'], offset_y=widgets.BoundedFloatText(value=options['offset_y'], min=-5, max=5, step=0.01, description='offset_y'), @@ -683,4 +854,3 @@ def reverse_diffractograms(diffractograms): return rev_diffractograms -#def plot_heatmap(): From 26c8813eaeb5cae5a2b620b9f6c1000545311519 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Jul 2022 17:43:44 +0200 Subject: [PATCH 223/355] Add comment I will be happy for in the future --- nafuma/xrd/plot.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 4253c9c..e3d2594 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -230,6 +230,9 @@ def plot_diffractogram(data, options={}): # PLOT HEATMAP if options['heatmap']: + # Add locators for y-axis - otherwise it will tend to break (too many ticks) when switching between diffractograms and heatmap in interactive mode. These values will be updated later anyway, and is only + # to allow the initial call to Seaborn to have values that are sensible. + # FIXME A more elegant solution to this? ax.yaxis.set_major_locator(MultipleLocator(100)) ax.yaxis.set_minor_locator(MultipleLocator(50)) From 132bc7274b33a3d416b079ce7b29a48cd0240e81 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 8 Jul 2022 17:46:15 +0200 Subject: [PATCH 224/355] Update version number --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0caa104..a0ec397 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# beamtime +# NAFUMA (v0.4) A package for processing and analysis of data from beamtime at SNBL diff --git a/setup.py b/setup.py index 59274fe..82b364d 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages setup(name='nafuma', - version='0.3', + version='0.4', description='Analysis tools for inorganic materials chemistry at the NAFUMA-group at the University of Oslo', url='https://github.com/rasmusthog/nafuma', author='Rasmus Vester Thøgersen, Halvor Høen Hval', From 649196570bff3106a9c99e0d7dfc467145af6633 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 11 Jul 2022 13:39:50 +0200 Subject: [PATCH 225/355] Add generic backup file function --- nafuma/auxillary.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 59eff5a..9879a19 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -1,6 +1,10 @@ import json import numpy as np import os +import shutil + +import time +from datetime import datetime def update_options(options, required_options, default_options): ''' Takes a dictionary of options along with a list of required options and dictionary of default options, and sets all keyval-pairs of options that is not already defined to the default values''' @@ -73,7 +77,7 @@ def floor(a, roundto=1): def write_log(message, options={}): - from datetime import datetime + required_options = ['logfile'] default_options = { @@ -111,4 +115,23 @@ def move_list_element_last(filenames,string): if string in file: del filenames[i] filenames.append(file) - return filenames \ No newline at end of file + return filenames + + + +def backup_file(filename, backup_dir): + # Creates backup-folder if it does not exist + if not os.path.isdir(backup_dir): + os.makedirs(backup_dir) + + + # Get a list of all previous backup files with the same basename as well as the creation time for the + prev_backup_files = [file for file in os.listdir(backup_dir) if os.path.basename(filename.split('.')[0]) in file] + creation_time = datetime.strptime(time.ctime(os.path.getctime(filename)), '%a %b %d %H:%M:%S %Y').strftime("%Y-%m-%d_%H-%M-%S") + ext = '.' + filename.split('.')[-1] + + dst_basename = creation_time + '_' + filename.split('.')[0] + '_' + f'{len(prev_backup_files)}'.zfill(4) + ext + dst = os.path.join(backup_dir, dst_basename) + + + shutil.copy(filename, dst) \ No newline at end of file From cf9499a988edf0afe967edeb6baf71524d828eb3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 11 Jul 2022 13:40:24 +0200 Subject: [PATCH 226/355] Initial commit of refinement submodule --- nafuma/xrd/__init__.py | 2 +- nafuma/xrd/refinement.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 nafuma/xrd/refinement.py diff --git a/nafuma/xrd/__init__.py b/nafuma/xrd/__init__.py index e0e052c..d89f20e 100644 --- a/nafuma/xrd/__init__.py +++ b/nafuma/xrd/__init__.py @@ -1 +1 @@ -from . import io, plot \ No newline at end of file +from . import io, plot, refinement \ No newline at end of file diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py new file mode 100644 index 0000000..64a98e8 --- /dev/null +++ b/nafuma/xrd/refinement.py @@ -0,0 +1,34 @@ +import os +import shutil +import time +import datetime + +import nafuma.auxillary as aux + +def make_big_inp(data: dict, options={}): + + required_options = ['output', 'overwrite', 'backup', 'backup_path'] + + default_options = { + 'output': 'big.inp', # Name of the output .INP file + 'overwrite': False, # Toggles overwrite on / off + 'backup': True, # Toggles backup on / off. Makes a backup of the file if it already exists. Only runs if overwrite is enabled. + 'backup_dir': 'backup' # Specifies the path where the backup files should be located + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + # Raises exception if files exists and overwrite is not enabled + if not options['overwrite']: + if os.path.exists(options['output']): + raise Exception(f'Overwrite disabled and file already exists: {options["output"]}') + + + # Makes a backup of file + elif options['backup'] and os.path.exists(options['output']): + aux.backup_file(filename=options['output'], backup_dir=options['backup_dir']) + + + + \ No newline at end of file From 7ff917bdcd73f56d74286b1c65d44da807f8c61f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 11 Jul 2022 17:38:13 +0200 Subject: [PATCH 227/355] Change timestamp to modified instead of creation --- nafuma/auxillary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 9879a19..96b9ce2 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -127,7 +127,7 @@ def backup_file(filename, backup_dir): # Get a list of all previous backup files with the same basename as well as the creation time for the prev_backup_files = [file for file in os.listdir(backup_dir) if os.path.basename(filename.split('.')[0]) in file] - creation_time = datetime.strptime(time.ctime(os.path.getctime(filename)), '%a %b %d %H:%M:%S %Y').strftime("%Y-%m-%d_%H-%M-%S") + creation_time = datetime.strptime(time.ctime(os.path.getmtime(filename)), '%a %b %d %H:%M:%S %Y').strftime("%Y-%m-%d_%H-%M-%S") ext = '.' + filename.split('.')[-1] dst_basename = creation_time + '_' + filename.split('.')[0] + '_' + f'{len(prev_backup_files)}'.zfill(4) + ext From 71f3940c12677e56089acb64266aba4e204d8dff Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 11 Jul 2022 17:38:40 +0200 Subject: [PATCH 228/355] Add creation of .INP-file for multiple refinements --- nafuma/xrd/refinement.py | 115 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 4 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 64a98e8..8637a2c 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -1,24 +1,45 @@ import os import shutil import time +import re import datetime +import warnings import nafuma.auxillary as aux def make_big_inp(data: dict, options={}): + ''' Generates a big .INP-file with all filenames found in data["path"]. Uses a template .INP-file (which has to be generated manually from an initial refinement in TOPAS) and appends this to a large .INP-file + while changing the filenames. ''' - required_options = ['output', 'overwrite', 'backup', 'backup_path'] + required_options = ['template', 'output', 'overwrite', 'backup', 'backup_dir', 'include', 'topas_options', 'save_dir', 'log', 'logfile'] default_options = { - 'output': 'big.inp', # Name of the output .INP file + 'template': 'start.inp', # Name of the template .INP-file + 'output': 'big.inp', # Name of the output .INP-file 'overwrite': False, # Toggles overwrite on / off 'backup': True, # Toggles backup on / off. Makes a backup of the file if it already exists. Only runs if overwrite is enabled. - 'backup_dir': 'backup' # Specifies the path where the backup files should be located + 'backup_dir': 'backup', # Specifies the path where the backup files should be located + 'include': [], + 'topas_options': { + 'bootstrap_errors': None, + 'A_matrix_memory_allowed_in_Mbytes': None, + 'approximate_A': False, + 'conserve_memory': False, + 'do_errors': False, + 'continue_after_convergence': False, + 'num_runs': None, + }, + 'save_dir': 'results', + 'log': False, + 'logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_generate_big_inp.log', } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + if not os.path.exists(options['template']): + raise Exception(f'Template file not found: {options["template"]}') + # Raises exception if files exists and overwrite is not enabled if not options['overwrite']: if os.path.exists(options['output']): @@ -27,8 +48,94 @@ def make_big_inp(data: dict, options={}): # Makes a backup of file elif options['backup'] and os.path.exists(options['output']): + + if options['log']: + aux.write_log(message=f'File {options["output"]} already exists. Creating backup in {options["backup_dir"]}.') + aux.backup_file(filename=options['output'], backup_dir=options['backup_dir']) - \ No newline at end of file + runlist = os.path.join(os.path.dirname(options['output']), 'runlist.txt') + options['include'].append(runlist) + + with open(options['template'], 'r') as template, open(options['output'], 'w', newline='\n') as output, open(runlist, 'w', newline='\n') as runlist: + + write_headers(output, options) + template = template.read() + + for i, path in enumerate(data['path']): + + s = change_labels_and_paths(template, path, i, options) + output.write(s) + + runlist.write('#define \tUSE_'+f'{i}'.zfill(4) + '\n') + + + +def write_headers(fout, options): + + for file in options['include']: + fout.write(f'#include {file} \n') + + fout.write('\n') + + for option, value in options['topas_options'].items(): + if value: + fout.write(f'{option} {value} \n') + + + +def change_labels_and_paths(template, path, i, options): + + temp_xdd = find_xdd_in_inp(options['template']) + + # Replace diffractogram-path + s = template.replace(temp_xdd, path).replace('XXXX', f'{i}'.zfill(4)) + + basename = os.path.basename(path).split(".")[0] + + # Define regular expressions for output lines + regs = [r'Out_Riet\([\S]*\)', + r'Out_CIF_STR\([\S]*\)', + r'Out_CIF_ADPs\([\S]*\)', + r'Out_CIF_Bonds_Angles\([\S]*\)', + r'Out_FCF\([\S]*\)', + r'Create_hklm_d_Th2_Ip_file\([\S]*\)', + r'out(.*?)append'] + + # Define substitute strings for output lines + subs = [f'Out_Riet({options["save_dir"]}/{basename}_riet.xy)', + f'Out_CIF_STR({options["save_dir"]}/{basename}.cif))', + f'Out_CIF_ADPs({options["save_dir"]}/{basename}.cif))', + f'Out_CIF_Bonds_Angles({options["save_dir"]}/{basename}.cif))', + f'Out_FCF({options["save_dir"]}/{basename}.fcf)', + f'Create_hklm_d_Th2_Ip_file({options["save_dir"]}/{basename}_hkl.dat)', + f'out \t {options["save_dir"]}/{basename}_refined_params.dat'] + + # Substitute strings in output lines + for reg, sub in zip(regs, subs): + s = re.sub(reg, sub, s) + + + return s + + + +def find_xdd_in_inp(path): + ''' Finds the path to the .xy / .xye scan in a given .INP-file. Assumes only one occurence of xdd and will return the last one no matter what, but outputs a UserWarning if more than one xdd is found.''' + + with open(path, 'r') as f: + lines = f.readlines() + + + xdds = 0 + for line in lines: + if 'xdd' in line: + xdd = line.split()[-1].strip('"') + xdds += 1 + + if xdds > 1: + warnings.warn(f'More than one path was found in {path}. Returning last occurence - please make sure this is what you want!') + + return xdd From d32b40409c9387e8c2193202dfe397009a02c334 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 12 Jul 2022 17:51:08 +0200 Subject: [PATCH 229/355] Add intial version of refinement function --- nafuma/xrd/refinement.py | 176 ++++++++++++++++++++++++++++++++++----- nafuma/xrd/topas.conf | 1 + 2 files changed, 156 insertions(+), 21 deletions(-) create mode 100644 nafuma/xrd/topas.conf diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 8637a2c..6a8fb1e 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -1,17 +1,22 @@ +from email.policy import default import os import shutil -import time +import subprocess import re + +import time import datetime + import warnings import nafuma.auxillary as aux + def make_big_inp(data: dict, options={}): ''' Generates a big .INP-file with all filenames found in data["path"]. Uses a template .INP-file (which has to be generated manually from an initial refinement in TOPAS) and appends this to a large .INP-file while changing the filenames. ''' - required_options = ['template', 'output', 'overwrite', 'backup', 'backup_dir', 'include', 'topas_options', 'save_dir', 'log', 'logfile'] + required_options = ['template', 'output', 'overwrite', 'backup', 'backup_dir', 'include', 'topas_options', 'save_results', 'save_dir', 'log', 'logfile'] default_options = { 'template': 'start.inp', # Name of the template .INP-file @@ -20,7 +25,19 @@ def make_big_inp(data: dict, options={}): 'backup': True, # Toggles backup on / off. Makes a backup of the file if it already exists. Only runs if overwrite is enabled. 'backup_dir': 'backup', # Specifies the path where the backup files should be located 'include': [], - 'topas_options': { + 'topas_options': None, + 'save_results': True, + 'save_dir': 'results', + 'log': False, + 'logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_generate_big_inp.log', + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + required_topas_options = ['iters', 'chi2_convergence_criteria', 'bootstrap_errors', 'A_matrix_memory_allowed_in_Mbytes', 'approximate_A', 'conserve_memory', 'do_errors', 'continue_after_convergence', 'num_runs'] + default_topas_options = { + 'iters': 100000, + 'chi2_convergence_criteria': 0.001, 'bootstrap_errors': None, 'A_matrix_memory_allowed_in_Mbytes': None, 'approximate_A': False, @@ -28,13 +45,9 @@ def make_big_inp(data: dict, options={}): 'do_errors': False, 'continue_after_convergence': False, 'num_runs': None, - }, - 'save_dir': 'results', - 'log': False, - 'logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_generate_big_inp.log', - } + } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options['topas_options'] = aux.update_options(options=options['topas_options'], required_options=required_topas_options, default_options=default_topas_options) if not os.path.exists(options['template']): @@ -55,45 +68,101 @@ def make_big_inp(data: dict, options={}): aux.backup_file(filename=options['output'], backup_dir=options['backup_dir']) - runlist = os.path.join(os.path.dirname(options['output']), 'runlist.txt') options['include'].append(runlist) with open(options['template'], 'r') as template, open(options['output'], 'w', newline='\n') as output, open(runlist, 'w', newline='\n') as runlist: - + write_headers(output, options) + template = template.read() for i, path in enumerate(data['path']): - s = change_labels_and_paths(template, path, i, options) + s = make_inp_entry(template=template, xdd=path, num=i, options=options) output.write(s) + output.write('\n\n') runlist.write('#define \tUSE_'+f'{i}'.zfill(4) + '\n') def write_headers(fout, options): + # FIXME Could modify this to make sure that certain combinations of options is not written, such as both do_errors and bootstrap_errors. for file in options['include']: fout.write(f'#include {file} \n') fout.write('\n') + if options['save_results']: + fout.write('#define output \n') + + fout.write('\n') + for option, value in options['topas_options'].items(): if value: - fout.write(f'{option} {value} \n') + if isinstance(value, bool): + fout.write(f'{option} \n') + else: + fout.write(f'{option} {value} \n') +def get_headers(inp): -def change_labels_and_paths(template, path, i, options): + with open(inp, 'r') as inp: + headers = [] + + line = inp.readline() + + while not all(keyword in line for keyword in ['out', 'append']): + line = inp.readline() + + # Jump down to lines + line = inp.readline() + line = inp.readline() + + while not 'Out_String' in line: + + if line.split(): + header = line.split()[1] + if all(keyword in header for keyword in ['Get', '(', ')']): + header = header[4:-1] + + headers.append(header) + + line = inp.readline() + + + return headers + + +def get_paths(inp): + + paths = [] + + with open(inp, 'r') as inp: + lines = inp.readlines() + + for line in lines: + if all(keyword in line for keyword in ['out', 'append']): + paths.append(line.split()[1]) + + + return paths + +def make_inp_entry(template: str, xdd: str, num: int, options: dict) -> str: + ''' Takes a template and creates an entry with xdd as path to file and with number num.''' temp_xdd = find_xdd_in_inp(options['template']) + + + num_str = f'{num}'.zfill(4) # Replace diffractogram-path - s = template.replace(temp_xdd, path).replace('XXXX', f'{i}'.zfill(4)) + s = template.replace(temp_xdd, xdd).replace('XXXX', num_str) - basename = os.path.basename(path).split(".")[0] + basename = os.path.basename(xdd).split(".")[0] # Define regular expressions for output lines regs = [r'Out_Riet\([\S]*\)', @@ -106,23 +175,24 @@ def change_labels_and_paths(template, path, i, options): # Define substitute strings for output lines subs = [f'Out_Riet({options["save_dir"]}/{basename}_riet.xy)', - f'Out_CIF_STR({options["save_dir"]}/{basename}.cif))', - f'Out_CIF_ADPs({options["save_dir"]}/{basename}.cif))', - f'Out_CIF_Bonds_Angles({options["save_dir"]}/{basename}.cif))', + f'Out_CIF_STR({options["save_dir"]}/{basename}.cif)', + f'Out_CIF_ADPs({options["save_dir"]}/{basename}.cif)', + f'Out_CIF_Bonds_Angles({options["save_dir"]}/{basename}.cif)', f'Out_FCF({options["save_dir"]}/{basename}.fcf)', f'Create_hklm_d_Th2_Ip_file({options["save_dir"]}/{basename}_hkl.dat)', - f'out \t {options["save_dir"]}/{basename}_refined_params.dat'] + f'out \t {options["save_dir"]}/{basename}_refined_params.dat \t append'] # Substitute strings in output lines for reg, sub in zip(regs, subs): s = re.sub(reg, sub, s) + return s -def find_xdd_in_inp(path): +def find_xdd_in_inp(path: str) -> str: ''' Finds the path to the .xy / .xye scan in a given .INP-file. Assumes only one occurence of xdd and will return the last one no matter what, but outputs a UserWarning if more than one xdd is found.''' with open(path, 'r') as f: @@ -139,3 +209,67 @@ def find_xdd_in_inp(path): warnings.warn(f'More than one path was found in {path}. Returning last occurence - please make sure this is what you want!') return xdd + + +def refine(data: dict, options={}): + ''' Calls TOPAS from options['topas_path'], which should point to tc.exe in the TOPAS-folder. If not explicitly passed, will try to use what is in topas.conf.''' + + required_options = ['topas_path', 'topas_log', 'topas_logfile', 'log', 'logfile', 'overwrite'] + + default_options = { + 'topas_path': None, + 'topas_log': False, + 'topas_logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_topas.out', + 'log': False, + 'logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_generate_big_inp.log', + 'overwrite': False + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + if not options['topas_path']: + import nafuma.xrd as xrd + + # Open topas.conf in the nafuma/xrd + with open(os.path.join(os.path.dirname(xrd.__file__), 'topas.conf'), 'r') as f: + topas_base = f.read() + + options['topas_path'] = os.path.join(topas_base, 'tc.exe') + + + # Check to see if the executable exists + if not os.path.exists(options['topas_path']): + raise Exception('TOPAS executable not found! Please explicitly pass path to tc.exe directly in options["topas_path"] or change base folder in topas.conf to the correct one.') + + + # Create folders if they don't exist + paths, headers = get_paths(data['inp']), get_headers(data['inp']) + + for path in paths: + dirname = os.path.dirname(path) + + if dirname and not os.path.isdir(dirname): + os.makedirs(dirname) + + if not os.path.exists(path) or options['overwrite']: + with open(path, 'w') as results: + for header in headers: + results.write(header+'\t') + + results.write('\n') + else: + raise Exception(f'Results file already exists: {path}') + + + # Create shell command + command = ' '.join([options['topas_path'], data['inp']]) + + # Append output if logging is enabled + if options['topas_log']: + command = ' '.join([command, f'>{options["topas_logfile"]}']) + + if os.path.dirname(options['topas_logfile']) and not os.path.isdir(os.path.dirname(options['topas_logfile'])): + os.makedirs(os.path.dirname(options['topas_logfile'])) + + + subprocess.call(command, shell=True) \ No newline at end of file diff --git a/nafuma/xrd/topas.conf b/nafuma/xrd/topas.conf new file mode 100644 index 0000000..3c6612a --- /dev/null +++ b/nafuma/xrd/topas.conf @@ -0,0 +1 @@ +C:/TOPAS6/ \ No newline at end of file From 51e29284726cc9b4f8a1af079ee328d7a800e8ac Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 13 Jul 2022 17:26:29 +0200 Subject: [PATCH 230/355] Add cif-reader and inital INP-generator --- nafuma/xrd/refinement.py | 270 +++++++++++++++++++++++++++++++++- nafuma/xrd/snippets.json | 14 ++ nafuma/xrd/topas_default.json | 15 ++ 3 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 nafuma/xrd/snippets.json create mode 100644 nafuma/xrd/topas_default.json diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 6a8fb1e..94f20fc 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -6,12 +6,266 @@ import re import time import datetime - import warnings +import json + +from ase import io import nafuma.auxillary as aux + +def make_initial_inp(data: dict, options={}): + + required_options = ['filename', 'overwrite', 'include', 'save_results', 'topas_options', 'background', 'capillary', 'start', 'finish', 'exclude'] + + default_options = { + 'filename': 'start.inp', + 'overwrite': False, + 'save_results': False, + 'include': [], # Any files that should be included with #include + 'topas_options': {}, + 'background': 7, + 'capillary': False, + 'interval': [None, None], # Start and finish values that TOPAS should refine on. Overrides 'start' and 'finish' + 'start': None, # Start value only. Overridden by 'interval' if this is set. + 'finish': None, # Finish value only. Overridden by 'interval' if this is set. + 'exlude': [] # Excluded regions. List of lists. + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options['topas_options'] = update_topas_options(options=options) + + + if not os.path.exists(options['filename']) or options['overwrite']: + with open(options['filename'], 'w') as fout: + write_headers(fout=fout, options=options) + + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write('rw_p 0 r_exp 0 r_p 0 r_wp_dash 0 r_p_dash 0 r_exp_dash 0 weighted_Durbin_Watson 0 gof 0') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + + write_xdd(fout=fout, data=data, options=options) + + for str in data['str']: + fout.write('\n\n') + write_str(fout=fout, str=str, options=options) + + + +def write_xdd(fout, data, options): + import nafuma.xrd as xrd + basedir = os.path.dirname(xrd.__file__) + + with open (os.path.join(basedir, 'snippets.json'), 'r') as snip: + snippets = json.load(snip) + + topas_options = options['topas_options'] + + # Write initial parameters + fout.write(f'xdd "{data["xdd"]}"\n') + fout.write('\t'+snippets['calculation_step'].format(topas_options['convolution_step'])+'\n') + + # Write background + fout.write('\tbkg @ ') + for i in range(options['background']): + fout.write('0 ') + + # Write wavelength and LP-factor + fout.write('\n\n') + + fout.write('\t'+snippets['wavelength'].format(data['wavelength'])+'\n') + fout.write('\t'+snippets['lp_factor'].format(topas_options['lp_factor'])+'\n') + + if options['capillary']: + fout.write('\n') + for i, line in enumerate(snippets['capillary']): + if i == 0: + line.format(topas_options['packing_density']) + if i == 1: + line.format(topas_options['capdia']) + + fout.write('\t'+line+'\n') + + + fout.write('\n') + if options['interval'][0] or options['start']: + options['start'] = options['interval'][0] if options['interval'][0] else options['start'] + fout.write(f'\tstart_X {options["start"]}\n') + + if options['interval'][1] or options['finish']: + options['finish'] = options['interval'][1] if options['interval'][1] else options['finish'] + fout.write(f'\tfinish_X {options["finish"]}\n') + + if options['exclude']: + for exclude in options['exclude']: + fout.write(f'\texclude {exclude[0]} {exclude[1]}\n') + + + +def write_str(fout, str, options): + + atoms = read_cif(str) + print(atoms["atoms"]["Fe1"].keys()) + + a = float(atoms['_cell_length_a'].split('(')[0]) + b = float(atoms['_cell_length_b'].split('(')[0]) + c = float(atoms['_cell_length_c'].split('(')[0]) + alpha = float(atoms['_cell_angle_alpha'].split('(')[0]) + beta = float(atoms['_cell_angle_beta'].split('(')[0]) + gamma = float(atoms['_cell_angle_gamma'].split('(')[0]) + + + fout.write('#ifdef start_values\n') + fout.write(f'local lpa \t{a} \t;:\n') + fout.write(f'local lpb \t{b} \t;:\n') + fout.write(f'local lpc \t{c} \t;:\n') + fout.write(f'local lpal \t{alpha} \t;:\n') + fout.write(f'local lpbe \t{beta} \t;:\n') + fout.write(f'local lpga \t{gamma} \t;:\n\n') + + fout.write('#else\n') + fout.write(f'local lpa \t{a}\n') + fout.write(f'local lpb \t{b}\n') + fout.write(f'local lpc \t{c}\n') + fout.write(f'local lpal \t{alpha}\n') + fout.write(f'local lpbe \t{beta}\n') + fout.write(f'local lpga \t{gamma}\n') + fout.write('#endif\n\n') + + sites = list(atoms['atoms'].keys()) + + attrs = {'_atom_site_fract_x': 'x', '_atom_site_fract_y': 'y', '_atom_site_fract_z': 'z', '_atom_site_B_iso_or_equiv': 'beq'} + + for attr in attrs: + for site in sites: + if attr in atoms["atoms"][site].keys(): + value = atoms["atoms"][site][attr].split("(")[0] + value = value if value != '.' else 0. + + fout.write(f'local {attrs[attr]}_{site}\t\t =\t {value} \t ;= \n') + + + fout.write('\n') + + + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write(atoms['_chemical_name_common'] + f'({atoms["_space_group_name_H-M_alt"]})') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + + + fout.write('\tstr\n') + fout.write(f'\t\tphase_name "{atoms["_chemical_name_common"]} ({atoms["_space_group_name_H-M_alt"]})"\n') + fout.write(f'\t\tspace_group {atoms["_space_group_IT_number"]}\n') + fout.write('\t\ta = lpa ;\n') + fout.write('\t\tb = lpb ;\n') + fout.write('\t\tc = lpc ;\n') + fout.write('\t\tal = lpal ;\n') + fout.write('\t\tbe = lpbe ;\n') + fout.write('\t\tga = lpga ;\n') + fout.write('\n') + fout.write(f'\t\tcell_volume vol_XXXX {atoms["_cell_volume"]}\n') + + for atom in atoms['atoms'].keys(): + atom_label = atom if len(atom) == 3 else atom+' ' + + fout.write(f'\t\tsite {atom_label}\t x = x_{atom_label} ;\t y = y_{atom_label} ;\t z = z_{atom_label} ; occ {atoms["atoms"][atom]["_atom_site_occupancy"]}\t beq = beq_{atom_label} \t ;\n') + + + fout.write('\n') + fout.write('\t\tscale @ 0.0') + fout.write('\t\t r_bragg 1.0') + + + + + +def read_cif(path): + + data = {'atoms': {}} # Initialise dictionary + read = True # Initialise read toggle + + # Lists attributes to get out of the .CIF-file. This will correspond to what VESTA writes out, not necessarily what you will find in ICSD + attrs = ['_chemical_name_common', '_cell', '_space_group_name_H-M_alt', '_space_group_IT_number'] + + # Open file + with open(path, 'r') as cif: + line = cif.readline() + + # Read until encountering #End + while read: + + # Break loop if #End is reached + if not line or line.startswith('#End'): + read = False + break + + # Handle loops + if line.lstrip().startswith('loop_'): + loop = [] + line = cif.readline() + + # Only handle loops with attributes that starts with _atom. Other loops are incompatible with below code due to slight differences in formatting (e.g. lineshifts) + while line.lstrip().startswith('_atom'): + loop.append(line) # add the attributes of the loop to a list (every keywords starting with _atom) + line = cif.readline() # Read next line + + + # If there were any attributes that started with _atom - if this is empty is just means there were other attributes in this loop + if loop: + # Read every line after the attribute listing has ended - until it encounters a new attribute, loop or #End tag + # FIXME WHat a horrible condition statement - need to fix this! + while line and not line.lstrip().startswith('_') and not line.lstrip().startswith('loop_') and not line.lstrip().startswith('#End') and not line=='\n': + # Initialise empty dictionary for a given atom if it has not already been created in another loop + if line.split()[0] not in data['atoms'].keys(): + data["atoms"][line.split()[0]] = {} + + # Add all the attribute / value pairs for the current loop + for i, attr in enumerate(loop): + data["atoms"][line.split()[0]][attr[:-1].lstrip()] = line.split()[i] + + # Read new line + line = cif.readline() + + # If loop list was empty, keep going to the next line + else: + line = cif.readline() + + + + # Handle free-standing attributes + elif any([line.lstrip().startswith(i) for i in attrs]): + #line.startswith() or line.startswith('_symmetry_space_group_name') or line.startswith('_symmetry_Int_Tables'): + # + # + attr, *value = line.split() + + value = ' '.join([str(i) for i in value]) + + data[attr] = value + line = cif.readline() + + else: + line = cif.readline() + + + + return data + + + + +def update_topas_options(options): + import nafuma.xrd as xrd + basedir = os.path.dirname(xrd.__file__) + + with open(os.path.join(basedir, 'topas_default.json'), 'r') as default: + topas_defaults = json.load(default) + + options['topas_options'] = aux.update_options(options=options['topas_options'], required_options=topas_defaults.keys(), default_options=topas_defaults) + + return options['topas_options'] + def make_big_inp(data: dict, options={}): ''' Generates a big .INP-file with all filenames found in data["path"]. Uses a template .INP-file (which has to be generated manually from an initial refinement in TOPAS) and appends this to a large .INP-file while changing the filenames. ''' @@ -90,6 +344,18 @@ def make_big_inp(data: dict, options={}): def write_headers(fout, options): # FIXME Could modify this to make sure that certain combinations of options is not written, such as both do_errors and bootstrap_errors. + headers = [ + "A_matrix_memory_allowed_in_Mbytes", + "approximate_A", + "bootstrap_errors", + "conserve_memory", + "continue_after_convergence", + "do_errors", + "chi2_convergence_criteria", + "iters", + "num_runs" ] + + for file in options['include']: fout.write(f'#include {file} \n') @@ -101,7 +367,7 @@ def write_headers(fout, options): fout.write('\n') for option, value in options['topas_options'].items(): - if value: + if value and option in headers: if isinstance(value, bool): fout.write(f'{option} \n') else: diff --git a/nafuma/xrd/snippets.json b/nafuma/xrd/snippets.json new file mode 100644 index 0000000..8636a1d --- /dev/null +++ b/nafuma/xrd/snippets.json @@ -0,0 +1,14 @@ +{ + "calculation_step": "x_calculation_step = Yobs_dx_at(Xo); convolution_step {}", + "capillary": [ + "local !packing_density {} min 0.1 max 1.0 'typically 0.2 to 0.5", + "local !capdia {} 'capillary diameter in mm", + "local !linab = Get(mixture_MAC) Get(mixture_density_g_on_cm3);: 'in cm-1", + "local muR = (capdia/20)*linab*packing_density;", + "Cylindrical_I_Correction(muR)" + ], + + "lp_factor": "LP_Factor({}) 'change the LP correction or lh value if required", + "wavelength": "lam ymin_on_ymax 0.0001 la 1.0 lo {} lh 0.1", + "zero_error": "Zero_Error(zero, 0)" +} \ No newline at end of file diff --git a/nafuma/xrd/topas_default.json b/nafuma/xrd/topas_default.json new file mode 100644 index 0000000..4ab6a81 --- /dev/null +++ b/nafuma/xrd/topas_default.json @@ -0,0 +1,15 @@ +{ + "A_matrix_memory_allowed_in_Mbytes": null, + "approximate_A": false, + "bootstrap_errors": null, + "capdia": 0.5, + "chi2_convergence_criteria": 0.001, + "conserve_memory": false, + "continue_after_convergence": false, + "convolution_step": 4, + "do_errors": false, + "iters": 100000, + "lp_factor": 90, + "num_runs": null, + "packing_density": 0.5 +} \ No newline at end of file From ace44cb48f2743fd9ae3778c498d539d61705f8a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 14 Jul 2022 19:15:39 +0200 Subject: [PATCH 231/355] Add initial INP-generation --- nafuma/xrd/refinement.py | 291 +++++++++++++++++++++++++++++----- nafuma/xrd/snippets.json | 2 +- nafuma/xrd/topas_default.json | 2 +- 3 files changed, 251 insertions(+), 44 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 94f20fc..d250172 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -17,12 +17,13 @@ import nafuma.auxillary as aux def make_initial_inp(data: dict, options={}): - required_options = ['filename', 'overwrite', 'include', 'save_results', 'topas_options', 'background', 'capillary', 'start', 'finish', 'exclude'] + required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'topas_options', 'background', 'capillary', 'start', 'finish', 'exclude'] default_options = { 'filename': 'start.inp', 'overwrite': False, 'save_results': False, + 'save_dir': 'results/', 'include': [], # Any files that should be included with #include 'topas_options': {}, 'background': 7, @@ -41,15 +42,34 @@ def make_initial_inp(data: dict, options={}): with open(options['filename'], 'w') as fout: write_headers(fout=fout, options=options) + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') - fout.write('rw_p 0 r_exp 0 r_p 0 r_wp_dash 0 r_p_dash 0 r_exp_dash 0 weighted_Durbin_Watson 0 gof 0') + fout.write('\'START OF INP - XXXX') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write('r_wp 0 r_exp 0 r_p 0 r_wp_dash 0 r_p_dash 0 r_exp_dash 0 weighted_Durbin_Watson 0 gof 0') fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') write_xdd(fout=fout, data=data, options=options) - for str in data['str']: + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write('\'PARAMETER DEFINITONS') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + + for i, str in enumerate(data['str']): fout.write('\n\n') - write_str(fout=fout, str=str, options=options) + write_params(fout=fout, data=data, options=options, index=i) + + + + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write('\'STR DEFINITIONS') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + + for i, str in enumerate(data['str']): + fout.write('\n\n') + write_str(fout=fout, data=data, options=options, index=i) + @@ -66,6 +86,19 @@ def write_xdd(fout, data, options): fout.write(f'xdd "{data["xdd"]}"\n') fout.write('\t'+snippets['calculation_step'].format(topas_options['convolution_step'])+'\n') + for include in options['include']: + if 'peak_width' in include: + with open(include, 'r') as f: + lines = f.readlines() + peak_width_params = [] + for line in lines: + param = line.split()[1][1:] + peak_width_params.append(param) + + + + fout.write('\t'+snippets['gauss_fwhm'].format(peak_width_params[0], peak_width_params[1], peak_width_params[2])+'\n') + # Write background fout.write('\tbkg @ ') for i in range(options['background']): @@ -76,14 +109,15 @@ def write_xdd(fout, data, options): fout.write('\t'+snippets['wavelength'].format(data['wavelength'])+'\n') fout.write('\t'+snippets['lp_factor'].format(topas_options['lp_factor'])+'\n') + fout.write('\t'+snippets['zero_error']+'\n') if options['capillary']: fout.write('\n') for i, line in enumerate(snippets['capillary']): if i == 0: - line.format(topas_options['packing_density']) + line = line.format(topas_options['packing_density']) if i == 1: - line.format(topas_options['capdia']) + line = line.format(topas_options['capdia']) fout.write('\t'+line+'\n') @@ -103,10 +137,19 @@ def write_xdd(fout, data, options): -def write_str(fout, str, options): - - atoms = read_cif(str) - print(atoms["atoms"]["Fe1"].keys()) + +def write_params(fout, data, options, index=0): + + atoms = read_cif(data['str'][index]) + if 'labels' in data.keys(): + label = data['labels'][index] + else: + label = index + + + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write(atoms['_chemical_name_common'] + f'({atoms["_space_group_name_H-M_alt"]}) - Parameters') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') a = float(atoms['_cell_length_a'].split('(')[0]) b = float(atoms['_cell_length_b'].split('(')[0]) @@ -116,67 +159,231 @@ def write_str(fout, str, options): gamma = float(atoms['_cell_angle_gamma'].split('(')[0]) - fout.write('#ifdef start_values\n') - fout.write(f'local lpa \t{a} \t;:\n') - fout.write(f'local lpb \t{b} \t;:\n') - fout.write(f'local lpc \t{c} \t;:\n') - fout.write(f'local lpal \t{alpha} \t;:\n') - fout.write(f'local lpbe \t{beta} \t;:\n') - fout.write(f'local lpga \t{gamma} \t;:\n\n') + # WRITE LATTICE PARAMETERS + # If start_values is defined: + fout.write(f'#ifdef start_values_{label}\n') + lpa = f'local lpa_{label} {a} ;: {a}' + lpb = f'local lpb_{label} {b} ;: {b}' + lpc = f'local lpc_{label} {c} ;: {c}' + fout.write('{: <55} {: <55} {: <55}\n'.format(lpa, lpb, lpc)) + lpal = f'local lpal_{label} {alpha} ;: {alpha}' + lpbe = f'local lpbe_{label} {beta} ;: {beta}' + lpga = f'local lpga_{label} {gamma} ;: {gamma}' + fout.write('{: <55} {: <55} {: <55}\n'.format(lpal, lpbe, lpga)) + # Otherwise + fout.write('\n') fout.write('#else\n') - fout.write(f'local lpa \t{a}\n') - fout.write(f'local lpb \t{b}\n') - fout.write(f'local lpc \t{c}\n') - fout.write(f'local lpal \t{alpha}\n') - fout.write(f'local lpbe \t{beta}\n') - fout.write(f'local lpga \t{gamma}\n') + lpa = f'local lpa_{label} {a}' + lpb = f'local lpb_{label} {b}' + lpc = f'local lpc_{label} {c}' + fout.write('{: <55} {: <55} {: <55}\n'.format(lpa, lpb, lpc)) + lpal = f'local lpal_{label} {alpha}' + lpbe = f'local lpbe_{label} {beta}' + lpga = f'local lpga_{label} {gamma}' + fout.write('{: <55} {: <55} {: <55}\n'.format(lpal, lpbe, lpga)) fout.write('#endif\n\n') sites = list(atoms['atoms'].keys()) - attrs = {'_atom_site_fract_x': 'x', '_atom_site_fract_y': 'y', '_atom_site_fract_z': 'z', '_atom_site_B_iso_or_equiv': 'beq'} + attrs = { + '_atom_site_fract_x': 'x', + '_atom_site_fract_y': 'y', + '_atom_site_fract_z': 'z', + '_atom_site_occupancy': '!occ', + '_atom_site_B_iso_or_equiv': '!beq' + } - for attr in attrs: - for site in sites: + + # WRITE SITE PARAMETERS + for site in sites: + + params = [] + for attr in attrs: if attr in atoms["atoms"][site].keys(): value = atoms["atoms"][site][attr].split("(")[0] value = value if value != '.' else 0. - fout.write(f'local {attrs[attr]}_{site}\t\t =\t {value} \t ;= \n') + params.append('{: <20} {: <20}'.format(f'local {attrs[attr]}_{site}_{label}', f' = {value} ;: {value}')) + #fout.write(f'local {attrs[attr]}_{site}_{label}\t\t =\t {value} \t ;= \t\t\t\t\t') + + fout.write('{: <55} {: <55} {: <55} {: <55} {: <55}\n'.format(*params)) + + fout.write('\n') + + fout.write('{: <55} {: <55} {: <55} {: <55}\n'.format( + f'local csgc_{label}_XXXX = 200 ;: 200', + f'local cslc_{label}_XXXX = 200 ;: 200', + f'local sgc_{label}_XXXX = 0 ;: 0', + f'local slc_{label}_XXXX = 0 ;: 0', + )) - fout.write('\n') + # fout.write(f'local csgc_{label} ;:\t\t\t\t\t') + # fout.write(f'local cslc_{label} ;:\t\t\t\t\t') + # fout.write(f'local sgc_{label} ;:\t\t\t\t\t') + # fout.write(f'local slc_{label} ;:\n') + + +def write_str(fout, data, options, index=0): + + atoms = read_cif(data['str'][index]) + if 'labels' in data.keys(): + label = data['labels'][index] + else: + label = index + + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') - fout.write(atoms['_chemical_name_common'] + f'({atoms["_space_group_name_H-M_alt"]})') + fout.write('\'' + atoms['_chemical_name_common'].strip('\'') + ' ' + f'({atoms["_space_group_name_H-M_alt"]})'.replace("\'", "")) fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') fout.write('\tstr\n') - fout.write(f'\t\tphase_name "{atoms["_chemical_name_common"]} ({atoms["_space_group_name_H-M_alt"]})"\n') + fout.write(f'\t\tphase_name "{atoms["_chemical_name_common"]} ({atoms["_space_group_name_H-M_alt"]})"\n'.replace('\'', '')) fout.write(f'\t\tspace_group {atoms["_space_group_IT_number"]}\n') - fout.write('\t\ta = lpa ;\n') - fout.write('\t\tb = lpb ;\n') - fout.write('\t\tc = lpc ;\n') - fout.write('\t\tal = lpal ;\n') - fout.write('\t\tbe = lpbe ;\n') - fout.write('\t\tga = lpga ;\n') + fout.write(f'\t\ta = lpa_{label} ;\n') + fout.write(f'\t\tb = lpb_{label} ;\n') + fout.write(f'\t\tc = lpc_{label} ;\n') + fout.write(f'\t\tal = lpal_{label} ;\n') + fout.write(f'\t\tbe = lpbe_{label} ;\n') + fout.write(f'\t\tga = lpga_{label} ;\n') fout.write('\n') - fout.write(f'\t\tcell_volume vol_XXXX {atoms["_cell_volume"]}\n') + fout.write(f'\t\tcell_volume\t vol_{label}_XXXX {atoms["_cell_volume"]}\n') + fout.write(f'\t\tcell_mass\t mass_{label}_XXXX 1\n') + fout.write(f'\t\tweight_percent\t wp_{label}_XXXX 100\n\n') + fout.write('\n') + fout.write('\t\tscale @ 1.0\n') + fout.write('\t\tr_bragg 1.0\n\n') + + fout.write(f'\t\t#ifdef crystallite_size_gaussian_{label}\n') + fout.write(f'\t\tCS_G(csgc_{label}_XXXX)\n') + fout.write('\t\t#endif\n') + fout.write(f'\t\t#ifdef crystallite_size_lorentzian_{label}\n') + fout.write(f'\t\tCS_L(cslc_{label}_XXXX)\n') + fout.write('\t\t#endif\n\n') + + fout.write(f'\t\t#ifdef strain_gaussian_{label}\n') + fout.write(f'\t\tStrain_G(sgc_{label}_XXXX)\n') + fout.write('\t\t#endif\n') + fout.write(f'\t\t#ifdef strain_lorentzian_{label}\n') + fout.write(f'\t\tStrain_L(slc_{label}_XXXX)\n') + fout.write('\t\t#endif\n\n') + + + + for atom in atoms['atoms'].keys(): - atom_label = atom if len(atom) == 3 else atom+' ' - fout.write(f'\t\tsite {atom_label}\t x = x_{atom_label} ;\t y = y_{atom_label} ;\t z = z_{atom_label} ; occ {atoms["atoms"][atom]["_atom_site_occupancy"]}\t beq = beq_{atom_label} \t ;\n') + specie = '{}{}'.format( + atoms["atoms"][atom]["_atom_site_type_symbol"], + data['oxidation_states'][index][atoms["atoms"][atom]["_atom_site_type_symbol"]] + ) + site = f'site {atom}' + x = f'\t\tx = x_{atom}_{label}' + y = f'\t\ty = y_{atom}_{label}' + z = f'\t\tz = z_{atom}_{label}' + occ = '\t\tocc {: <4} = occ_{}_{}'.format(specie, atom, label) + #occ = f'\t\tocc {atoms["atoms"][atom]["_atom_site_type_symbol"]} occ_{atom}_{label}' + beq = f'\t\tbeq = beq_{atom}_{label}' + + # FIXME Fix alignment here at some point + #fout.write(f'\t\tsite {atom}\t x = x_{atom}_{label} ;\t y = y_{atom}_{label} ;\t z = z_{atom}_{label} ;\t occ {atoms["atoms"][atom]["_atom_site_type_symbol"]} occ_{atom}_{label} \t beq = beq_{atom}_{label} \t ;\n') + fout.write('\t\t{: <9} {: <30}; {: <30}; {: <30}; {: <30}; {: <30};'.format(site, x, y, z, occ, beq)) + fout.write('\n') + + + + if options['save_results']: + fout.write('\n\n') + write_output(fout=fout, data=data, options=options, index=index) + + + +def write_output(fout, data, options, index=0): + + + filename = os.path.basename(data['xdd']).split('.')[0] + + atoms = read_cif(data['str'][index]) + + if 'labels' in data.keys(): + label = data['labels'][index] + else: + label = index + + + fout.write('#ifdef output\n') + fout.write(f'\t\tOut_Riet({options["save_dir"]}/{label}_XXXX_riet.xy)\n') + fout.write(f'\t\tOut_CIF_STR({options["save_dir"]}/{label}_XXXX.cif)\n') + fout.write(f'\t\tOut_CIF_ADPs({options["save_dir"]}/{label}_XXXX.cif)\n') + fout.write(f'\t\tOut_CIF_Bonds_Angles({options["save_dir"]}/{label}_XXXX.cif)\n') + fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file({options["save_dir"]}/{label}_XXXX_hkl.dat)\n') fout.write('\n') - fout.write('\t\tscale @ 0.0') - fout.write('\t\t r_bragg 1.0') - + fout.write(f'out {options["save_dir"]}/{filename}_{label}.dat append\n') + fout.write(f'\t\tOut_String("XXXX")\n') + + fout.write('\t\t{: <40} {: <40} {: <40} {: <40} {: <40} {: <40}'.format( + f'Out(Get(r_wp), "%11.5f")', + f'Out(Get(r_exp), "%11.5f")', + f'Out(Get(r_p), "%11.5f")', + f'Out(Get(r_p_dash), "%11.5f")', + f'Out(Get(r_exp_dash), "%11.5f")', + f'Out(Get(gof), "%11.5f")', + ) + ) + + fout.write('\n') + + + fout.write('\t\t{: <40} {: <40} {: <40}'.format( + f'Out(vol_{label}_XXXX, "%11.5f")', + f'Out(mass_{label}_XXXX, "%11.5f")', + f'Out(wp_{label}_XXXX, "%11.5f")', + + ) + ) + + + fout.write('\n') + + + fout.write('\t\t{: <40} {: <40} {: <40} {: <40} {: <40} {: <40}'.format( + f'Out(lpa_{label}, "%11.5f")', + f'Out(lpb_{label}, "%11.5f")', + f'Out(lpc_{label}, "%11.5f")', + f'Out(lpal_{label}, "%11.5f")', + f'Out(lpbe_{label}, "%11.5f")', + f'Out(lpga_{label}, "%11.5f")', + + ) + ) + + fout.write('\n\n') + + for atom in atoms['atoms']: + fout.write('\t\t{: <40} {: <40} {: <40} {: <40} {: <40}'.format( + f'Out(x_{atom}_{label}, "%11.5f")', + f'Out(y_{atom}_{label}, "%11.5f")', + f'Out(z_{atom}_{label}, "%11.5f")', + f'Out(occ_{atom}_{label}, "%11.5f")', + f'Out(beq_{atom}_{label}, "%11.5f")', + ) + ) + + fout.write('\n') + + fout.write('\n\n') + + + + fout.write('#endif') diff --git a/nafuma/xrd/snippets.json b/nafuma/xrd/snippets.json index 8636a1d..5b0f97d 100644 --- a/nafuma/xrd/snippets.json +++ b/nafuma/xrd/snippets.json @@ -7,7 +7,7 @@ "local muR = (capdia/20)*linab*packing_density;", "Cylindrical_I_Correction(muR)" ], - + "gauss_fwhm": "gauss_fwhm = Sqrt({} Cos(2 * Th)^4 + {} Cos(2 * Th)^2 + {});", "lp_factor": "LP_Factor({}) 'change the LP correction or lh value if required", "wavelength": "lam ymin_on_ymax 0.0001 la 1.0 lo {} lh 0.1", "zero_error": "Zero_Error(zero, 0)" diff --git a/nafuma/xrd/topas_default.json b/nafuma/xrd/topas_default.json index 4ab6a81..c92e8a4 100644 --- a/nafuma/xrd/topas_default.json +++ b/nafuma/xrd/topas_default.json @@ -6,7 +6,7 @@ "chi2_convergence_criteria": 0.001, "conserve_memory": false, "continue_after_convergence": false, - "convolution_step": 4, + "convolution_step": 1, "do_errors": false, "iters": 100000, "lp_factor": 90, From 92075fbb66f3a48fa4f6e05e3f1563bd3dde83cb Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 14 Jul 2022 19:47:36 +0200 Subject: [PATCH 232/355] Fix filenames in make_big_inp and add fixmes --- nafuma/xrd/refinement.py | 67 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index d250172..57bfa72 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -9,8 +9,6 @@ import datetime import warnings import json -from ase import io - import nafuma.auxillary as aux @@ -329,6 +327,8 @@ def write_output(fout, data, options, index=0): fout.write(f'out {options["save_dir"]}/{filename}_{label}.dat append\n') fout.write(f'\t\tOut_String("XXXX")\n') + + # FIXME Does not write out weighted_Durbin_Watson, TOPAS complained about this fout.write('\t\t{: <40} {: <40} {: <40} {: <40} {: <40} {: <40}'.format( f'Out(Get(r_wp), "%11.5f")', f'Out(Get(r_exp), "%11.5f")', @@ -477,6 +477,9 @@ def make_big_inp(data: dict, options={}): ''' Generates a big .INP-file with all filenames found in data["path"]. Uses a template .INP-file (which has to be generated manually from an initial refinement in TOPAS) and appends this to a large .INP-file while changing the filenames. ''' + + # FIXME Strip headers from initial INP file before copying it. + required_options = ['template', 'output', 'overwrite', 'backup', 'backup_dir', 'include', 'topas_options', 'save_results', 'save_dir', 'log', 'logfile'] default_options = { @@ -635,29 +638,29 @@ def make_inp_entry(template: str, xdd: str, num: int, options: dict) -> str: # Replace diffractogram-path s = template.replace(temp_xdd, xdd).replace('XXXX', num_str) - basename = os.path.basename(xdd).split(".")[0] + # basename = os.path.basename(xdd).split(".")[0] - # Define regular expressions for output lines - regs = [r'Out_Riet\([\S]*\)', - r'Out_CIF_STR\([\S]*\)', - r'Out_CIF_ADPs\([\S]*\)', - r'Out_CIF_Bonds_Angles\([\S]*\)', - r'Out_FCF\([\S]*\)', - r'Create_hklm_d_Th2_Ip_file\([\S]*\)', - r'out(.*?)append'] + # # Define regular expressions for output lines + # regs = [r'Out_Riet\([\S]*\)', + # r'Out_CIF_STR\([\S]*\)', + # r'Out_CIF_ADPs\([\S]*\)', + # r'Out_CIF_Bonds_Angles\([\S]*\)', + # r'Out_FCF\([\S]*\)', + # r'Create_hklm_d_Th2_Ip_file\([\S]*\)', + # r'out(.*?)append'] - # Define substitute strings for output lines - subs = [f'Out_Riet({options["save_dir"]}/{basename}_riet.xy)', - f'Out_CIF_STR({options["save_dir"]}/{basename}.cif)', - f'Out_CIF_ADPs({options["save_dir"]}/{basename}.cif)', - f'Out_CIF_Bonds_Angles({options["save_dir"]}/{basename}.cif)', - f'Out_FCF({options["save_dir"]}/{basename}.fcf)', - f'Create_hklm_d_Th2_Ip_file({options["save_dir"]}/{basename}_hkl.dat)', - f'out \t {options["save_dir"]}/{basename}_refined_params.dat \t append'] + # # Define substitute strings for output lines + # subs = [f'Out_Riet({options["save_dir"]}/{basename}_riet.xy)', + # f'Out_CIF_STR({options["save_dir"]}/{basename}.cif)', + # f'Out_CIF_ADPs({options["save_dir"]}/{basename}.cif)', + # f'Out_CIF_Bonds_Angles({options["save_dir"]}/{basename}.cif)', + # f'Out_FCF({options["save_dir"]}/{basename}.fcf)', + # f'Create_hklm_d_Th2_Ip_file({options["save_dir"]}/{basename}_hkl.dat)', + # f'out \t {options["save_dir"]}/{basename}_refined_params.dat \t append'] - # Substitute strings in output lines - for reg, sub in zip(regs, subs): - s = re.sub(reg, sub, s) + # # Substitute strings in output lines + # for reg, sub in zip(regs, subs): + # s = re.sub(reg, sub, s) @@ -716,7 +719,12 @@ def refine(data: dict, options={}): # Create folders if they don't exist - paths, headers = get_paths(data['inp']), get_headers(data['inp']) + + # FIXME Since the big INP files now have the same filename for all iterations, we need to adjust the code to only get unique values from the get_paths function + # FIXME get_headers() is also not working now. Needs to be adjusted to the new way of writing the Out-parameters + paths = get_paths(data['inp']) + headers = get_headers(data['inp']) + for path in paths: dirname = os.path.dirname(path) @@ -745,4 +753,15 @@ def refine(data: dict, options={}): os.makedirs(os.path.dirname(options['topas_logfile'])) - subprocess.call(command, shell=True) \ No newline at end of file + subprocess.call(command, shell=True) + + + + + +def read_results(): + # FIXME Write the function + + return None + + From 4e579bd07b1cf3013f4af88094ab5a6f6d022274 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 14 Jul 2022 21:44:36 +0200 Subject: [PATCH 233/355] dirty fix to enable read cif w/o cell_volume --- nafuma/xrd/refinement.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 57bfa72..b94e1fb 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -249,7 +249,11 @@ def write_str(fout, data, options, index=0): fout.write(f'\t\tbe = lpbe_{label} ;\n') fout.write(f'\t\tga = lpga_{label} ;\n') fout.write('\n') - fout.write(f'\t\tcell_volume\t vol_{label}_XXXX {atoms["_cell_volume"]}\n') + #FIXME fix the if-statement below, so that cell-volume is the correct formula, based on lattice params and angles. + if '_cell_volume' not in atoms.keys(): + atoms['_cell_volume'] = 0 + else: + fout.write(f'\t\tcell_volume\t vol_{label}_XXXX {atoms["_cell_volume"]}\n') fout.write(f'\t\tcell_mass\t mass_{label}_XXXX 1\n') fout.write(f'\t\tweight_percent\t wp_{label}_XXXX 100\n\n') fout.write('\n') @@ -456,7 +460,7 @@ def read_cif(path): line = cif.readline() - + print(data.keys()) return data From 7ea27abf3a41abc4ec86966c80fe4bae767164fa Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 15 Jul 2022 11:25:27 +0200 Subject: [PATCH 234/355] Add function to get unique entries in list --- nafuma/auxillary.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 96b9ce2..4734802 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -134,4 +134,15 @@ def backup_file(filename, backup_dir): dst = os.path.join(backup_dir, dst_basename) - shutil.copy(filename, dst) \ No newline at end of file + shutil.copy(filename, dst) + + +def get_unique(full_list): + + unique_list = [] + + for entry in full_list: + if not entry in unique_list: + unique_list.append(entry) + + return unique_list \ No newline at end of file From c987fc689f404340d1ab77bc68dc1a096e46c6b7 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 15 Jul 2022 11:26:57 +0200 Subject: [PATCH 235/355] Fix get_headers to read new formatting correctly --- nafuma/xrd/refinement.py | 92 +++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index b94e1fb..f519b43 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -1,4 +1,3 @@ -from email.policy import default import os import shutil import subprocess @@ -9,6 +8,8 @@ import datetime import warnings import json +import pandas as pd + import nafuma.auxillary as aux @@ -160,25 +161,25 @@ def write_params(fout, data, options, index=0): # WRITE LATTICE PARAMETERS # If start_values is defined: fout.write(f'#ifdef start_values_{label}\n') - lpa = f'local lpa_{label} {a} ;: {a}' - lpb = f'local lpb_{label} {b} ;: {b}' - lpc = f'local lpc_{label} {c} ;: {c}' + lpa = f'local !lpa_{label} {a} ;: {a}' + lpb = f'local !lpb_{label} {b} ;: {b}' + lpc = f'local !lpc_{label} {c} ;: {c}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpa, lpb, lpc)) - lpal = f'local lpal_{label} {alpha} ;: {alpha}' - lpbe = f'local lpbe_{label} {beta} ;: {beta}' - lpga = f'local lpga_{label} {gamma} ;: {gamma}' + lpal = f'local !lpal_{label} {alpha} ;: {alpha}' + lpbe = f'local !lpbe_{label} {beta} ;: {beta}' + lpga = f'local !lpga_{label} {gamma} ;: {gamma}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpal, lpbe, lpga)) # Otherwise fout.write('\n') fout.write('#else\n') - lpa = f'local lpa_{label} {a}' - lpb = f'local lpb_{label} {b}' - lpc = f'local lpc_{label} {c}' + lpa = f'local !lpa_{label} {a}' + lpb = f'local !lpb_{label} {b}' + lpc = f'local !lpc_{label} {c}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpa, lpb, lpc)) - lpal = f'local lpal_{label} {alpha}' - lpbe = f'local lpbe_{label} {beta}' - lpga = f'local lpga_{label} {gamma}' + lpal = f'local !lpal_{label} {alpha}' + lpbe = f'local !lpbe_{label} {beta}' + lpga = f'local !lpga_{label} {gamma}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpal, lpbe, lpga)) fout.write('#endif\n\n') @@ -188,8 +189,8 @@ def write_params(fout, data, options, index=0): '_atom_site_fract_x': 'x', '_atom_site_fract_y': 'y', '_atom_site_fract_z': 'z', - '_atom_site_occupancy': '!occ', - '_atom_site_B_iso_or_equiv': '!beq' + '_atom_site_occupancy': 'occ', + '_atom_site_B_iso_or_equiv': 'beq' } @@ -202,7 +203,7 @@ def write_params(fout, data, options, index=0): value = atoms["atoms"][site][attr].split("(")[0] value = value if value != '.' else 0. - params.append('{: <20} {: <20}'.format(f'local {attrs[attr]}_{site}_{label}', f' = {value} ;: {value}')) + params.append('{: <20} {: <20}'.format(f'local !{attrs[attr]}_{site}_{label}', f' = {value} ;: {value}')) #fout.write(f'local {attrs[attr]}_{site}_{label}\t\t =\t {value} \t ;= \t\t\t\t\t') fout.write('{: <55} {: <55} {: <55} {: <55} {: <55}\n'.format(*params)) @@ -210,10 +211,10 @@ def write_params(fout, data, options, index=0): fout.write('\n') fout.write('{: <55} {: <55} {: <55} {: <55}\n'.format( - f'local csgc_{label}_XXXX = 200 ;: 200', - f'local cslc_{label}_XXXX = 200 ;: 200', - f'local sgc_{label}_XXXX = 0 ;: 0', - f'local slc_{label}_XXXX = 0 ;: 0', + f'local !csgc_{label}_XXXX = 200 ;: 200', + f'local !cslc_{label}_XXXX = 200 ;: 200', + f'local !sgc_{label}_XXXX = 0 ;: 0', + f'local !slc_{label}_XXXX = 0 ;: 0', )) @@ -382,14 +383,13 @@ def write_output(fout, data, options, index=0): ) fout.write('\n') - + + fout.write('\t\tOut_String("\\n")\n') + fout.write('#endif') fout.write('\n\n') - fout.write('#endif') - - def read_cif(path): @@ -588,32 +588,40 @@ def write_headers(fout, options): fout.write(f'{option} {value} \n') -def get_headers(inp): +def get_headers(inp, path): with open(inp, 'r') as inp: - headers = [] + headers = ['index'] line = inp.readline() - while not all(keyword in line for keyword in ['out', 'append']): + while not path in line: line = inp.readline() # Jump down to lines line = inp.readline() line = inp.readline() - while not 'Out_String' in line: - + while not '#endif' in line: if line.split(): - header = line.split()[1] - if all(keyword in header for keyword in ['Get', '(', ')']): - header = header[4:-1] + + regx = r"\([\S]*" + headers_line = re.findall(regx, line) - headers.append(header) + for i, header in enumerate(headers_line): + header = header[1:-1] + + if all(keyword in header for keyword in ['Get', '(', ')']): + header = header[4:-1] + + headers_line[i] = header + + for header in headers_line: + if header != '"\\n"': + headers.append(header) line = inp.readline() - return headers @@ -726,11 +734,14 @@ def refine(data: dict, options={}): # FIXME Since the big INP files now have the same filename for all iterations, we need to adjust the code to only get unique values from the get_paths function # FIXME get_headers() is also not working now. Needs to be adjusted to the new way of writing the Out-parameters - paths = get_paths(data['inp']) - headers = get_headers(data['inp']) - + paths = get_paths(data['inp']) + paths = aux.get_unique(paths) + + for path in paths: + headers = get_headers(data['inp'], path) + dirname = os.path.dirname(path) if dirname and not os.path.isdir(dirname): @@ -763,9 +774,10 @@ def refine(data: dict, options={}): -def read_results(): - # FIXME Write the function +def read_results(path): - return None + results = pd.read_csv(path, delim_whitespace=True, index_col=0) + + return results From ed5d96bf4e84750d21294b9051177a8359566406 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 15 Jul 2022 11:35:04 +0200 Subject: [PATCH 236/355] read_cif now can read cifs w/o cell_volume --- nafuma/xrd/refinement.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index b94e1fb..1a5808d 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -3,7 +3,7 @@ import os import shutil import subprocess import re - +import numpy as np import time import datetime import warnings @@ -249,11 +249,7 @@ def write_str(fout, data, options, index=0): fout.write(f'\t\tbe = lpbe_{label} ;\n') fout.write(f'\t\tga = lpga_{label} ;\n') fout.write('\n') - #FIXME fix the if-statement below, so that cell-volume is the correct formula, based on lattice params and angles. - if '_cell_volume' not in atoms.keys(): - atoms['_cell_volume'] = 0 - else: - fout.write(f'\t\tcell_volume\t vol_{label}_XXXX {atoms["_cell_volume"]}\n') + fout.write(f'\t\tcell_volume\t vol_{label}_XXXX {atoms["_cell_volume"]}\n') fout.write(f'\t\tcell_mass\t mass_{label}_XXXX 1\n') fout.write(f'\t\tweight_percent\t wp_{label}_XXXX 100\n\n') fout.write('\n') @@ -393,7 +389,7 @@ def write_output(fout, data, options, index=0): def read_cif(path): - data = {'atoms': {}} # Initialise dictionary + atoms = {'atoms': {}} # Initialise dictionary read = True # Initialise read toggle # Lists attributes to get out of the .CIF-file. This will correspond to what VESTA writes out, not necessarily what you will find in ICSD @@ -428,12 +424,12 @@ def read_cif(path): # FIXME WHat a horrible condition statement - need to fix this! while line and not line.lstrip().startswith('_') and not line.lstrip().startswith('loop_') and not line.lstrip().startswith('#End') and not line=='\n': # Initialise empty dictionary for a given atom if it has not already been created in another loop - if line.split()[0] not in data['atoms'].keys(): - data["atoms"][line.split()[0]] = {} + if line.split()[0] not in atoms['atoms'].keys(): + atoms["atoms"][line.split()[0]] = {} # Add all the attribute / value pairs for the current loop for i, attr in enumerate(loop): - data["atoms"][line.split()[0]][attr[:-1].lstrip()] = line.split()[i] + atoms["atoms"][line.split()[0]][attr[:-1].lstrip()] = line.split()[i] # Read new line line = cif.readline() @@ -453,15 +449,22 @@ def read_cif(path): value = ' '.join([str(i) for i in value]) - data[attr] = value + atoms[attr] = value line = cif.readline() else: line = cif.readline() + if '_cell_volume' not in atoms.keys(): + a = float(atoms['_cell_length_a'].split('(')[0]) + b = float(atoms['_cell_length_b'].split('(')[0]) + c = float(atoms['_cell_length_c'].split('(')[0]) + alpha = float(atoms['_cell_angle_alpha'].split('(')[0]) + beta = float(atoms['_cell_angle_beta'].split('(')[0]) + gamma = float(atoms['_cell_angle_gamma'].split('(')[0]) - print(data.keys()) - return data + atoms['_cell_volume'] = a * b * c * np.sqrt(1-np.cos(alpha)**2 - np.cos(beta)**2 - np.cos(gamma)**2 + 2 * np.cos(alpha) * np.cos(beta) * np. cos(gamma)) + return atoms From fb040aa0e54f1a2d09d2626cc63d5fb154e099f5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 15 Jul 2022 11:51:43 +0200 Subject: [PATCH 237/355] Strip headers from INPs when making big file --- nafuma/xrd/refinement.py | 60 ++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index f519b43..55150a1 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -539,7 +539,11 @@ def make_big_inp(data: dict, options={}): runlist = os.path.join(os.path.dirname(options['output']), 'runlist.txt') options['include'].append(runlist) - with open(options['template'], 'r') as template, open(options['output'], 'w', newline='\n') as output, open(runlist, 'w', newline='\n') as runlist: + with open(options['template'], 'r') as template: + strip_headers(template) + + + with open('tmp_template.inp', 'r') as template, open(options['output'], 'w', newline='\n') as output, open(runlist, 'w', newline='\n') as runlist: write_headers(output, options) @@ -553,6 +557,34 @@ def make_big_inp(data: dict, options={}): runlist.write('#define \tUSE_'+f'{i}'.zfill(4) + '\n') + os.remove('tmp_template.inp') + + +def strip_headers(fin): + + line = fin.readline() + newlines = [] + + while 'r_wp' not in line: + if line[0] != '\'': + line = fin.readline() + else: + newlines.append(line) + line = fin.readline() + + newlines.append(line) + newlines = newlines + fin.readlines() + + with open('tmp_template.inp', 'w') as fout: + for line in newlines: + fout.write(line) + + + + + + + def write_headers(fout, options): @@ -650,32 +682,6 @@ def make_inp_entry(template: str, xdd: str, num: int, options: dict) -> str: # Replace diffractogram-path s = template.replace(temp_xdd, xdd).replace('XXXX', num_str) - # basename = os.path.basename(xdd).split(".")[0] - - # # Define regular expressions for output lines - # regs = [r'Out_Riet\([\S]*\)', - # r'Out_CIF_STR\([\S]*\)', - # r'Out_CIF_ADPs\([\S]*\)', - # r'Out_CIF_Bonds_Angles\([\S]*\)', - # r'Out_FCF\([\S]*\)', - # r'Create_hklm_d_Th2_Ip_file\([\S]*\)', - # r'out(.*?)append'] - - # # Define substitute strings for output lines - # subs = [f'Out_Riet({options["save_dir"]}/{basename}_riet.xy)', - # f'Out_CIF_STR({options["save_dir"]}/{basename}.cif)', - # f'Out_CIF_ADPs({options["save_dir"]}/{basename}.cif)', - # f'Out_CIF_Bonds_Angles({options["save_dir"]}/{basename}.cif)', - # f'Out_FCF({options["save_dir"]}/{basename}.fcf)', - # f'Create_hklm_d_Th2_Ip_file({options["save_dir"]}/{basename}_hkl.dat)', - # f'out \t {options["save_dir"]}/{basename}_refined_params.dat \t append'] - - # # Substitute strings in output lines - # for reg, sub in zip(regs, subs): - # s = re.sub(reg, sub, s) - - - return s From 7e952a4556921cfcf2cd1727dc7fd5d3658fd332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Halvor=20H=C3=B8en=20Hval?= Date: Wed, 20 Jul 2022 18:00:33 +0200 Subject: [PATCH 238/355] tester fra laptop --- test.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..e69de29 From 9f9d364a2f778d050aa541ad9475da882d03a465 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 1 Aug 2022 11:04:59 +0200 Subject: [PATCH 239/355] Allow loading of Neware datasets from parent dirs --- nafuma/electrochemistry/io.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 8256749..1ebc7f8 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -7,10 +7,10 @@ import os import nafuma.auxillary as aux from sympy import re -def read_data(data, options=None): +def read_data(data, options={}): if data['kind'] == 'neware': - df = read_neware(data['path']) + df = read_neware(data['path'], options=options) cycles = process_neware_data(df=df, options=options) elif data['kind'] == 'batsmall': @@ -25,7 +25,7 @@ def read_data(data, options=None): -def read_neware(path, summary=False): +def read_neware(path, options={}): ''' Reads electrochemistry data, currently only from the Neware battery cycler. Will convert to .csv if the filetype is .xlsx, which is the file format the Neware provides for the backup data. In this case it matters if summary is False or not. If file type is .csv, it will just open the datafile and it does not matter if summary is False or not.''' @@ -34,17 +34,18 @@ def read_neware(path, summary=False): # FIXME Do a check if a .csv-file already exists even if the .xlsx is passed # Convert from .xlsx to .csv to make readtime faster - if path.split('.')[-1] == 'xlsx': - csv_details = ''.join(path.split('.')[:-1]) + '_details.csv' - csv_summary = ''.join(path.split('.')[:-1]) + '_summary.csv' + if path.endswith('xlsx'): + csv_details = ''.join(path[:-5]) + '_details.csv' + csv_summary = ''.join(path[:-5]) + '_summary.csv' + if not os.path.isfile(csv_summary): Xlsx2csv(path, outputencoding="utf-8").convert(csv_summary, sheetid=3) if not os.path.isfile(csv_details): Xlsx2csv(path, outputencoding="utf-8").convert(csv_details, sheetid=4) - if summary: + if options['summary']: df = pd.read_csv(csv_summary) else: df = pd.read_csv(csv_details) From 574f633db0f337366b9f81177d975de87cdca35b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 1 Aug 2022 14:50:10 +0200 Subject: [PATCH 240/355] Add correct unit conversion of Neware summaries --- nafuma/electrochemistry/io.py | 177 +++++++++++++++++++++++----------- 1 file changed, 120 insertions(+), 57 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 1ebc7f8..108c5eb 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -1,4 +1,3 @@ -from email.policy import default import pandas as pd import numpy as np import matplotlib.pyplot as plt @@ -50,7 +49,7 @@ def read_neware(path, options={}): else: df = pd.read_csv(csv_details) - elif path.split('.')[-1] == 'csv': + elif path.endswith('csv'): df = pd.read_csv(path) @@ -237,56 +236,66 @@ def process_neware_data(df, options={}): options['kind'] = 'neware' - # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. - set_units(options=options) # sets options['units'] - options['old_units'] = get_old_units(df=df, options=options) - - df = add_columns(df=df, options=options) # adds columns to the DataFrame if active material weight and/or molecular weight has been passed in options + if not options['summary']: + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. + set_units(options=options) # sets options['units'] + options['old_units'] = get_old_units(df=df, options=options) + + df = add_columns(df=df, options=options) # adds columns to the DataFrame if active material weight and/or molecular weight has been passed in options - df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units + df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units - # Creates masks for charge and discharge curves - chg_mask = df['status'] == 'CC Chg' - dchg_mask = df['status'] == 'CC DChg' + # Creates masks for charge and discharge curves + chg_mask = df['status'] == 'CC Chg' + dchg_mask = df['status'] == 'CC DChg' - # Initiate cycles list - cycles = [] + # Initiate cycles list + cycles = [] - # Loop through all the cycling steps, change the current and capacities in the - for i in range(df["cycle"].max()): + # Loop through all the cycling steps, change the current and capacities in the + for i in range(df["cycle"].max()): - sub_df = df.loc[df['cycle'] == i+1].copy() + sub_df = df.loc[df['cycle'] == i+1].copy() - #sub_df.loc[dchg_mask, 'current'] *= -1 - #sub_df.loc[dchg_mask, 'capacity'] *= -1 + #sub_df.loc[dchg_mask, 'current'] *= -1 + #sub_df.loc[dchg_mask, 'capacity'] *= -1 - chg_df = sub_df.loc[chg_mask] - dchg_df = sub_df.loc[dchg_mask] + chg_df = sub_df.loc[chg_mask] + dchg_df = sub_df.loc[dchg_mask] - # Continue to next iteration if the charge and discharge DataFrames are empty (i.e. no current) - if chg_df.empty and dchg_df.empty: - continue + # Continue to next iteration if the charge and discharge DataFrames are empty (i.e. no current) + if chg_df.empty and dchg_df.empty: + continue - # Reverses the discharge curve if specified - if options['reverse_discharge']: - max_capacity = dchg_df['capacity'].max() - dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) + # Reverses the discharge curve if specified + if options['reverse_discharge']: + max_capacity = dchg_df['capacity'].max() + dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) - if 'specific_capacity' in df.columns: - max_capacity = dchg_df['specific_capacity'].max() - dchg_df['specific_capacity'] = np.abs(dchg_df['specific_capacity'] - max_capacity) + if 'specific_capacity' in df.columns: + max_capacity = dchg_df['specific_capacity'].max() + dchg_df['specific_capacity'] = np.abs(dchg_df['specific_capacity'] - max_capacity) - if 'ions' in df.columns: - max_capacity = dchg_df['ions'].max() - dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) + if 'ions' in df.columns: + max_capacity = dchg_df['ions'].max() + dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) - cycles.append((chg_df, dchg_df)) + cycles.append((chg_df, dchg_df)) + + return cycles + elif options['summary']: + set_units(options=options) + options['old_units'] = get_old_units(df=df, options=options) + + df = add_columns(df=df, options=options) + df = unit_conversion(df=df, options=options) + + return df - return cycles def process_biologic_data(df, options=None): @@ -403,30 +412,84 @@ def unit_conversion(df, options): if options['kind'] == 'neware': - df['Current({})'.format(options['old_units']['current'])] = df['Current({})'.format(options['old_units']['current'])] * unit_tables.current()[options['old_units']['current']].loc[options['units']['current']] - df['Voltage({})'.format(options['old_units']['voltage'])] = df['Voltage({})'.format(options['old_units']['voltage'])] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] - df['Capacity({})'.format(options['old_units']['capacity'])] = df['Capacity({})'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] - df['Energy({})'.format(options['old_units']['energy'])] = df['Energy({})'.format(options['old_units']['energy'])] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] - df['CycleTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=options['units']['time']), axis=1) - df['RunTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_datetime_string(row['Real Time(h:min:s.ms)'], reference=df['Real Time(h:min:s.ms)'].iloc[0], unit=options['units']['time']), axis=1) - columns = ['status', 'jump', 'cycle', 'steps', 'current', 'voltage', 'capacity', 'energy'] + + if options['summary']: + # Add the charge and discharge energy columns to get a single energy column + df[f'Energy({options["old_units"]["energy"]})'] = df[f'Chg Eng({options["old_units"]["energy"]})'] + df[f'DChg Eng({options["old_units"]["energy"]})'] + + df[f'Starting current({options["old_units"]["current"]})'] = df[f'Starting current({options["old_units"]["current"]})'] * unit_tables.current()[options['old_units']['current']].loc[options['units']['current']] + df[f'Start Volt({options["old_units"]["voltage"]})'] = df[f'Start Volt({options["old_units"]["voltage"]})'] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] + df[f'Capacity({options["old_units"]["capacity"]})'] = df[f'Capacity({options["old_units"]["capacity"]})'] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] + df[f'Energy({options["old_units"]["energy"]})'] = df[f'Energy({options["old_units"]["energy"]})'] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] + df[f'CycleTime({options["units"]["time"]})'] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=options['units']['time']), axis=1) + df[f'RunTime({options["units"]["time"]})'] = df.apply(lambda row : convert_datetime_string(row['Real Time'], reference=df['Real Time'].iloc[0], ref_time=df[f'CycleTime({options["units"]["time"]})'].iloc[0],unit=options['units']['time']), axis=1) - if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: - df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] - columns.append('specific_capacity') + + # Drop all undesireable columns + df.drop( + [ + 'Chnl', + 'Original step', + f'End Volt({options["old_units"]["voltage"]})', + f'Termination current({options["old_units"]["current"]})', + 'Relative Time(h:min:s.ms)', + 'Real Time', + 'Continuous Time(h:min:s.ms)', + f'Net discharge capacity({options["old_units"]["capacity"]})', + f'Chg Cap({options["old_units"]["capacity"]})', + f'DChg Cap({options["old_units"]["capacity"]})', + f'Net discharge energy({options["old_units"]["energy"]})', + f'Chg Eng({options["old_units"]["energy"]})', + f'DChg Eng({options["old_units"]["energy"]})' + ], + axis=1, inplace=True + ) + + columns = ['cycle', 'steps', 'status', 'voltage', 'current', 'capacity'] + + + + + # Add column labels for specific capacity and ions if they exist + if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] + columns.append('specific_capacity') if 'IonsExtracted' in df.columns: columns.append('ions') + # Append energy column label here as it was the last column to be generated + columns.append('energy') + columns.append('cycle_time') + columns.append('runtime') + + # Apply new column labels + df.columns = columns + + + else: + df['Current({})'.format(options['old_units']['current'])] = df['Current({})'.format(options['old_units']['current'])] * unit_tables.current()[options['old_units']['current']].loc[options['units']['current']] + df['Voltage({})'.format(options['old_units']['voltage'])] = df['Voltage({})'.format(options['old_units']['voltage'])] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] + df['Capacity({})'.format(options['old_units']['capacity'])] = df['Capacity({})'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] + df['Energy({})'.format(options['old_units']['energy'])] = df['Energy({})'.format(options['old_units']['energy'])] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] + df['CycleTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=options['units']['time']), axis=1) + df['RunTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_datetime_string(row['Real Time(h:min:s.ms)'], reference=df['Real Time(h:min:s.ms)'].iloc[0], ref_time=df[f'CycleTime({options["units"]["time"]})'].iloc[0], unit=options['units']['time']), axis=1) + columns = ['status', 'jump', 'cycle', 'steps', 'current', 'voltage', 'capacity', 'energy'] + + if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] + columns.append('specific_capacity') + + if 'IonsExtracted' in df.columns: + columns.append('ions') + + columns.append('cycle_time') + columns.append('time') - columns.append('cycle_time') - columns.append('time') + df.drop(['Record number', 'Relative Time(h:min:s.ms)', 'Real Time(h:min:s.ms)'], axis=1, inplace=True) - - df.drop(['Record number', 'Relative Time(h:min:s.ms)', 'Real Time(h:min:s.ms)'], axis=1, inplace=True) - - df.columns = columns + df.columns = columns if options['kind'] == 'biologic': df['time/{}'.format(options['old_units']['time'])] = df["time/{}".format(options['old_units']["time"])] * unit_tables.time()[options['old_units']["time"]].loc[options['units']['time']] @@ -488,13 +551,13 @@ def get_old_units(df: pd.DataFrame, options: dict) -> dict: if options['kind']=='neware': for column in df.columns: - if 'Voltage' in column: + if 'Voltage' in column or 'Start Volt' in column: voltage = column.split('(')[-1].strip(')') - elif 'Current' in column: + elif 'Current' in column or 'Starting current' in column: current = column.split('(')[-1].strip(')') elif 'Capacity' in column: capacity = column.split('(')[-1].strip(')') - elif 'Energy' in column: + elif 'Energy' in column or 'Eng' in column: energy = column.split('(')[-1].strip(')') old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy} @@ -521,7 +584,7 @@ def get_old_units(df: pd.DataFrame, options: dict) -> dict: def convert_time_string(time_string, unit='ms'): ''' Convert time string from Neware-data with the format hh:mm:ss.xx to any given unit''' - h, m, s = time_string.split(':') + h, m, s = time_string.split(':') ms = float(s)*1000 + int(m)*1000*60 + int(h)*1000*60*60 factors = {'ms': 1, 's': 1/1000, 'min': 1/(1000*60), 'h': 1/(1000*60*60)} @@ -532,7 +595,7 @@ def convert_time_string(time_string, unit='ms'): -def convert_datetime_string(datetime_string, reference, unit='s'): +def convert_datetime_string(datetime_string, reference, ref_time, unit='s'): ''' Convert time string from Neware-data with the format yyy-mm-dd hh:mm:ss to any given unit''' from datetime import datetime @@ -555,7 +618,7 @@ def convert_datetime_string(datetime_string, reference, unit='s'): factors = {'ms': 1000, 's': 1, 'min': 1/(60), 'h': 1/(60*60)} - time = s * factors[unit] + time = s * factors[unit] + ref_time return time From 9ebab7d6ee979a7bf61dbdd8e3602d8d70c014c1 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 1 Aug 2022 15:47:46 +0200 Subject: [PATCH 241/355] Add splice cycles for Neware (summary + cycles) --- nafuma/electrochemistry/io.py | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 108c5eb..71fa93b 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -206,6 +206,47 @@ def splice_cycles(df, options: dict) -> pd.DataFrame: add = df['specific_capacity'].iloc[i-1] df['specific_capacity'].iloc[i:last_chg] = df['specific_capacity'].iloc[i:last_chg] + add + + if options['kind'] == 'neware': + + if options['summary']: + for i in range(df['cycle'].max()): + sub_df = df.loc[df['cycle'] == i+1].copy() + + if sub_df['status'].loc[sub_df['status'] == 'CC Chg'].count() > 1: + indices = sub_df.index[sub_df['status'] == 'CC Chg'] + + add_columns = ['capacity', 'specific_capacity', 'ions', 'energy', 'cycle_time'] + + for column in add_columns: + if column in df.columns: + df[column].iloc[indices[-1]] = df[column].iloc[indices[-1]] + df[column].iloc[indices[0]] + + df.drop(index=indices[0], inplace=True) + df.reset_index(inplace=True, drop=True) + + else: + for i in range(df['cycle'].max()): + sub_df = df.loc[df['cycle'] == i+1].copy() + sub_chg_df = sub_df.loc[sub_df['status'] == 'CC Chg'].copy() + + steps_indices = sub_chg_df['steps'].unique() + + if len(steps_indices) > 1: + + add_columns = ['capacity', 'specific_capacity', 'ions', 'energy', 'cycle_time'] + + for column in add_columns: + if column in df.columns: + # Extract the maximum value from the first of the two cycles by accessing the column value of the highest index of the first cycle + add = df[column].iloc[df.loc[df['steps'] == steps_indices[0]].index.max()] + + df[column].loc[df['steps'] == steps_indices[1]] += add + + + + + return df @@ -245,6 +286,9 @@ def process_neware_data(df, options={}): df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units + if options['splice_cycles']: + df = splice_cycles(df=df, options=options) + # Creates masks for charge and discharge curves chg_mask = df['status'] == 'CC Chg' @@ -294,6 +338,9 @@ def process_neware_data(df, options={}): df = add_columns(df=df, options=options) df = unit_conversion(df=df, options=options) + if options['splice_cycles']: + df = splice_cycles(df=df, options=options) + return df From 0699399d1a1592202c3eda856e91f6a9d57e4f61 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 1 Aug 2022 16:37:37 +0200 Subject: [PATCH 242/355] Mix intervals and ints in update_cycles --- nafuma/electrochemistry/plot.py | 90 ++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 9e80471..7b1f000 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -19,10 +19,11 @@ def plot_gc(data, options=None): # Update options - required_options = ['x_vals', 'y_vals', 'which_cycles', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params'] + required_options = ['x_vals', 'y_vals', 'which_cycles', 'exclude_cycles', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params'] default_options = { 'x_vals': 'capacity', 'y_vals': 'voltage', - 'which_cycles': 'all', + 'which_cycles': 'all', + 'exclude_cycles': [], 'charge': True, 'discharge': True, 'colours': None, 'differentiate_charge_discharge': True, @@ -35,43 +36,45 @@ def plot_gc(data, options=None): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - - if not 'cycles' in data.keys(): data['cycles'] = ec.io.read_data(data=data, options=options) - # Update list of cycles to correct indices - update_cycles_list(cycles=data['cycles'], options=options) - - colours = generate_colours(cycles=data['cycles'], options=options) - - if options['interactive']: - options['interactive'], options['interactive_session_active'] = False, True - plot_gc_interactive(data=data, options=options) - return - - - # Prepare plot, and read and process data - fig, ax = btp.prepare_plot(options=options) + if not options['summary']: + # Update list of cycles to correct indices + update_cycles_list(data=data, options=options) - for i, cycle in enumerate(data['cycles']): - if i in options['which_cycles']: - if options['charge']: - cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + colours = generate_colours(cycles=data['cycles'], options=options) - if options['discharge']: - cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + if options['interactive']: + options['interactive'], options['interactive_session_active'] = False, True + plot_gc_interactive(data=data, options=options) + return - if options['interactive_session_active']: - update_labels(options, force=True) - else: - update_labels(options) + # Prepare plot, and read and process data + + fig, ax = btp.prepare_plot(options=options) - fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + for i, cycle in enumerate(data['cycles']): + if i in options['which_cycles']: + if options['charge']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) - #if options['interactive_session_active']: + if options['discharge']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + + + if options['interactive_session_active']: + update_labels(options, force=True) + else: + update_labels(options) + + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + elif options['summary']: + + fig, ax = 0, 0 return data['cycles'], fig, ax @@ -118,29 +121,46 @@ def update_labels(options, force=False): -def update_cycles_list(cycles, options: dict) -> None: +def update_cycles_list(data, options: dict) -> None: if options['which_cycles'] == 'all': - options['which_cycles'] = [i for i in range(len(cycles))] + options['which_cycles'] = [i for i in range(len(data['cycles']))] - elif type(options['which_cycles']) == list: - options['which_cycles'] = [i-1 for i in options['which_cycles']] + elif isinstance(options['which_cycles'], list): + + cycles =[] + + for cycle in options['which_cycles']: + if isinstance(cycle, int): + cycles.append(cycle-1) + + elif isinstance(cycle, tuple): + interval = [i-1 for i in range(cycle[0], cycle[1]+1)] + cycles.extend(interval) + + + options['which_cycles'] = cycles # Tuple is used to define an interval - as elements tuples can't be assigned, I convert it to a list here. - elif type(options['which_cycles']) == tuple: + elif isinstance(options['which_cycles'], tuple): which_cycles = list(options['which_cycles']) if which_cycles[0] <= 0: which_cycles[0] = 1 elif which_cycles[1] < 0: - which_cycles[1] = len(cycles) + which_cycles[1] = len(options['which_cycles']) options['which_cycles'] = [i-1 for i in range(which_cycles[0], which_cycles[1]+1)] + + for i, cycle in enumerate(options['which_cycles']): + if cycle in options['exclude_cycles']: + del options['which_cycles'][i] + From 224d05e0e95b8137ae25f1ccf7bb9fcc79461ede Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 1 Aug 2022 17:00:10 +0200 Subject: [PATCH 243/355] Return summary as separte chg / dchg dataframes --- nafuma/electrochemistry/io.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 71fa93b..b4213ee 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -52,7 +52,6 @@ def read_neware(path, options={}): elif path.endswith('csv'): df = pd.read_csv(path) - return df @@ -243,10 +242,6 @@ def splice_cycles(df, options: dict) -> pd.DataFrame: df[column].loc[df['steps'] == steps_indices[1]] += add - - - - return df @@ -297,14 +292,13 @@ def process_neware_data(df, options={}): # Initiate cycles list cycles = [] + + # Loop through all the cycling steps, change the current and capacities in the for i in range(df["cycle"].max()): sub_df = df.loc[df['cycle'] == i+1].copy() - #sub_df.loc[dchg_mask, 'current'] *= -1 - #sub_df.loc[dchg_mask, 'capacity'] *= -1 - chg_df = sub_df.loc[chg_mask] dchg_df = sub_df.loc[dchg_mask] @@ -328,7 +322,8 @@ def process_neware_data(df, options={}): cycles.append((chg_df, dchg_df)) - return cycles + + return cycles elif options['summary']: @@ -341,7 +336,13 @@ def process_neware_data(df, options={}): if options['splice_cycles']: df = splice_cycles(df=df, options=options) - return df + + chg_df = df.loc[df['status'] == 'CC Chg'] + dchg_df = df.loc[df['status'] == 'CC DChg'] + + cycles = [chg_df, dchg_df] + + return cycles From 20111a745786c5d1b5d3252e2ea84527e67014d0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 1 Aug 2022 17:00:26 +0200 Subject: [PATCH 244/355] Allow plotting of summary through plot_gc() --- nafuma/electrochemistry/plot.py | 37 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 7b1f000..2d26b4b 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -36,26 +36,26 @@ def plot_gc(data, options=None): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + # Read data if not already loaded if not 'cycles' in data.keys(): data['cycles'] = ec.io.read_data(data=data, options=options) + # Update list of cycles to correct indices + update_cycles_list(data=data, options=options) + + if options['interactive']: + options['interactive'], options['interactive_session_active'] = False, True + plot_gc_interactive(data=data, options=options) + return + + + # Prepare plot + fig, ax = btp.prepare_plot(options=options) + colours = generate_colours(cycles=data['cycles'], options=options) + if not options['summary']: - # Update list of cycles to correct indices - update_cycles_list(data=data, options=options) - - colours = generate_colours(cycles=data['cycles'], options=options) - - if options['interactive']: - options['interactive'], options['interactive_session_active'] = False, True - plot_gc_interactive(data=data, options=options) - return - - - # Prepare plot, and read and process data - fig, ax = btp.prepare_plot(options=options) - for i, cycle in enumerate(data['cycles']): if i in options['which_cycles']: if options['charge']: @@ -70,13 +70,18 @@ def plot_gc(data, options=None): else: update_labels(options) - fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) elif options['summary']: - fig, ax = 0, 0 + # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. + if options['charge']: + data['cycles'][0].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', s=plt.rcParams['lines.markersize']) + data['cycles'][1].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', s=plt.rcParams['lines.markersize']) + + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + return data['cycles'], fig, ax From 0e053ea1e2de77e403900ce13cf95b6b2119da38 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 10:03:49 +0200 Subject: [PATCH 245/355] Make ticks face in by default --- nafuma/plotting.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 4bf0ed9..367b74c 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -182,8 +182,13 @@ def adjust_plot(fig, ax, options): # Hide x- and y-ticks: if options['hide_y_ticks']: ax.tick_params(axis='y', direction='in', which='both', left=False, right=False) + else: + ax.tick_params(axis='y', direction='in', which='both', left=True, right=True) + if options['hide_x_ticks']: ax.tick_params(axis='x', direction='in', which='both', bottom=False, top=False) + else: + ax.tick_params(axis='x', direction='in', which='both', bottom=True, top=True) From 130e1206903ae7740ebdc9bc00c6d3c6db30cb0c Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 10:04:49 +0200 Subject: [PATCH 246/355] Chg and dchg of summary can be plotted independent --- nafuma/electrochemistry/plot.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 2d26b4b..4a3a1ee 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -75,8 +75,17 @@ def plot_gc(data, options=None): # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. if options['charge']: - data['cycles'][0].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', s=plt.rcParams['lines.markersize']) - data['cycles'][1].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', s=plt.rcParams['lines.markersize']) + data['cycles'][0].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + # + + if options['discharge']: + data['cycles'][1].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + + + if options['interactive_session_active']: + update_labels(options, force=True) + else: + update_labels(options) From 3a6a000b14c89ceb64366a6084d9681b6a005156 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 13:24:39 +0200 Subject: [PATCH 247/355] Allow older formats and manual incrementing of cycles --- nafuma/electrochemistry/io.py | 67 ++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index b4213ee..328cf24 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -258,14 +258,15 @@ def process_neware_data(df, options={}): active_materiale_weight: weight of the active material (in mg) used in the cell. molecular_weight: the molar mass (in g mol^-1) of the active material, to calculate the number of ions extracted. Assumes one electron per Li+/Na+-ion """ - required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles'] + required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles', 'increment_cycles_from'] default_options = { 'units': None, 'active_material_weight': None, 'molecular_weight': None, 'reverse_discharge': False, - 'splice_cycles': None} + 'splice_cycles': None, + 'increment_cycles_from': None} # index aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -281,6 +282,9 @@ def process_neware_data(df, options={}): df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units + if options['increment_cycles_from']: + df['cycle'].iloc[options['increment_cycles_from']:] += 1 + if options['splice_cycles']: df = splice_cycles(df=df, options=options) @@ -460,6 +464,13 @@ def unit_conversion(df, options): if options['kind'] == 'neware': + + + record_number = 'Data serial number' if 'Data serial number' in df.columns else 'Record number' + relative_time = 'Relative Time(h:min:s.ms)' if 'Relative Time(h:min:s.ms)' in df.columns else 'Relative Time' + continuous_time = 'Continuous Time(h:min:s.ms)' if 'Continuous Time(h:min:s.ms)' in df.columns else 'Continuous Time' + real_time = 'Real Time(h:min:s:ms)' if 'Real Time(h:min:s:ms)' in df.columns else 'Real Time' + if options['summary']: # Add the charge and discharge energy columns to get a single energy column @@ -469,29 +480,31 @@ def unit_conversion(df, options): df[f'Start Volt({options["old_units"]["voltage"]})'] = df[f'Start Volt({options["old_units"]["voltage"]})'] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] df[f'Capacity({options["old_units"]["capacity"]})'] = df[f'Capacity({options["old_units"]["capacity"]})'] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] df[f'Energy({options["old_units"]["energy"]})'] = df[f'Energy({options["old_units"]["energy"]})'] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] - df[f'CycleTime({options["units"]["time"]})'] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=options['units']['time']), axis=1) - df[f'RunTime({options["units"]["time"]})'] = df.apply(lambda row : convert_datetime_string(row['Real Time'], reference=df['Real Time'].iloc[0], ref_time=df[f'CycleTime({options["units"]["time"]})'].iloc[0],unit=options['units']['time']), axis=1) + df[f'CycleTime({options["units"]["time"]})'] = df.apply(lambda row : convert_time_string(row[relative_time], unit=options['units']['time']), axis=1) + df[f'RunTime({options["units"]["time"]})'] = df.apply(lambda row : convert_datetime_string(row[real_time], reference=df[real_time].iloc[0], ref_time=df[f'CycleTime({options["units"]["time"]})'].iloc[0],unit=options['units']['time']), axis=1) + + droplist = [ + 'Chnl', + 'Original step', + f'End Volt({options["old_units"]["voltage"]})', + f'Termination current({options["old_units"]["current"]})', + relative_time, + real_time, + continuous_time, + f'Net discharge capacity({options["old_units"]["capacity"]})', + f'Chg Cap({options["old_units"]["capacity"]})', + f'DChg Cap({options["old_units"]["capacity"]})', + f'Net discharge energy({options["old_units"]["energy"]})', + f'Chg Eng({options["old_units"]["energy"]})', + f'DChg Eng({options["old_units"]["energy"]})' + ] + # Drop all undesireable columns - df.drop( - [ - 'Chnl', - 'Original step', - f'End Volt({options["old_units"]["voltage"]})', - f'Termination current({options["old_units"]["current"]})', - 'Relative Time(h:min:s.ms)', - 'Real Time', - 'Continuous Time(h:min:s.ms)', - f'Net discharge capacity({options["old_units"]["capacity"]})', - f'Chg Cap({options["old_units"]["capacity"]})', - f'DChg Cap({options["old_units"]["capacity"]})', - f'Net discharge energy({options["old_units"]["energy"]})', - f'Chg Eng({options["old_units"]["energy"]})', - f'DChg Eng({options["old_units"]["energy"]})' - ], - axis=1, inplace=True - ) + for drop in droplist: + if drop in df.columns: + df.drop(drop, axis=1, inplace=True) columns = ['cycle', 'steps', 'status', 'voltage', 'current', 'capacity'] @@ -520,8 +533,8 @@ def unit_conversion(df, options): df['Voltage({})'.format(options['old_units']['voltage'])] = df['Voltage({})'.format(options['old_units']['voltage'])] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] df['Capacity({})'.format(options['old_units']['capacity'])] = df['Capacity({})'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] df['Energy({})'.format(options['old_units']['energy'])] = df['Energy({})'.format(options['old_units']['energy'])] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] - df['CycleTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_time_string(row['Relative Time(h:min:s.ms)'], unit=options['units']['time']), axis=1) - df['RunTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_datetime_string(row['Real Time(h:min:s.ms)'], reference=df['Real Time(h:min:s.ms)'].iloc[0], ref_time=df[f'CycleTime({options["units"]["time"]})'].iloc[0], unit=options['units']['time']), axis=1) + df['CycleTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_time_string(row[relative_time], unit=options['units']['time']), axis=1) + df['RunTime({})'.format(options['units']['time'])] = df.apply(lambda row : convert_datetime_string(row[real_time], reference=df[real_time].iloc[0], ref_time=df[f'CycleTime({options["units"]["time"]})'].iloc[0], unit=options['units']['time']), axis=1) columns = ['status', 'jump', 'cycle', 'steps', 'current', 'voltage', 'capacity', 'energy'] if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: @@ -535,7 +548,11 @@ def unit_conversion(df, options): columns.append('time') - df.drop(['Record number', 'Relative Time(h:min:s.ms)', 'Real Time(h:min:s.ms)'], axis=1, inplace=True) + droplist = [record_number, relative_time, real_time] + + for drop in droplist: + if drop in df.columns: + df.drop(drop, axis=1, inplace=True) df.columns = columns From 116e65e0e1649d73184d7c23ad8a51d2b5e5ee16 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 13:24:55 +0200 Subject: [PATCH 248/355] Make 'which_cycles' work for summaries --- nafuma/electrochemistry/plot.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 4a3a1ee..ee8284c 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -73,13 +73,21 @@ def plot_gc(data, options=None): elif options['summary']: + mask = [] + for i in range(data['cycles'][0].shape[0]): + if i+1 in options['which_cycles']: + mask.append(True) + else: + mask.append(False) + + # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. if options['charge']: - data['cycles'][0].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + data['cycles'][0].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) # if options['discharge']: - data['cycles'][1].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + data['cycles'][1].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) if options['interactive_session_active']: From e6a4e2c81f5c72544d3894b925fa4faf8eac520f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 13:48:29 +0200 Subject: [PATCH 249/355] Allow deletion of certain datapoints from the datasets --- nafuma/electrochemistry/io.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 328cf24..d3d6ad9 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -266,7 +266,9 @@ def process_neware_data(df, options={}): 'molecular_weight': None, 'reverse_discharge': False, 'splice_cycles': None, - 'increment_cycles_from': None} # index + 'increment_cycles_from': None,# index + 'delete_datapoints': None, # list of indices + } aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -282,9 +284,15 @@ def process_neware_data(df, options={}): df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units + print(df.iloc[1:10]) + if options['increment_cycles_from']: df['cycle'].iloc[options['increment_cycles_from']:] += 1 + if options['delete_datapoints']: + for datapoint in options['delete_datapoints']: + df.drop(index=datapoint, inplace=True) + if options['splice_cycles']: df = splice_cycles(df=df, options=options) From 8c20c029ae0c789de2884919a21f9fe78854b30f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 13:48:53 +0200 Subject: [PATCH 250/355] Fix bug --- nafuma/electrochemistry/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index d3d6ad9..a29d4a0 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -258,7 +258,7 @@ def process_neware_data(df, options={}): active_materiale_weight: weight of the active material (in mg) used in the cell. molecular_weight: the molar mass (in g mol^-1) of the active material, to calculate the number of ions extracted. Assumes one electron per Li+/Na+-ion """ - required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles', 'increment_cycles_from'] + required_options = ['units', 'active_material_weight', 'molecular_weight', 'reverse_discharge', 'splice_cycles', 'increment_cycles_from', 'delete_datapoints'] default_options = { 'units': None, From 43663331f19addc7e97d27b8fa061dc44697b52d Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 13:49:09 +0200 Subject: [PATCH 251/355] Remove print statement --- nafuma/electrochemistry/io.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index a29d4a0..eec472b 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -284,8 +284,6 @@ def process_neware_data(df, options={}): df = unit_conversion(df=df, options=options) # converts all units from the old units to the desired units - print(df.iloc[1:10]) - if options['increment_cycles_from']: df['cycle'].iloc[options['increment_cycles_from']:] += 1 From 4269ac5d4661128f2d9d9be823994148f773b999 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 21:55:42 +0200 Subject: [PATCH 252/355] Fix formatting error in unit conversion --- nafuma/electrochemistry/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index eec472b..26867bd 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -475,7 +475,7 @@ def unit_conversion(df, options): record_number = 'Data serial number' if 'Data serial number' in df.columns else 'Record number' relative_time = 'Relative Time(h:min:s.ms)' if 'Relative Time(h:min:s.ms)' in df.columns else 'Relative Time' continuous_time = 'Continuous Time(h:min:s.ms)' if 'Continuous Time(h:min:s.ms)' in df.columns else 'Continuous Time' - real_time = 'Real Time(h:min:s:ms)' if 'Real Time(h:min:s:ms)' in df.columns else 'Real Time' + real_time = 'Real Time(h:min:s.ms)' if 'Real Time(h:min:s.ms)' in df.columns else 'Real Time' if options['summary']: From b6780e8a90b63b1c4dbb81676e2d5c22f4e32144 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 2 Aug 2022 21:55:55 +0200 Subject: [PATCH 253/355] Add option to make animation of GC plots --- nafuma/electrochemistry/plot.py | 89 ++++++++++++++++++++++++++------- 1 file changed, 70 insertions(+), 19 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index ee8284c..5c150c8 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -4,6 +4,9 @@ from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLoca import pandas as pd import numpy as np import math +import os +import shutil +from PIL import Image import ipywidgets as widgets from IPython.display import display @@ -19,11 +22,12 @@ def plot_gc(data, options=None): # Update options - required_options = ['x_vals', 'y_vals', 'which_cycles', 'exclude_cycles', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params'] + required_options = ['x_vals', 'y_vals', 'which_cycles', 'exclude_cycles', 'show_plot', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] default_options = { 'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', 'exclude_cycles': [], + 'show_plot': True, 'charge': True, 'discharge': True, 'colours': None, 'differentiate_charge_discharge': True, @@ -31,7 +35,11 @@ def plot_gc(data, options=None): 'interactive': False, 'interactive_session_active': False, 'rc_params': {}, - 'format_params': {}} + 'format_params': {}, + 'save_gif': False, + 'save_path': 'animation.gif', + 'fps': 1 + } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -50,28 +58,69 @@ def plot_gc(data, options=None): return - # Prepare plot - fig, ax = btp.prepare_plot(options=options) - colours = generate_colours(cycles=data['cycles'], options=options) + colours = generate_colours(cycles=data['cycles'], options=options) if not options['summary']: - for i, cycle in enumerate(data['cycles']): - if i in options['which_cycles']: - if options['charge']: - cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + if options['show_plot']: + # Prepare plot + fig, ax = btp.prepare_plot(options=options) + for i, cycle in enumerate(data['cycles']): + if i in options['which_cycles']: + if options['charge']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) - if options['discharge']: - cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + if options['discharge']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) - if options['interactive_session_active']: - update_labels(options, force=True) - else: - update_labels(options) + if options['interactive_session_active']: + update_labels(options, force=True) + else: + update_labels(options) + + + + if options['save_gif'] and not options['interactive_session_active']: + if not os.path.isdir('tmp'): + os.makedirs('tmp') + + for i, cycle in enumerate(data['cycles']): + if i in options['which_cycles']: + giffig, gifax = btp.prepare_plot(options=options) + + if options['charge']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=gifax, c=colours[i][0]) + if options['discharge']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=gifax, c=colours[i][1]) + + gifax.text(x=options['xlim'][1]*0.8, y=3, s=f'{i+1}') + update_labels(options) + + giffig, gifax = btp.adjust_plot(fig=giffig, ax=gifax, options=options) + + plt.savefig(os.path.join('tmp', str(i+1).zfill(4)+'.png')) + plt.close() + + + img_paths = [os.path.join('tmp', path) for path in os.listdir('tmp') if path.endswith('png')] + frames = [] + for path in img_paths: + frame = Image.open(path) + frames.append(frame) + + + frames[0].save(options['save_path'], format='GIF', append_images=frames[1:], save_all=True, duration=len(data['cycles'])/options['fps'], loop=0) + + shutil.rmtree('tmp') - elif options['summary']: + + + elif options['summary'] and options['show_plot']: + # Prepare plot + fig, ax = btp.prepare_plot(options=options) + mask = [] for i in range(data['cycles'][0].shape[0]): @@ -97,9 +146,11 @@ def plot_gc(data, options=None): - fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) - - return data['cycles'], fig, ax + if options['show_plot']: + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + return data['cycles'], fig, ax + else: + return data['cycles'], None, None def plot_gc_interactive(data, options): From 642f166d718471616fa44ec9424e93d4bc427631 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 4 Aug 2022 18:57:27 +0200 Subject: [PATCH 254/355] Constrain size for GIFs, fix fps and incomp cycles --- nafuma/electrochemistry/plot.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 5c150c8..b77d6f4 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -85,8 +85,15 @@ def plot_gc(data, options=None): if not os.path.isdir('tmp'): os.makedirs('tmp') + # Scale image to make GIF smaller + options['format_params']['width'] = 7.5 + options['format_params']['height'] = 3 + + options['format_params']['dpi'] = 200 + for i, cycle in enumerate(data['cycles']): if i in options['which_cycles']: + giffig, gifax = btp.prepare_plot(options=options) if options['charge']: @@ -109,8 +116,7 @@ def plot_gc(data, options=None): frame = Image.open(path) frames.append(frame) - - frames[0].save(options['save_path'], format='GIF', append_images=frames[1:], save_all=True, duration=len(data['cycles'])/options['fps'], loop=0) + frames[0].save(options['save_path'], format='GIF', append_images=frames[1:], save_all=True, duration=(1/options['fps'])*1000, loop=0) shutil.rmtree('tmp') @@ -129,6 +135,12 @@ def plot_gc(data, options=None): else: mask.append(False) + if len(mask) > len(data['cycles'][1]): + del mask[-1] + data['cycles'][0].drop(data['cycles'][0].tail(1).index, inplace=True) + + + # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. if options['charge']: From 745522ed82daef3294b4b116db92e28824551aa3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 5 Aug 2022 09:51:28 +0200 Subject: [PATCH 255/355] Calculate specific energy in Neware data --- nafuma/electrochemistry/io.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 26867bd..9b7b3d6 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -342,6 +342,7 @@ def process_neware_data(df, options={}): df = add_columns(df=df, options=options) df = unit_conversion(df=df, options=options) + #df = calculate_efficiency(df=df, options=options) if options['splice_cycles']: df = splice_cycles(df=df, options=options) @@ -427,8 +428,13 @@ def process_biologic_data(df, options=None): def add_columns(df, options): if options['kind'] == 'neware': + + if options['summary']: + df[f'Energy({options["old_units"]["energy"]})'] = np.abs(df[f'Net discharge energy({options["old_units"]["energy"]})']) + if options['active_material_weight']: - df["SpecificCapacity({}/mg)".format(options['old_units']["capacity"])] = df["Capacity({})".format(options['old_units']['capacity'])] / (options['active_material_weight']) + df[f"SpecificCapacity({options['old_units']['capacity']}/mg)"] = df["Capacity({})".format(options['old_units']['capacity'])] / (options['active_material_weight']) + df[f"SpecificEnergy({options['old_units']['energy']}/mg)"] = df["Energy({})".format(options['old_units']['energy'])] / (options['active_material_weight']) if options['molecular_weight']: faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 @@ -456,6 +462,11 @@ def add_columns(df, options): return df +#def calculate_efficiency(df, options): +# +# df['coulombic_efficiency'] = + + def unit_conversion(df, options): from . import unit_tables @@ -479,9 +490,7 @@ def unit_conversion(df, options): if options['summary']: - # Add the charge and discharge energy columns to get a single energy column - df[f'Energy({options["old_units"]["energy"]})'] = df[f'Chg Eng({options["old_units"]["energy"]})'] + df[f'DChg Eng({options["old_units"]["energy"]})'] - + df[f'Energy({options["old_units"]["energy"]})'] = df[f'Energy({options["old_units"]["energy"]})'] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] df[f'Starting current({options["old_units"]["current"]})'] = df[f'Starting current({options["old_units"]["current"]})'] * unit_tables.current()[options['old_units']['current']].loc[options['units']['current']] df[f'Start Volt({options["old_units"]["voltage"]})'] = df[f'Start Volt({options["old_units"]["voltage"]})'] * unit_tables.voltage()[options['old_units']['voltage']].loc[options['units']['voltage']] df[f'Capacity({options["old_units"]["capacity"]})'] = df[f'Capacity({options["old_units"]["capacity"]})'] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] @@ -512,9 +521,7 @@ def unit_conversion(df, options): if drop in df.columns: df.drop(drop, axis=1, inplace=True) - columns = ['cycle', 'steps', 'status', 'voltage', 'current', 'capacity'] - - + columns = ['cycle', 'steps', 'status', 'voltage', 'current', 'capacity', 'energy'] # Add column labels for specific capacity and ions if they exist @@ -522,15 +529,19 @@ def unit_conversion(df, options): df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] columns.append('specific_capacity') + if f'SpecificEnergy({options["old_units"]["energy"]}/mg)' in df.columns: + df[f'SpecificEnergy({options["old_units"]["energy"]}/mg)'] = df[f'SpecificEnergy({options["old_units"]["energy"]}/mg)'] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] + columns.append('specific_energy') + if 'IonsExtracted' in df.columns: columns.append('ions') # Append energy column label here as it was the last column to be generated - columns.append('energy') columns.append('cycle_time') columns.append('runtime') # Apply new column labels + print(df.columns, columns) df.columns = columns From b8ce2b64cc0373e540de3225a8f46122d4310de5 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 5 Aug 2022 10:35:35 +0200 Subject: [PATCH 256/355] Add efficiency calculations and new format of summary df --- nafuma/electrochemistry/io.py | 39 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 9b7b3d6..31626a6 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -342,18 +342,31 @@ def process_neware_data(df, options={}): df = add_columns(df=df, options=options) df = unit_conversion(df=df, options=options) - #df = calculate_efficiency(df=df, options=options) + if options['splice_cycles']: df = splice_cycles(df=df, options=options) chg_df = df.loc[df['status'] == 'CC Chg'] + chg_df.reset_index(inplace=True) dchg_df = df.loc[df['status'] == 'CC DChg'] + dchg_df.reset_index(inplace=True) - cycles = [chg_df, dchg_df] + # Construct new DataFrame + new_df = pd.DataFrame(chg_df["cycle"]) + new_df.insert(1,'charge_capacity',chg_df['capacity']) + new_df.insert(1,'charge_specific_capacity',chg_df['specific_capacity']) + new_df.insert(1,'discharge_capacity',dchg_df['capacity']) + new_df.insert(1,'discharge_specific_capacity',dchg_df['specific_capacity']) + new_df.insert(1,'charge_energy',chg_df['energy']) + new_df.insert(1,'charge_specific_energy',chg_df['specific_energy']) + new_df.insert(1,'discharge_energy',dchg_df['energy']) + new_df.insert(1,'discharge_specific_energy',dchg_df['specific_energy']) - return cycles + new_df = calculate_efficiency(df=new_df, options=options) + + return new_df @@ -462,9 +475,23 @@ def add_columns(df, options): return df -#def calculate_efficiency(df, options): -# -# df['coulombic_efficiency'] = +def calculate_efficiency(df: pd.DataFrame, options: dict) -> pd.DataFrame: + + + default_options = { + 'reference_index': 0 + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + df['charge_capacity_fade'] = (df['charge_capacity'] / df['charge_capacity'].iloc[options['reference_index']])*100 + df['discharge_capacity_fade'] = (df['discharge_capacity'] / df['discharge_capacity'].iloc[options['reference_index']])*100 + + df['coulombic_efficiency'] = (df['discharge_capacity'] / df['charge_capacity'])*100 + df['energy_efficiency'] = (df['discharge_energy'] / df['charge_energy'])*100 + + + return df def unit_conversion(df, options): From 790048098a7ef75aea69ec56f2cb50aa1037a706 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 5 Aug 2022 10:35:59 +0200 Subject: [PATCH 257/355] Make plotting match new format of summary df --- nafuma/electrochemistry/plot.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index b77d6f4..5cdf869 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -127,28 +127,29 @@ def plot_gc(data, options=None): # Prepare plot fig, ax = btp.prepare_plot(options=options) - mask = [] - for i in range(data['cycles'][0].shape[0]): + for i in range(data['cycles'].shape[0]): if i+1 in options['which_cycles']: mask.append(True) else: mask.append(False) - if len(mask) > len(data['cycles'][1]): + + # Drop the last row if it is midway through a charge in order to avoid mismatch of length of mask and dataset. + if len(mask) > data['cycles'].shape[0]: del mask[-1] - data['cycles'][0].drop(data['cycles'][0].tail(1).index, inplace=True) + data['cycles'].drop(data['cycles'].tail(1).index, inplace=True) - - # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. if options['charge']: - data['cycles'][0].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + yval = 'charge_' + options['x_vals'] + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) # if options['discharge']: - data['cycles'][1].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + yval = 'discharge_' + options['x_vals'] + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) if options['interactive_session_active']: From b69a4b95210265c412300b1e6143bd33f922b79a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 5 Aug 2022 11:01:09 +0200 Subject: [PATCH 258/355] Plot efficiencies without changing options --- nafuma/electrochemistry/plot.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 5cdf869..5039d0f 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -1,3 +1,4 @@ +from pickle import MARK import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) @@ -142,14 +143,23 @@ def plot_gc(data, options=None): # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. - if options['charge']: - yval = 'charge_' + options['x_vals'] - data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) - # + if options['x_vals'] in ['coulombic_efficiency', 'energy_efficiency']: + data['cycles'].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + if options['limit']: + ax.axhline(y=options['limit'], ls='--', c='black') + + else: + if options['charge']: + yval = 'charge_' + options['x_vals'] + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) - if options['discharge']: - yval = 'discharge_' + options['x_vals'] - data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + if options['discharge']: + yval = 'discharge_' + options['x_vals'] + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + + + if options['limit']: + ax.axhline(y=options['limit'], ls='--', c='black') if options['interactive_session_active']: From 9a73f57a8297b031c293e3304c59cb751eb83317 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 8 Aug 2022 15:44:55 +0200 Subject: [PATCH 259/355] Allow reading CV data from BioLogic --- nafuma/electrochemistry/io.py | 92 +++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 31626a6..7ad764d 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -6,6 +6,12 @@ import os import nafuma.auxillary as aux from sympy import re + +# FIXME This is not good practice, but a temporary fix as I don't have time to understand what causes the SettingWithCopyWarning. +# Read this: https://www.dataquest.io/blog/settingwithcopywarning/ +pd.set_option('mode.chained_assignment', None) + + def read_data(data, options={}): if data['kind'] == 'neware': @@ -382,11 +388,22 @@ def process_biologic_data(df, options=None): 'splice_cycles': None} + # Check if the DataFrame contains GC or CV data. + # FIXME This might not be a very rigorous method of checking. E.g. Rest has mode == 3, so if loading a short GC with many Rest-datapoints, the mean will be 2 and it will be treated as CV. For now manual override is sufficient + if not 'mode' in options.keys(): + options['mode'] = 'GC' if int(df['mode'].mean()) == 1 else 'CV' + aux.update_options(options=options, required_options=required_options, default_options=default_options) options['kind'] = 'biologic' # Pick out necessary columns - df = df[['Ns changes', 'Ns', 'time/s', 'Ewe/V', 'Energy charge/W.h', 'Energy discharge/W.h', '/mA', 'Capacity/mA.h', 'cycle number']].copy() + headers = [ + 'Ns changes', 'Ns', 'time/s', 'Ewe/V', 'Energy charge/W.h', 'Energy discharge/W.h', '/mA', 'Capacity/mA.h', 'cycle number' ] if options['mode'] == 'GC' else [ + 'ox/red', 'time/s', 'control/V', 'Ewe/V', '/mA', 'cycle number', '(Q-Qo)/C', 'P/W' + ] + + + df = df[headers].copy() # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. set_units(options) @@ -396,10 +413,15 @@ def process_biologic_data(df, options=None): df = unit_conversion(df=df, options=options) - # Creates masks for charge and discharge curves - chg_mask = (df['status'] == 1) & (df['status_change'] != 1) - dchg_mask = (df['status'] == 2) & (df['status_change'] != 1) + # Creates masks for charge and discharge curves + if options['mode'] == 'GC': + chg_mask = (df['status'] == 1) & (df['status_change'] != 1) + dchg_mask = (df['status'] == 2) & (df['status_change'] != 1) + + elif options['mode'] == 'CV': + chg_mask = (df['status'] == 1) # oxidation + dchg_mask = (df['status'] == 0) # reduction # Initiate cycles list cycles = [] @@ -419,7 +441,7 @@ def process_biologic_data(df, options=None): if chg_df.empty and dchg_df.empty: continue - if options['reverse_discharge']: + if options['mode'] == 'GC' and options['reverse_discharge']: max_capacity = dchg_df['capacity'].max() dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) @@ -431,10 +453,14 @@ def process_biologic_data(df, options=None): max_capacity = dchg_df['ions'].max() dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) + + if options['mode'] == 'CV': + chg_df = chg_df.sort_values(by='voltage').reset_index(drop=True) + dchg_df = dchg_df.sort_values(by='voltage', ascending=False).reset_index(drop=True) + cycles.append((chg_df, dchg_df)) - return cycles @@ -568,7 +594,6 @@ def unit_conversion(df, options): columns.append('runtime') # Apply new column labels - print(df.columns, columns) df.columns = columns @@ -585,6 +610,11 @@ def unit_conversion(df, options): df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] columns.append('specific_capacity') + if f'SpecificEnergy({options["old_units"]["energy"]}/mg)' in df.columns: + df[f'SpecificEnergy({options["old_units"]["energy"]}/mg)'] = df[f'SpecificEnergy({options["old_units"]["energy"]}/mg)'] * unit_tables.energy()[options['old_units']['energy']].loc[options['units']['energy']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] + columns.append('specific_energy') + + if 'IonsExtracted' in df.columns: columns.append('ions') @@ -601,21 +631,34 @@ def unit_conversion(df, options): df.columns = columns if options['kind'] == 'biologic': - df['time/{}'.format(options['old_units']['time'])] = df["time/{}".format(options['old_units']["time"])] * unit_tables.time()[options['old_units']["time"]].loc[options['units']['time']] - df["Ewe/{}".format(options['old_units']["voltage"])] = df["Ewe/{}".format(options['old_units']["voltage"])] * unit_tables.voltage()[options['old_units']["voltage"]].loc[options['units']['voltage']] - df["/{}".format(options['old_units']["current"])] = df["/{}".format(options['old_units']["current"])] * unit_tables.current()[options['old_units']["current"]].loc[options['units']['current']] + for column in df.columns: + if 'time' in column: + df['time/{}'.format(options['old_units']['time'])] = df["time/{}".format(options['old_units']["time"])] * unit_tables.time()[options['old_units']["time"]].loc[options['units']['time']] + + if 'Ewe' in column: + df["Ewe/{}".format(options['old_units']["voltage"])] = df["Ewe/{}".format(options['old_units']["voltage"])] * unit_tables.voltage()[options['old_units']["voltage"]].loc[options['units']['voltage']] + + if '' in column: + df["/{}".format(options['old_units']["current"])] = df["/{}".format(options['old_units']["current"])] * unit_tables.current()[options['old_units']["current"]].loc[options['units']['current']] - capacity = options['old_units']['capacity'].split('h')[0] + '.h' - df["Capacity/{}".format(capacity)] = df["Capacity/{}".format(capacity)] * (unit_tables.capacity()[options['old_units']["capacity"]].loc[options['units']["capacity"]]) + if 'Capacity' in column: + capacity = options['old_units']['capacity'].split('h')[0] + '.h' + df["Capacity/{}".format(capacity)] = df["Capacity/{}".format(capacity)] * (unit_tables.capacity()[options['old_units']["capacity"]].loc[options['units']["capacity"]]) - columns = ['status_change', 'status', 'time', 'voltage', 'energy_charge', 'energy_discharge', 'current', 'capacity', 'cycle'] + - if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: - df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] - columns.append('specific_capacity') + columns = [ + 'status_change', 'status', 'time', 'voltage', 'energy_charge', 'energy_discharge', 'current', 'capacity', 'cycle'] if options['mode'] == 'GC' else [ # GC headers + 'status', 'time', 'control_voltage', 'voltage', 'current', 'cycle', 'charge', 'power' # CV headers + ] - if 'IonsExtracted' in df.columns: - columns.append('ions') + if options['mode'] == 'GC': + if 'SpecificCapacity({}/mg)'.format(options['old_units']['capacity']) in df.columns: + df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] = df['SpecificCapacity({}/mg)'.format(options['old_units']['capacity'])] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] / unit_tables.mass()['mg'].loc[options['units']["mass"]] + columns.append('specific_capacity') + + if 'IonsExtracted' in df.columns: + columns.append('ions') df.columns = columns @@ -674,19 +717,18 @@ def get_old_units(df: pd.DataFrame, options: dict) -> dict: if options['kind'] == 'biologic': + old_units = {} for column in df.columns: if 'time' in column: - time = column.split('/')[-1] + old_units['time'] = column.split('/')[-1] elif 'Ewe' in column: - voltage = column.split('/')[-1] + old_units['voltage'] = column.split('/')[-1] elif 'Capacity' in column: - capacity = column.split('/')[-1].replace('.', '') + old_units['capacity'] = column.split('/')[-1].replace('.', '') elif 'Energy' in column: - energy = column.split('/')[-1].replace('.', '') + old_units['energy'] = column.split('/')[-1].replace('.', '') elif '' in column: - current = column.split('/')[-1] - - old_units = {'voltage': voltage, 'current': current, 'capacity': capacity, 'energy': energy, 'time': time} + old_units['current'] = column.split('/')[-1] return old_units From df5190667eed1dd0d9a7eb29ef7955ec8d2e7bd8 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 8 Aug 2022 15:45:09 +0200 Subject: [PATCH 260/355] Ad plotting of CV-data --- nafuma/electrochemistry/plot.py | 109 +++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 5039d0f..7a0246c 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -23,10 +23,12 @@ def plot_gc(data, options=None): # Update options - required_options = ['x_vals', 'y_vals', 'which_cycles', 'exclude_cycles', 'show_plot', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] + required_options = ['force_reload', 'x_vals', 'y_vals', 'which_cycles', 'limit', 'exclude_cycles', 'show_plot', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] default_options = { + 'force_reload': False, 'x_vals': 'capacity', 'y_vals': 'voltage', 'which_cycles': 'all', + 'limit': None, # Limit line to be drawn 'exclude_cycles': [], 'show_plot': True, 'charge': True, 'discharge': True, @@ -46,7 +48,7 @@ def plot_gc(data, options=None): # Read data if not already loaded - if not 'cycles' in data.keys(): + if not 'cycles' in data.keys() or options['force_reload']: data['cycles'] = ec.io.read_data(data=data, options=options) @@ -102,7 +104,7 @@ def plot_gc(data, options=None): if options['discharge']: cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=gifax, c=colours[i][1]) - gifax.text(x=options['xlim'][1]*0.8, y=3, s=f'{i+1}') + gifax.text(x=gifax.get_xlim()[1]*0.8, y=3, s=f'{i+1}') update_labels(options) giffig, gifax = btp.adjust_plot(fig=giffig, ax=gifax, options=options) @@ -189,6 +191,107 @@ def plot_gc_interactive(data, options): display(w) + + +def plot_cv(data, options): + + # Update options + required_options = ['force_reload', 'x_vals', 'y_vals', 'which_cycles', 'limit', 'exclude_cycles', 'show_plot', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] + default_options = { + 'force_reload': False, + 'x_vals': 'voltage', 'y_vals': 'current', + 'which_cycles': 'all', + 'limit': None, # Limit line to be drawn + 'exclude_cycles': [], + 'show_plot': True, + 'charge': True, 'discharge': True, + 'colours': None, + 'differentiate_charge_discharge': True, + 'gradient': False, + 'interactive': False, + 'interactive_session_active': False, + 'rc_params': {}, + 'format_params': {}, + 'save_gif': False, + 'save_path': 'animation.gif', + 'fps': 1 + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + # Read data if not already loaded + if not 'cycles' in data.keys() or options['force_reload']: + data['cycles'] = ec.io.read_data(data=data, options=options) + + + # Update list of cycles to correct indices + update_cycles_list(data=data, options=options) + + colours = generate_colours(cycles=data['cycles'], options=options) + + if options['show_plot']: + # Prepare plot + fig, ax = btp.prepare_plot(options=options) + for i, cycle in enumerate(data['cycles']): + if i in options['which_cycles']: + if options['charge']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + + if options['discharge']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + + update_labels(options) + + + + if options['save_gif'] and not options['interactive_session_active']: + if not os.path.isdir('tmp'): + os.makedirs('tmp') + + # Scale image to make GIF smaller + options['format_params']['width'] = 7.5 + options['format_params']['height'] = 3 + + options['format_params']['dpi'] = 200 + + for i, cycle in enumerate(data['cycles']): + if i in options['which_cycles']: + + giffig, gifax = btp.prepare_plot(options=options) + + if options['charge']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=gifax, c=colours[i][0]) + if options['discharge']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=gifax, c=colours[i][1]) + + gifax.text(x=gifax.get_xlim()[1]*0.8, y=3, s=f'{i+1}') + update_labels(options) + + giffig, gifax = btp.adjust_plot(fig=giffig, ax=gifax, options=options) + + plt.savefig(os.path.join('tmp', str(i+1).zfill(4)+'.png')) + plt.close() + + + img_paths = [os.path.join('tmp', path) for path in os.listdir('tmp') if path.endswith('png')] + frames = [] + for path in img_paths: + frame = Image.open(path) + frames.append(frame) + + frames[0].save(options['save_path'], format='GIF', append_images=frames[1:], save_all=True, duration=(1/options['fps'])*1000, loop=0) + + shutil.rmtree('tmp') + + + + if options['show_plot']: + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + return data['cycles'], fig, ax + else: + return data['cycles'], None, None + def update_labels(options, force=False): if 'xlabel' not in options.keys() or force: From 9cda7694d0846236077dac113f6f7ef5787f7278 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 8 Aug 2022 16:40:45 +0200 Subject: [PATCH 261/355] Add choice to load transmission data --- nafuma/xanes/io.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index a842418..bdf268c 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -189,9 +189,10 @@ def read_data(data: dict, options={}) -> pd.DataFrame: # FIXME Handle the case when dataseries are not the same size # FIXME Add possibility to extract TIME (for operando runs) and Blower Temp (for variable temperature runs) # FIXME Add possibility to iport transmission data - required_options = ['adjust'] + required_options = ['adjust', 'mode'] default_options = { - 'adjust': 0 + 'adjust': 0, + 'mode': 'fluoresence' } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -211,11 +212,15 @@ def read_data(data: dict, options={}) -> pd.DataFrame: scan_data = pd.read_csv(filename, skiprows=1) - if not options['active_roi']: - scan_data = scan_data[[determine_active_roi(scan_data)]] - else: - scan_data = scan_data[options['active_roi']] - + if options['mode'] == 'fluoresence': + if not options['active_roi']: + scan_data = scan_data[[determine_active_roi(scan_data)]] + else: + scan_data = scan_data[options['active_roi']] + + elif options['mode'] == 'transmission': + scan_data = scan_data['Ion2'] / scan_data['Ion1'] + xanes_data = pd.concat([xanes_data, scan_data], axis=1) From 731ea70f77f490f31391cff728c697752324dce3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 9 Aug 2022 16:26:49 +0200 Subject: [PATCH 262/355] Add functions to save and load data --- nafuma/xanes/io.py | 73 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index bdf268c..70c3ad8 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -81,10 +81,19 @@ def split_scan_data(data: dict, options={}) -> list: for i, scan_data in enumerate(scan_datas): - + + if 'ZapEnergy' not in headers[i]: + if options['log']: + aux.write_log(message=f'... No valid scan data found... ({i+1}/{len(scan_datas)})', options=options) + continue + xanes_df = pd.DataFrame(scan_data).apply(pd.to_numeric) xanes_df.columns = headers[i] + + edge = find_element({'xanes_data_original': xanes_df}) + + if options['log']: aux.write_log(message=f'... Starting data clean-up ({edge}-edge)... ({i+1}/{len(scan_datas)})', options=options) @@ -183,6 +192,66 @@ def split_scan_data(data: dict, options={}) -> list: +def save_data(data: dict, options={}) -> None: + + required_options = ['save_folder', 'overwrite', 'log', 'logfile', 'filename'] + + default_options = { + 'log': False, + 'logfile': f'{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_save_files.log', + 'save_folder': 'saved_scans', + 'overwrite': False, + 'filename': f'{datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_exported_data.dat', + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + # Check if there is any data to be saved + if not 'xanes_data' in data.keys(): + if options['log']: + aux.write_log(message=f'There is not saved scan data in data. Exiting without saving...', options=options) + + return None + + if not isinstance(data['xanes_data'], pd.DataFrame): + if options['log']: + aux.write_log(message=f'data["xanes_data"] has an invalid format. Exiting without saving...', options=options) + + return None + + + # Make folder(s) if it/they do(es)n't exist + if not os.path.exists(options['save_folder']): + if options['log']: + aux.write_log(message=f'Destination folder does not exist. Creating folder...', options=options) + + os.makedirs(options['save_folder']) + + + + if os.path.exists(os.path.join('save_folder', options['filename'])): + if not options['overwrite']: + if options['log']: + aux.write_log(message=f'File already exists and overwrite disabled. Exiting without saving...', options=options) + return None + + data['xanes_data'].to_csv(os.path.join(options['save_folder'], options['filename']), sep='\t', index=False) + + + +def load_data(path: str): + + data = {} + + data['xanes_data'] = pd.read_csv(path, sep='\t') + data['path'] = data['xanes_data'].columns.to_list() + data['path'].remove('ZapEnergy') + + + return data + + def read_data(data: dict, options={}) -> pd.DataFrame: @@ -219,7 +288,7 @@ def read_data(data: dict, options={}) -> pd.DataFrame: scan_data = scan_data[options['active_roi']] elif options['mode'] == 'transmission': - scan_data = scan_data['Ion2'] / scan_data['Ion1'] + scan_data = scan_data['MonEx'] / scan_data['Ion2'] xanes_data = pd.concat([xanes_data, scan_data], axis=1) From b40b023cbc656fe272cabf81f2e1879196801511 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 9 Aug 2022 16:41:39 +0200 Subject: [PATCH 263/355] Let load_data conform to data/options-standard --- nafuma/xanes/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 70c3ad8..d7e9f94 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -240,11 +240,12 @@ def save_data(data: dict, options={}) -> None: -def load_data(path: str): +def load_data(data: dict, options={}) -> dict: + # FIXME Let this function be called by read_data() if some criterium is passed data = {} - data['xanes_data'] = pd.read_csv(path, sep='\t') + data['xanes_data'] = pd.read_csv(data['path'], sep='\t') data['path'] = data['xanes_data'].columns.to_list() data['path'].remove('ZapEnergy') From e67846d6f27968186ccc20aaacbd124646e8aa2e Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 9 Aug 2022 16:41:53 +0200 Subject: [PATCH 264/355] Initial commit of plot_xanes --- nafuma/xanes/plot.py | 95 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 nafuma/xanes/plot.py diff --git a/nafuma/xanes/plot.py b/nafuma/xanes/plot.py new file mode 100644 index 0000000..3341b9e --- /dev/null +++ b/nafuma/xanes/plot.py @@ -0,0 +1,95 @@ +import matplotlib.pyplot as plt +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) + +import pandas as pd +import numpy as np +import math + +#import ipywidgets as widgets +#from IPython.display import display + +import nafuma.xanes as xas +import nafuma.plotting as btp +import nafuma.auxillary as aux + + +def plot_xanes(data, options=None): + + + # Update options + required_options = ['which_scans', 'colours', 'gradient', 'rc_params', 'format_params'] + default_options = { + 'which_scans': 'all', + 'colours': None, + 'gradient': False, + 'rc_params': {}, + 'format_params': {}} + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + if not 'xanes_data' in data.keys(): + data['xanes_data'] = xas.io.load_data(data=data, options=options) + + # Update list of cycles to correct indices + update_scans_list(scans=data['path'], options=options) + + colours = generate_colours(cycles=data['cycles'], options=options) + + if options['interactive']: + options['interactive'], options['interactive_session_active'] = False, True + plot_gc_interactive(data=data, options=options) + return + + + # Prepare plot, and read and process data + + fig, ax = btp.prepare_plot(options=options) + + for i, cycle in enumerate(data['cycles']): + if i in options['which_cycles']: + if options['charge']: + cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + + if options['discharge']: + cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + + + if options['interactive_session_active']: + update_labels(options, force=True) + else: + update_labels(options) + + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + #if options['interactive_session_active']: + + + return data['cycles'], fig, ax + + + + + +def update_scans_list(scans, options: dict) -> None: + + if options['which_scans'] == 'all': + options['which_scans'] = [i for i in range(len(scans))] + + + elif type(options['which_scans']) == list: + options['which_scans'] = [i-1 for i in options['which_scans']] + + + # Tuple is used to define an interval - as elements tuples can't be assigned, I convert it to a list here. + elif type(options['which_cycles']) == tuple: + which_cycles = list(options['which_cycles']) + + if which_cycles[0] <= 0: + which_cycles[0] = 1 + + elif which_cycles[1] < 0: + which_cycles[1] = len(cycles) + + + options['which_cycles'] = [i-1 for i in range(which_cycles[0], which_cycles[1]+1)] \ No newline at end of file From 0a6d6826495b7b2cfec8ced4c38914630fff08f9 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 9 Aug 2022 17:35:41 +0200 Subject: [PATCH 265/355] Add working plot_xanes function --- nafuma/xanes/__init__.py | 2 +- nafuma/xanes/plot.py | 109 +++++++++++++++++++++++++++------------ 2 files changed, 78 insertions(+), 33 deletions(-) diff --git a/nafuma/xanes/__init__.py b/nafuma/xanes/__init__.py index a3834e8..af49ee6 100644 --- a/nafuma/xanes/__init__.py +++ b/nafuma/xanes/__init__.py @@ -1 +1 @@ -from . import io, calib, edges \ No newline at end of file +from . import io, calib, plot, edges \ No newline at end of file diff --git a/nafuma/xanes/plot.py b/nafuma/xanes/plot.py index 3341b9e..99e5d9f 100644 --- a/nafuma/xanes/plot.py +++ b/nafuma/xanes/plot.py @@ -17,9 +17,12 @@ def plot_xanes(data, options=None): # Update options - required_options = ['which_scans', 'colours', 'gradient', 'rc_params', 'format_params'] + required_options = ['which_scans', 'xlabel', 'ylabel', 'xunit', 'yunit', 'exclude_scans', 'colours', 'gradient', 'rc_params', 'format_params'] default_options = { 'which_scans': 'all', + 'xlabel': 'Energy', 'ylabel': 'Intensity', + 'xunit': 'keV', 'yunit': 'arb. u.', + 'exclude_scans': [], 'colours': None, 'gradient': False, 'rc_params': {}, @@ -27,69 +30,111 @@ def plot_xanes(data, options=None): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + print(options['xunit']) + if not 'xanes_data' in data.keys(): data['xanes_data'] = xas.io.load_data(data=data, options=options) # Update list of cycles to correct indices - update_scans_list(scans=data['path'], options=options) + update_scans_list(data=data, options=options) - colours = generate_colours(cycles=data['cycles'], options=options) - - if options['interactive']: - options['interactive'], options['interactive_session_active'] = False, True - plot_gc_interactive(data=data, options=options) - return + colours = generate_colours(scans=data['path'], options=options) # Prepare plot, and read and process data fig, ax = btp.prepare_plot(options=options) - for i, cycle in enumerate(data['cycles']): - if i in options['which_cycles']: - if options['charge']: - cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + for i, path in enumerate(data['path']): + if i in options['which_scans']: + data['xanes_data'].plot(x='ZapEnergy', y=path, ax=ax, c=colours[i]) - if options['discharge']: - cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) - - - if options['interactive_session_active']: - update_labels(options, force=True) - else: - update_labels(options) fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) #if options['interactive_session_active']: - return data['cycles'], fig, ax + return fig, ax -def update_scans_list(scans, options: dict) -> None: +def update_scans_list(data, options: dict) -> None: if options['which_scans'] == 'all': - options['which_scans'] = [i for i in range(len(scans))] + options['which_scans'] = [i for i in range(len(data['path']))] - elif type(options['which_scans']) == list: - options['which_scans'] = [i-1 for i in options['which_scans']] + elif isinstance(options['which_scans'], list): + + scans =[] + + for scan in options['which_scans']: + if isinstance(scan, int): + scans.append(scan-1) + + elif isinstance(scan, tuple): + interval = [i-1 for i in range(scan[0], scan[1]+1)] + scans.extend(interval) + + + options['which_scans'] = scans # Tuple is used to define an interval - as elements tuples can't be assigned, I convert it to a list here. - elif type(options['which_cycles']) == tuple: - which_cycles = list(options['which_cycles']) + elif isinstance(options['which_scans'], tuple): + which_scans = list(options['which_scans']) - if which_cycles[0] <= 0: - which_cycles[0] = 1 + if which_scans[0] <= 0: + which_scans[0] = 1 - elif which_cycles[1] < 0: - which_cycles[1] = len(cycles) + elif which_scans[1] < 0: + which_scans[1] = len(options['which_scans']) - options['which_cycles'] = [i-1 for i in range(which_cycles[0], which_cycles[1]+1)] \ No newline at end of file + options['which_scans'] = [i-1 for i in range(which_scans[0], which_scans[1]+1)] + + + for i, scan in enumerate(options['which_scans']): + if scan in options['exclude_scans']: + del options['which_scans'][i] + + + + +def generate_colours(scans, options): + # FIXME Make this a generalised function and use this instead of this and in the electrochemsitry submodule + + # Assign colours from the options dictionary if it is defined, otherwise use standard colours. + if options['colours']: + colour = options['colours'] + + else: + #colour = (214/255, 143/255, 214/255) # Plum Web (#D68FD6), coolors.co + colour = (90/255, 42/255, 39/255) # Caput Mortuum(#5A2A27), coolors.co + + # If gradient is enabled, find start and end points for each colour + if options['gradient']: + + add = min([(1-x)*0.75 for x in colour]) + + colour_start = colour + colour_end = [x+add for x in colour] + + + # Generate lists of colours + colours = [] + + for scan_number in range(0, len(scans)): + if options['gradient']: + weight_start = (len(scans) - scan_number)/len(scans) + weight_end = scan_number/len(scans) + + colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(colour_start, colour_end)] + + colours.append(colour) + + return colours \ No newline at end of file From cd0eaff25b851b8242a164d3b2bef301f58e546e Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 15 Aug 2022 17:35:01 +0200 Subject: [PATCH 266/355] Make BATSMALL-read more general and import decimal = , --- nafuma/electrochemistry/io.py | 28 ++++++++++++++++++++++------ nafuma/electrochemistry/plot.py | 3 ++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 7ad764d..296bd4f 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -70,7 +70,9 @@ def read_batsmall(path): Output: df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' - df = pd.read_csv(path, skiprows=2, sep='\t') + + # FIXME Now it is hardcoded that the decimal is a comma. It should do a check, as datasets can vary depending on the system settings of the machine that does the data conversion + df = pd.read_csv(path, skiprows=2, sep='\t', decimal=',') df = df.loc[:, ~df.columns.str.contains('^Unnamed')] return df @@ -133,9 +135,11 @@ def process_batsmall_data(df, options=None): df = unit_conversion(df=df, options=options) + if options['splice_cycles']: df = splice_cycles(df=df, options=options) + # Replace NaN with empty string in the Comment-column and then remove all steps where the program changes - this is due to inconsistent values for current df[["comment"]] = df[["comment"]].fillna(value={'comment': ''}) df = df[df["comment"].str.contains("program")==False] @@ -694,11 +698,23 @@ def get_old_units(df: pd.DataFrame, options: dict) -> dict: if options['kind'] == 'batsmall': - time = df.columns[0].split()[-1].strip('[]') - voltage = df.columns[1].split()[-1].strip('[]') - current = df.columns[2].split()[-1].strip('[]') - capacity, mass = df.columns[4].split()[-1].strip('[]').split('/') - old_units = {'time': time, 'current': current, 'voltage': voltage, 'capacity': capacity, 'mass': mass} + old_units = {} + + for column in df.columns: + if 'TT [' in column: + old_units['time'] = column.split()[-1].strip('[]') + elif 'U [' in column: + old_units['voltage'] = column.split()[-1].strip('[]') + elif 'I [' in column: + old_units['current'] = column.split()[-1].strip('[]') + elif 'C [' in column: + old_units['capacity'], old_units['mass'] = column.split()[-1].strip('[]').split('/') + + # time = df.columns[0].split()[-1].strip('[]') + # voltage = df.columns[1].split()[-1].strip('[]') + # current = df.columns[2].split()[-1].strip('[]') + # capacity, mass = df.columns[4].split()[-1].strip('[]').split('/') + # old_units = {'time': time, 'current': current, 'voltage': voltage, 'capacity': capacity, 'mass': mass} if options['kind']=='neware': diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 7a0246c..8fbee49 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -23,7 +23,7 @@ def plot_gc(data, options=None): # Update options - required_options = ['force_reload', 'x_vals', 'y_vals', 'which_cycles', 'limit', 'exclude_cycles', 'show_plot', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] + required_options = ['force_reload', 'x_vals', 'y_vals', 'which_cycles', 'limit', 'exclude_cycles', 'show_plot', 'summary', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] default_options = { 'force_reload': False, 'x_vals': 'capacity', 'y_vals': 'voltage', @@ -31,6 +31,7 @@ def plot_gc(data, options=None): 'limit': None, # Limit line to be drawn 'exclude_cycles': [], 'show_plot': True, + 'summary': False, 'charge': True, 'discharge': True, 'colours': None, 'differentiate_charge_discharge': True, From 24a7b12299e7fe6cb37ff24897fbbd596e5ec75f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 16 Aug 2022 11:16:20 +0200 Subject: [PATCH 267/355] Extend save / load functions to include e0 --- nafuma/xanes/io.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index d7e9f94..03e7c88 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -236,19 +236,53 @@ def save_data(data: dict, options={}) -> None: aux.write_log(message=f'File already exists and overwrite disabled. Exiting without saving...', options=options) return None - data['xanes_data'].to_csv(os.path.join(options['save_folder'], options['filename']), sep='\t', index=False) + with open(os.path.join(options['save_folder'], options['filename']), 'w') as f: + + if 'e0_diff' in data.keys(): + f.write(f'# Number of header lines: {len(data["path"])+1} \n') + + for i, (path, e0) in enumerate(data['e0_diff'].items()): + f.write(f'# Scan_{i} \t {e0} \n') + + else: + f.write(f'# Number of header lines: {1}') + + + data['xanes_data'].to_csv(f, sep='\t', index=False) + + + #data['xanes_data'].to_csv(os.path.join(options['save_folder'], options['filename']), sep='\t', index=False) -def load_data(data: dict, options={}) -> dict: +def load_data(path: str) -> dict: # FIXME Let this function be called by read_data() if some criterium is passed data = {} - data['xanes_data'] = pd.read_csv(data['path'], sep='\t') + + with open(path, 'r') as f: + line = f.readline() + header_lines = int(line.split()[-1]) + + if header_lines > 1: + edge_positions = [] + line = f.readline() + while line[0] == '#': + edge_positions.append(line.split()[-1]) + line = f.readline() + + data['xanes_data'] = pd.read_csv(path, sep='\t', skiprows=header_lines) data['path'] = data['xanes_data'].columns.to_list() data['path'].remove('ZapEnergy') + if header_lines > 1: + data['e0_diff'] = {} + + for path, edge_position in zip(data['path'], edge_positions): + data['e0_diff'][path] = edge_position + + return data From a77eb23a38ec1a661f857d38dd9de74d54a38d82 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 16 Aug 2022 11:43:56 +0200 Subject: [PATCH 268/355] Make sure e0_diff is loaded as float, not str --- nafuma/xanes/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 03e7c88..e73674e 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -280,7 +280,7 @@ def load_data(path: str) -> dict: data['e0_diff'] = {} for path, edge_position in zip(data['path'], edge_positions): - data['e0_diff'][path] = edge_position + data['e0_diff'][path] = float(edge_position) From f31849b08bf5a42ef996ae522dd7d0d49fa53dfc Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 16 Aug 2022 12:08:52 +0200 Subject: [PATCH 269/355] Add function to pick out scans based on timestamps --- nafuma/xanes/plot.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/nafuma/xanes/plot.py b/nafuma/xanes/plot.py index 99e5d9f..f4389d5 100644 --- a/nafuma/xanes/plot.py +++ b/nafuma/xanes/plot.py @@ -4,6 +4,7 @@ from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLoca import pandas as pd import numpy as np import math +import datetime #import ipywidgets as widgets #from IPython.display import display @@ -13,7 +14,7 @@ import nafuma.plotting as btp import nafuma.auxillary as aux -def plot_xanes(data, options=None): +def plot_xanes(data, options={}): # Update options @@ -30,8 +31,6 @@ def plot_xanes(data, options=None): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - print(options['xunit']) - if not 'xanes_data' in data.keys(): data['xanes_data'] = xas.io.load_data(data=data, options=options) @@ -59,6 +58,28 @@ def plot_xanes(data, options=None): return fig, ax +def pick_out_scans(metadata: dict, timestamp: list): + + # If either start or end are None, set to way back when or way into the future + if not timestamp[0]: + timestamp[0] = datetime.datetime.strptime('1970 01 01 00:00:00', '%Y %m %d %H:%M:%S') + else: + timestamp[0] = datetime.datetime.strptime(timestamp[0], "%d.%b %y %H.%M.%S") + if not timestamp[1]: + timestamp[1] = datetime.datetime.strptime('3000 01 01 00:00:00', '%Y %m %d %H:%M:%S') + else: + timestamp[1] = datetime.datetime.strptime(timestamp[1], "%d.%b %y %H.%M.%S") + + scans = [] + for i, time in enumerate(metadata['time']): + if time >= timestamp[0] and time <= timestamp[1]: + scans.append(i) + + + return scans + + + From 8ca73e4687459803e4e172daea89fb73b4be8d22 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 16 Aug 2022 12:09:37 +0200 Subject: [PATCH 270/355] Estimate edge position from xanes_data instead of xanes_data_original --- nafuma/xanes/calib.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index beaeba7..6162d99 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -462,8 +462,6 @@ def smoothing(data: dict, options={}): # Make plots ... if options['save_plots'] or options['show_plots']: - - edge_pos = estimate_edge_position(data=data, options=options) step_length = data['xanes_data']['ZapEnergy'].iloc[1] - data['xanes_data']['ZapEnergy'].iloc[0] @@ -563,8 +561,8 @@ def estimate_edge_position(data: dict, options={}, index=0): options = aux.update_options(options=options, required_options=required_options, default_options=default_options) #making new dataframe to keep the differentiated data - df_diff = pd.DataFrame(data['xanes_data_original']["ZapEnergy"]) - df_diff[data['path'][index]]=data['xanes_data_original'][data['path'][index]].diff(periods=options['periods']) + df_diff = pd.DataFrame(data['xanes_data']["ZapEnergy"]) + df_diff[data['path'][index]]=data['xanes_data'][data['path'][index]].diff(periods=options['periods']) #shifting column values up so that average differential fits right between the points used in the calculation df_diff[data['path'][index]]=df_diff[data['path'][index]].shift(-int(options['periods']/2)) From 9f2199e69aad86c736b323480e912ec5e7721d2b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 18 Aug 2022 09:39:14 +0200 Subject: [PATCH 271/355] Make colour gradient work as intended for subsets --- nafuma/xanes/plot.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/nafuma/xanes/plot.py b/nafuma/xanes/plot.py index f4389d5..440308c 100644 --- a/nafuma/xanes/plot.py +++ b/nafuma/xanes/plot.py @@ -38,16 +38,19 @@ def plot_xanes(data, options={}): # Update list of cycles to correct indices update_scans_list(data=data, options=options) - colours = generate_colours(scans=data['path'], options=options) - + colours = generate_colours(scans=options['which_scans'], options=options) # Prepare plot, and read and process data fig, ax = btp.prepare_plot(options=options) + + # Add counter to pick out correct colour + counter = 0 for i, path in enumerate(data['path']): if i in options['which_scans']: - data['xanes_data'].plot(x='ZapEnergy', y=path, ax=ax, c=colours[i]) + data['xanes_data'].plot(x='ZapEnergy', y=path, ax=ax, c=colours[counter]) + counter += 1 fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) From 9a3efbf5054d22e7515f14a1c655993006381eac Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 22 Aug 2022 17:02:42 +0200 Subject: [PATCH 272/355] Determine decimal point when reading batsmall data --- nafuma/electrochemistry/io.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 296bd4f..6cf462f 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -71,8 +71,19 @@ def read_batsmall(path): df: pandas DataFrame containing the data as-is, but without additional NaN-columns.''' - # FIXME Now it is hardcoded that the decimal is a comma. It should do a check, as datasets can vary depending on the system settings of the machine that does the data conversion - df = pd.read_csv(path, skiprows=2, sep='\t', decimal=',') + # Determine if decimal point is . or , + with open(path, 'r') as f: + for i, line in enumerate(f): + if i == 10: + values = line.split() + if len(values[1].split('.')) == 2: + decimal_point = '.' + elif len(values[1].split(',')) == 2: + decimal_point = ',' + + + + df = pd.read_csv(path, skiprows=2, sep='\t', decimal=decimal_point) df = df.loc[:, ~df.columns.str.contains('^Unnamed')] return df From 1461d71b992829cd9f7d4c6dd625006cea64ba67 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 22 Aug 2022 17:03:06 +0200 Subject: [PATCH 273/355] Convert timestamps in metadata based on ref time --- nafuma/xanes/io.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index e73674e..cda893d 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -341,7 +341,10 @@ def read_metadata(data: dict, options={}) -> dict: default_options = { 'get_temperature': True, 'get_timestamp': True, - 'adjust_time': False + 'adjust_time': False, + 'convert_time': False, + 'reference_time': None, + 'time_unit': 's' } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -371,6 +374,24 @@ def read_metadata(data: dict, options={}) -> dict: timestamps.append(time) + if options['reference_time'] and options['convert_time']: + from . import unit_tables + new_times = [] + + if isinstance(options['reference_time'], str): + options['reference_time'] = datetime.datetime.strptime(options['reference_time'], "%d.%b %y %H.%M.%S") + + for time in timestamps: + new_time = (time.timestamp() - options['reference_time'].timestamp()) * unit_tables.time()['s'].loc[options['time_unit']] + + new_times.append(new_time) + + + timestamps = new_times + + + + metadata = {'time': timestamps, 'temperature': temperatures} return metadata @@ -378,9 +399,6 @@ def read_metadata(data: dict, options={}) -> dict: - - - def determine_active_roi(scan_data): # FIXME For Co-edge, this gave a wrong scan From 0e5f7dba55d1d7d773c2e8000c8ac80dd1af8cce Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Mon, 22 Aug 2022 17:03:17 +0200 Subject: [PATCH 274/355] Add unit_tables to xanes-module --- nafuma/xanes/unit_tables.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 nafuma/xanes/unit_tables.py diff --git a/nafuma/xanes/unit_tables.py b/nafuma/xanes/unit_tables.py new file mode 100644 index 0000000..52ca61c --- /dev/null +++ b/nafuma/xanes/unit_tables.py @@ -0,0 +1,11 @@ +import pandas as pd + +def time(): + # Define matrix for unit conversion for time + time = {'h': [1, 60, 3600, 3600000], 'min': [1/60, 1, 60, 60000], 's': [1/3600, 1/60, 1, 1000], 'ms': [1/3600000, 1/60000, 1/1000, 1]} + time = pd.DataFrame(time) + time.index = ['h', 'min', 's', 'ms'] + + return time + + From f92636370a905ea3a2822f06d33fbdea16e16017 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 24 Aug 2022 13:26:15 +0200 Subject: [PATCH 275/355] Fix required options --- nafuma/xanes/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index cda893d..4b98727 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -336,7 +336,7 @@ def read_data(data: dict, options={}) -> pd.DataFrame: def read_metadata(data: dict, options={}) -> dict: - required_options = ['get_temperature', 'get_timestamp', 'adjust_time'] + required_options = ['get_temperature', 'get_timestamp', 'adjust_time', 'convert_time', 'time_unit', 'reference_time'] default_options = { 'get_temperature': True, From f3bf6f88d04b44b3fb0bff3d78a66b1c90aad16f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 24 Aug 2022 13:27:44 +0200 Subject: [PATCH 276/355] Add combined averaging and dark subtraction --- nafuma/xrd/io.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 41821ca..36eac11 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -160,6 +160,82 @@ def generate_image_list(path, options=None): } +def process_2d_scans(data: dict, options={}): + + default_options = { + 'scans': 15, # number of scans per image + 'img_filename': 'img_', + 'extension': '.edf', + 'darks': True, # whether there are darks + 'dark_filename': 'dark_', + 'save': False, + 'save_folder': './average/', + 'save_filename': 'avg_', + 'save_extension': '.dat' + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + + all_imgs = [os.path.join(data['path'], img) for img in os.listdir(data['path']) if img.endswith(options['extension']) and img.startswith(options['img_filename'])] + + if options['darks']: + all_darks = [os.path.join(data['path'], img) for img in os.listdir(data['path']) if img.endswith(options['extension']) and img.startswith(options['dark_filename'])] + + + scans = int(len(all_imgs) / options['scans']) + + assert scans - (len(all_imgs) / options['scans']) == 0 + + + imgs = [] + darks = [] + + for i in range(scans): + img = [] + dark = [] + for i in range(options['scans']): + img.append(all_imgs.pop(0)) + + if options['darks']: + dark.append(all_darks.pop(0)) + + imgs.append(img) + + if options['darks']: + darks.append(dark) + + + img_avgs = [] + for img in imgs: + img_avg = average_images(img) + + if options['darks']: + dark_avg = average_images(dark) + img_avg = subtract_dark(img_avg, dark_avg) + + img_avgs.append(img_avg) + + + if options['save']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + for i, img in enumerate(img_avgs): + np.savetxt(os.path.join(options['save_folder'], options['save_filename']+f'{i}'.zfill(4)+options['save_extension']), img, fmt='%.1f', delimiter=";") + + + + return img_avgs + + + + + + + + + def average_images(images): ''' Takes a list of path to image files, reads them and averages them before returning the average image''' From f76a742fff2ecee8e17fb3bef2e730a8b7071f9d Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Wed, 24 Aug 2022 16:15:56 +0200 Subject: [PATCH 277/355] Add metadata and expand image array read-in --- nafuma/xrd/io.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 36eac11..cd571dd 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -16,8 +16,14 @@ import nafuma.auxillary as aux def get_image_array(path): - image = fabio.open(path) - image_array = image.data + beamline_extension = ['.edf', '.cbf', '.mar3450'] + + if path.endswith(tuple(beamline_extension)): + image = fabio.open(path) + image_array = image.data + + elif path.endswith('.dat'): + image_array = np.loadtxt(path, skiprows=1, delimiter=';') return image_array @@ -185,6 +191,7 @@ def process_2d_scans(data: dict, options={}): scans = int(len(all_imgs) / options['scans']) + assert scans - (len(all_imgs) / options['scans']) == 0 @@ -194,6 +201,7 @@ def process_2d_scans(data: dict, options={}): for i in range(scans): img = [] dark = [] + for i in range(options['scans']): img.append(all_imgs.pop(0)) @@ -206,15 +214,18 @@ def process_2d_scans(data: dict, options={}): darks.append(dark) - img_avgs = [] - for img in imgs: + img_avgs = [] + headers = [] + for img, dark in zip(imgs,darks): img_avg = average_images(img) + header = get_image_headers(img[0]) if options['darks']: dark_avg = average_images(dark) img_avg = subtract_dark(img_avg, dark_avg) img_avgs.append(img_avg) + headers.append(header) if options['save']: @@ -222,7 +233,9 @@ def process_2d_scans(data: dict, options={}): os.makedirs(options['save_folder']) for i, img in enumerate(img_avgs): - np.savetxt(os.path.join(options['save_folder'], options['save_filename']+f'{i}'.zfill(4)+options['save_extension']), img, fmt='%.1f', delimiter=";") + with open(os.path.join(options['save_folder'], options['save_filename']+f'{i}'.zfill(4)+options['save_extension']), 'w') as f: + f.write(f'# Time: {headers[i]["time"]}\n') + np.savetxt(f, img, fmt='%.2f', delimiter=";") From ab1752e179436bd2c7839bb71a08af66ceeb6ae3 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 25 Aug 2022 15:15:44 +0200 Subject: [PATCH 278/355] Add batch integration --- nafuma/xrd/io.py | 71 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index cd571dd..040c034 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -35,6 +35,37 @@ def get_image_headers(path): return image.header + +def integrate_scans(data: dict, options={}): + + default_options = { + 'extension': '.dat', + 'save': True, + 'integration_save_folder': './integrated/', + 'integration_save_filename': None, + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + + if not isinstance(data['path'], list): + imgs = aux.get_filenames(data['path'], ext=options['extension']) + + + diffractograms, wavelengths = [], [] + for img in imgs: + data['image'] = get_image_array(img) + + options['integration_save_filename'] = os.path.basename(img).split('.')[0] + '_int.xy' + + diff, wl = integrate_1d(data=data, options=options) + + diffractograms.append(diff) + wavelengths.append(wl) + + return diffractograms, wavelengths + + def integrate_1d(data, options={}, index=0): ''' Integrates an image file to a 1D diffractogram. @@ -48,17 +79,17 @@ def integrate_1d(data, options={}, index=0): df: DataFrame contianing 1D diffractogram if option 'return' is True ''' - required_options = ['unit', 'npt', 'save', 'save_filename', 'save_extension', 'save_folder', 'overwrite', 'extract_folder', 'error_model'] + required_options = ['unit', 'npt', 'save', 'integration_save_filename', 'save_extension', 'integration_save_folder', 'overwrite', 'extract_folder', 'error_model'] default_options = { 'unit': '2th_deg', - 'npt': 3000, + 'npt': 5000, 'extract_folder': 'tmp', 'error_model': None, 'save': False, - 'save_filename': None, + 'integration_save_filename': None, 'save_extension': '_integrated.xy', - 'save_folder': '.', + 'integration_save_folder': '.', 'overwrite': False} options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -68,7 +99,7 @@ def integrate_1d(data, options={}, index=0): # Get image array from filename if not passed - if 'image' not in data.keys() or not data['image']: + if 'image' not in data.keys() or not isinstance(data['image'], np.ndarray): data['image'] = get_image_array(data['path'][index]) @@ -89,8 +120,8 @@ def integrate_1d(data, options={}, index=0): if not os.path.isdir(options['extract_folder']): os.makedirs(options['extract_folder']) - if not os.path.isdir(options['save_folder']): - os.makedirs(options['save_folder']) + if not os.path.isdir(options['integration_save_folder']): + os.makedirs(options['integration_save_folder']) @@ -104,10 +135,6 @@ def integrate_1d(data, options={}, index=0): if not options['save']: os.remove(filename) shutil.rmtree(f'tmp') - - - # Reset this option - options['save_folder'] = None return diffractogram, wavelength @@ -122,19 +149,18 @@ def make_filename(options, path=None): elif options['save']: # Case 1: No filename is given. - if not options['save_filename']: + if not options['integration_save_filename']: # If a path is given instead of an image array, the path is taken as the trunk of the savename if path: # Make filename by joining the save_folder, the filename (with extension deleted) and adding the save_extension - filename = os.path.join(options['save_folder'], os.path.split(path)[-1].split('.')[0] + options['save_extension']) + filename = os.path.join(options['integration_save_folder'], os.path.split(path)[-1].split('.')[0] + options['save_extension']) else: # Make filename just "integrated.dat" in the save_folder - filename = os.path.join(options['save_folder'], 'integrated.xy') + filename = os.path.join(options['integration_save_folder'], 'integrated.xy') else: - filename = os.path.join(options['save_folder'], options['save_filename']) - + filename = os.path.join(options['integration_save_folder'], options['integration_save_filename']) if not options['overwrite']: trunk = filename.split('.')[0] @@ -216,9 +242,10 @@ def process_2d_scans(data: dict, options={}): img_avgs = [] headers = [] + for img, dark in zip(imgs,darks): img_avg = average_images(img) - header = get_image_headers(img[0]) + header = get_image_headers(img[0]) if options['darks']: dark_avg = average_images(dark) @@ -233,12 +260,12 @@ def process_2d_scans(data: dict, options={}): os.makedirs(options['save_folder']) for i, img in enumerate(img_avgs): - with open(os.path.join(options['save_folder'], options['save_filename']+f'{i}'.zfill(4)+options['save_extension']), 'w') as f: - f.write(f'# Time: {headers[i]["time"]}\n') - np.savetxt(f, img, fmt='%.2f', delimiter=";") + if options['save_extension'] == '.dat': + with open(os.path.join(options['save_folder'], options['save_filename']+f'{i}'.zfill(4)+options['save_extension']), 'w') as f: + f.write(f'# Time: {headers[i]["time"]}\n') + np.savetxt(f, img, fmt='%.2f', delimiter=";") + - - return img_avgs From 42cddb8fa47ebc20908fdbe70a105a5a30e397d6 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Thu, 25 Aug 2022 15:16:14 +0200 Subject: [PATCH 279/355] Add GIF-animation for 2D scans --- nafuma/xrd/plot.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index e3d2594..743bfc8 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -5,9 +5,12 @@ from matplotlib.ticker import MultipleLocator import pandas as pd import numpy as np import math +import os +import shutil import ipywidgets as widgets from IPython.display import display +from PIL import Image import nafuma.xrd as xrd import nafuma.auxillary as aux @@ -857,3 +860,59 @@ def reverse_diffractograms(diffractograms): return rev_diffractograms + + +def make_animation(data: dict, options={}): + + default_options = { + 'cmap': 'inferno', + 'contrast': False, + 'contrast_factor': 1/3, + 'save_path': 'diff_animation.gif', + 'fps': 5 + } + + options = aux.update_options(options=options, default_options=default_options, required_options=default_options.keys()) + + if not isinstance(data['path'], list): + data['path'] = aux.get_filenames(data['path'], ext='dat') + + + if not os.path.isdir('tmp'): + os.makedirs('tmp') + + # Scale image to make GIF smaller + # + options['format_params']['width'] = 5 + options['format_params']['height'] = 5 + + options['format_params']['dpi'] = 200 + + for i, scan in enumerate(data['path']): + + giffig, gifax = btp.prepare_plot(options=options) + + img = xrd.io.get_image_array(scan) + + if options['contrast']: + img[img < 0] = 0.00000001 + img = np.log(img) + img[img < 0] = 0 + + gifax.imshow(img, cmap=options['cmap']) + + plt.savefig(os.path.join('tmp', str(i+1).zfill(4)+'.png')) + plt.close() + + + img_paths = aux.get_filenames('tmp', ext='png') + + frames = [] + for path in img_paths: + frame = Image.open(path) + frames.append(frame) + + frames[0].save(options['save_path'], format='GIF', append_images=frames[1:], save_all=True, duration=(1/options['fps'])*1000, loop=0) + + shutil.rmtree('tmp') + From 7285f47db368cc5534c2ac44568225d9b8c6531e Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 26 Aug 2022 13:23:32 +0200 Subject: [PATCH 280/355] Strip headers from exported EVA-files also --- nafuma/xrd/io.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 040c034..78375d2 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -228,7 +228,7 @@ def process_2d_scans(data: dict, options={}): img = [] dark = [] - for i in range(options['scans']): + for j in range(options['scans']): img.append(all_imgs.pop(0)) if options['darks']: @@ -240,7 +240,7 @@ def process_2d_scans(data: dict, options={}): darks.append(dark) - img_avgs = [] + img_avgs = [] headers = [] for img, dark in zip(imgs,darks): @@ -679,6 +679,9 @@ def strip_headers_from_xy(path: str, filename=None) -> None: for line in lines: if line[0] == '#': headerlines += 1 + elif line[0] == "\'": + headerlines += 1 + else: xy.append(line) From 9ed675b8cb2ccd06a846f947900e154e7d19669a Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Fri, 26 Aug 2022 13:24:17 +0200 Subject: [PATCH 281/355] Short term fix for heatmap generation with beamline data --- nafuma/xrd/plot.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 743bfc8..4ed6d7a 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -362,6 +362,10 @@ def generate_heatmap(data, options={}): if options['increase_contrast']: + + if d['I'].min() < 0: + d['I'] = d['I'] - d['I'].min() + d['I'] = d['I']**(1/options['contrast_factor']) twotheta = np.append(twotheta, d['2th'].to_numpy()) @@ -418,23 +422,29 @@ def generate_heatmap(data, options={}): scan_numbers = [] temperatures = [] - for i, filename in enumerate(data['path']): - scan_numbers.append(i) - temperatures.append(xrd.io.read_metadata_from_xy(filename)['temperature']) + + # FIXME This is a very bad check for whether it is HTXRD or not - it bascailly just excludes any files that has a .poni-file passed. Make more rigorous in future! + if not 'calibrant' in data.keys(): + for i, filename in enumerate(data['path']): + scan_numbers.append(i) + temperatures.append(xrd.io.read_metadata_from_xy(filename)['temperature']) + + yticks = scan_numbers[0::options['heatmap_y_tick_locators'][0]] + yticks.append(scan_numbers[-1]) + + if not temperatures[0]: + yticklabels = yticks + options['heatmap.ylabel'] = 'Scan number' + options['heatmap.yunit'] = None + + else: + yticklabels = temperatures[0::options['heatmap_y_tick_locators'][0]] + yticklabels.append(temperatures[-1]) + options['heatmap.ylabel'] = 'Temperature' + options['heatmap.yunit'] = '$^\circ$C' - yticks = scan_numbers[0::options['heatmap_y_tick_locators'][0]] - yticks.append(scan_numbers[-1]) - - if not temperatures[0]: - yticklabels = yticks - options['heatmap.ylabel'] = 'Scan number' - options['heatmap.yunit'] = None - else: - yticklabels = temperatures[0::options['heatmap_y_tick_locators'][0]] - yticklabels.append(temperatures[-1]) - options['heatmap.ylabel'] = 'Temperature' - options['heatmap.yunit'] = '$^\circ$C' + yticks, yticklabels = None, None From 8c3861c984147e13ef54f7b4899fcfff655be14a Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 31 Aug 2022 20:36:31 +0200 Subject: [PATCH 282/355] Fixed split_scan_data for trans data w/o roi --- nafuma/xanes/io.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 4b98727..3311746 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -10,7 +10,7 @@ def split_scan_data(data: dict, options={}) -> list: ''' Splits a XANES-file from BM31 into different files depending on the edge. Has the option to add intensities of all scans of same edge into the same file. As of now only picks out xmap_rois (fluoresence mode) and for Mn, Fe, Co and Ni K-edges.''' - required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'active_roi', 'add_rois', 'return'] + required_options = ['log', 'logfile', 'save', 'save_folder', 'replace', 'active_roi', 'add_rois', 'return', 'skip_if_no_roi'] default_options = { 'log': False, @@ -20,7 +20,8 @@ def split_scan_data(data: dict, options={}) -> list: 'replace': False, # whether to replace the files if they already exist 'active_roi': None, 'add_rois': False, # Whether to add the rois of individual scans of the same edge together - 'return': True + 'return': True, + 'skip_if_no_roi': True } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -100,10 +101,14 @@ def split_scan_data(data: dict, options={}) -> list: if not ('xmap_roi00' in headers[i]) and (not 'xmap_roi01' in headers[i]): + if options['skip_if_no_roi']: + if options['log']: + aux.write_log(message='... ... Did not find fluoresence data. Skipping...', options=options) + continue if options['log']: - aux.write_log(message='... ... Did not find fluoresence data. Skipping...', options=options) + aux.write_log(message='... ... Did not find fluoresence data, but still proceeding ...', options=options) - continue + From 8714e86753ddeb699d004d6f1084b9d7d95d40c7 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 1 Sep 2022 14:08:26 +0200 Subject: [PATCH 283/355] Also hh:mm:ss is importable as time stamp from xas --- nafuma/xanes/io.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 3311746..373a3bc 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -367,8 +367,13 @@ def read_metadata(data: dict, options={}) -> dict: if options['get_timestamp']: with open(filename, 'r') as f: - time = f.readline().strip('# Time: ') - time = datetime.datetime.strptime(time, "%a %b %d %H:%M:%S %Y ") + #time = f.readline().strip('# Time: ') #<-- Previous code + time = f.readline().split('# Time: ')[-1] #Hope this does not fuck you up, Rasmus - but I needed another space here + split_operator=time[-9] #This should be the operator that splits hours, minutes and seconds + if split_operator == ".": + time = datetime.datetime.strptime(time, "%a %b %d %H.%M.%S %Y ") + if split_operator == ":": + time = datetime.datetime.strptime(time, "%a %b %d %H:%M:%S %Y ") if options['adjust_time']: time_elapsed = scan_data['Htime'].iloc[-1] - scan_data['Htime'].iloc[0] From 716e4acb822a78f690fe1492b5ef9bdd2642a5df Mon Sep 17 00:00:00 2001 From: halvorhv Date: Thu, 1 Sep 2022 14:09:03 +0200 Subject: [PATCH 284/355] Also hh:mm:ss is usable when selecting spectra --- nafuma/xanes/plot.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/plot.py b/nafuma/xanes/plot.py index 440308c..78230c6 100644 --- a/nafuma/xanes/plot.py +++ b/nafuma/xanes/plot.py @@ -64,14 +64,22 @@ def plot_xanes(data, options={}): def pick_out_scans(metadata: dict, timestamp: list): # If either start or end are None, set to way back when or way into the future + split_operator=timestamp[0][-3] #Adding this to enable reading of both "." and ":" as operators to split hour:minute:second + if not timestamp[0]: timestamp[0] = datetime.datetime.strptime('1970 01 01 00:00:00', '%Y %m %d %H:%M:%S') else: - timestamp[0] = datetime.datetime.strptime(timestamp[0], "%d.%b %y %H.%M.%S") + if split_operator == ".": + timestamp[0] = datetime.datetime.strptime(timestamp[0], "%d.%b %y %H.%M.%S") + if split_operator == ":": + timestamp[0] = datetime.datetime.strptime(timestamp[0], "%d.%b %y %H:%M:%S") if not timestamp[1]: timestamp[1] = datetime.datetime.strptime('3000 01 01 00:00:00', '%Y %m %d %H:%M:%S') else: - timestamp[1] = datetime.datetime.strptime(timestamp[1], "%d.%b %y %H.%M.%S") + if split_operator == ".": + timestamp[1] = datetime.datetime.strptime(timestamp[1], "%d.%b %y %H.%M.%S") + if split_operator == ":": + timestamp[1] = datetime.datetime.strptime(timestamp[1], "%d.%b %y %H:%M:%S") scans = [] for i, time in enumerate(metadata['time']): From 686f74a9296bb79fc0926efabc763f5cb995614b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 18 Sep 2022 15:50:54 +0200 Subject: [PATCH 285/355] Add initial script files for DFT processing --- nafuma/dft/__init__.py | 1 + nafuma/dft/electrons.py | 1109 +++++++++++++++++++++++++++++++++++++++ nafuma/dft/io.py | 545 +++++++++++++++++++ nafuma/dft/structure.py | 906 ++++++++++++++++++++++++++++++++ 4 files changed, 2561 insertions(+) create mode 100644 nafuma/dft/__init__.py create mode 100644 nafuma/dft/electrons.py create mode 100644 nafuma/dft/io.py create mode 100644 nafuma/dft/structure.py diff --git a/nafuma/dft/__init__.py b/nafuma/dft/__init__.py new file mode 100644 index 0000000..db87b5d --- /dev/null +++ b/nafuma/dft/__init__.py @@ -0,0 +1 @@ +from . import electrons, io, structure \ No newline at end of file diff --git a/nafuma/dft/electrons.py b/nafuma/dft/electrons.py new file mode 100644 index 0000000..d96d2ad --- /dev/null +++ b/nafuma/dft/electrons.py @@ -0,0 +1,1109 @@ +import re +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +import os +import linecache +import nafuma.dft as dft +import nafuma.auxillary as aux +import nafuma.plotting as btp + +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) +import importlib +import mpl_toolkits.axisartist as axisartist +from cycler import cycler +import itertools +import matplotlib.patches as mpatches + + + + +def count_electrons(pdos, orbital, interval=None, r=None, scale=None): + ''' Counts electrons the specified oribtals from a projected density of states DataFrame. Interval can be specified, as well as a scaling factor and whether the number should be rounded. + Inputs: + dos: either an individual DOS (as read from read_pdos()), or a list of DOSes. If a single DataFrame is passed, it will be appended to a list + orbital: list of which orbitals to count the electrons from + interval: a list specifying where to start counting from (lower limit) to where to stop counting (upper limit) in eV + r: Number of decimals points the number should be rounded to + scale: A scaling factor to scale the number of electrons to a desired size, e.g. if you have a set containing two atoms per unit cell and you want to know how many electrons per atom there is + + Output: + nelec: The total number of electrons given your choices + nelec_dos: A list where each element is the total number of electrons per DOS passed (e.g. you pass three PDOS from three individual atoms, then you will get total electron count per atom) + nelec_orbitals: A list of lists, where each list contains the number of electrons per orbital specified (e.g. you pass three PDOS from three individual atoms, you will get three lists each containing electron count per orbital specified) + ''' + + + + if not type(orbital) == list: + orbital = [orbital] + + if not type(pdos) == list: + pdos = [pdos] + + + nelec = 0 + nelec_per_dos = [] + nelec_per_orbital = [] + + for d in pdos: + + energy = d["Energy"] + + nelec_orbitals = [] + + for o in orbital: + orbital_dos = d[o] + dE = (energy.max()-energy.min()) / len(energy) + + + if not interval: + interval = [energy.min(), energy.max()] + + emin, emax = interval[0], interval[1] + + + nelec_orbital = 0 + for od, e in zip(orbital_dos, energy): + if e > emin and e < emax: + nelec_orbital += np.abs(od)*dE + #print(nelec_orbital) + + nelec += nelec_orbital + nelec_orbitals.append(nelec_orbital) + + + # Scale the values if specified + if scale: + + for ind, nelec_orbital in enumerate(nelec_orbitals): + nelec_orbitals[ind] = nelec_orbital * scale + + + # If rounding is specified, does so to the electron count per DOS and the electron count per orbital + if r: + # First sums the electron count per orbital, and then round this number + nelec_dos = np.round(sum(nelec_orbitals), r) + + # Then each individual orbital electron count + for ind, nelec_orbital in enumerate(nelec_orbitals): + nelec_orbitals[ind] = np.round(nelec_orbital, r) + + # If no rounding is specified, just adds the electron count per orbital together + else: + nelec_dos = sum(nelec_orbitals) + + # Appends the total electron count for this DOS to the list of all individual DOS electron count + nelec_per_dos.append(nelec_dos) + + # Appends the list of orbital electron counts to the list of all the individual DOS orbital electron count (phew...) + nelec_per_orbital.append(nelec_orbitals) + + + # The total electron count is then scaled in the end. At this point the other values will have been scaled already + if scale: + nelec = nelec * scale + + # And lastly round if this is specified. Again, the electron counts in the lists are already rounded so they don't have to be rounded again + if r: + nelec = np.round(nelec, r) + + return nelec, [nelec_per_dos, nelec_per_orbital] + + + +def integrate_coop(coopcar, interval=None, r=None, scale=None, interactions=None, kind='individual', up=True, down=True, collapse=False): + ''' As of now does not support not passing in interactions. Very much copy and paste from the plotting function - not every choice here might make sense for integration of COOP''' + + coopcar, coop_interactions = dft.io.read_coop(coopcar, collapse=collapse) + + # If interactions has been specified + if interactions: + + # Make interactions into a list of lists for correct looping below + if type(interactions[0]) != list: + interactions_list = [interactions] + else: + interactions_list = interactions + + for ind, interactions in enumerate(interactions_list): + + # Determine which columns to integrate if collapse is enabled + if collapse: + to_integrate = [2*(i-1)+3 for i in interactions] + + + # Make mean column for integration if mean mode is enabeld (is this sensible to include?) + if kind == 'avg' or kind == 'average' or kind == 'mean': + coopcar["mean"] = coopcar.iloc[:, to_integrate].mean(axis=1) + to_integrate = [coopcar.columns.get_loc('mean')] + + # Determine which columns to integrate if collapse is disabled and both up and down should be plotted + elif up and down: + to_integrate_up = [2*(i-1)+3 for i in interactions] + to_integrate_down = [2*(i-1)+5 +2*len(coop_interactions) for i in interactions] + to_integrate = to_integrate_up + to_integrate_down + + if kind == 'avg' or kind == 'average' or kind == 'mean': + coopcar["mean_up"] = coopcar.iloc[:, to_integrate_up].mean(axis=1) + coopcar["mean_down"] = coopcar.iloc[:, to_integrate_down].mean(axis=1) + to_integrate = [coopcar.columns.get_loc('mean_up'), coopcar.columns.get_loc('mean_down')] + + # Determine which columns to plot if collapse is disabled and only up should be plotted + elif up: + to_integrate = [2*(i-1)+3 for i in interactions] + + if kind == 'avg' or kind == 'average' or kind == 'mean': + coopcar["mean_up"] = coopcar.iloc[:, to_integrate].mean(axis=1) + to_integrate = [coopcar.columns.get_loc('mean_up')] + + + # Determine which columns to plot if collapse is disabled and only down should be plotted + elif down: + to_integrate = [2*(i-1)+5 +2*len(coop_interactions) for i in interactions] + + if kind == 'avg' or kind == 'average' or kind == 'mean': + coopcar["mean_down"] = coopcar.iloc[:, to_integrate].mean(axis=1) + to_integrate = [coopcar.columns.get_loc('mean_down')] + + + + bonding = 0 + antibonding = 0 + bonding_interactions = [] + antibonding_interactions = [] + difference_interactions = [] + percentage_bonding_interactions = [] + + + for integrate in to_integrate: + + bonding_interaction = 0 + antibonding_interaction = 0 + + coop = coopcar.iloc[:, integrate] + + energy = coopcar["Energy"] + dE = (energy.max()-energy.min()) / len(energy) + + # Sets interval to everything below the Fermi-level by default if not specified in function call + if not interval: + interval = [energy.min(), 0] + + emin, emax = interval[0], interval[1] + + + for c, e in zip(coop, energy): + if e > emin and e < emax: + if c > 0: + bonding_interaction += c*dE + elif c < 0: + antibonding_interaction += np.abs(c)*dE + + + bonding += bonding_interaction + antibonding += antibonding_interaction + + difference_interaction = bonding_interaction - antibonding_interaction + percentage_bonding_interaction = bonding_interaction / (bonding_interaction + antibonding_interaction) * 100 + + if scale: + bonding_interaction = bonding_interaction * scale + antibonding_interaction = antibonding_interaction * scale + difference_interaction = difference_interaction * scale + + if r: + bonding_interaction = np.round(bonding_interaction, r) + antibonding_interaction = np.round(antibonding_interaction, r) + difference_interaction = np.round(difference_interaction, r) + percentage_bonding_interaction = np.round(percentage_bonding_interaction, r) + + bonding_interactions.append(bonding_interaction) + antibonding_interactions.append(antibonding_interaction) + difference_interactions.append(difference_interaction) + percentage_bonding_interactions.append(percentage_bonding_interaction) + + difference = bonding - antibonding + percentage_bonding = (bonding/(bonding+antibonding)) * 100 + + if scale: + bonding = bonding * scale + antibonding = antibonding * scale + difference = difference * scale + + if r: + bonding = np.round(bonding, r) + antibonding = np.round(antibonding, r) + difference = np.round(difference, r) + percentage_bonding = np.round(percentage_bonding, r) + + return [bonding, antibonding, difference, percentage_bonding], [bonding_interactions, antibonding_interactions, difference_interactions, percentage_bonding_interactions] + + + + +def plot_partial_dos(data: dict, options={}): + + + required_options = ['atoms', 'orbitals', 'up', 'down', 'sum_atoms', 'collapse_spin', 'sum_orbitals', 'palettes', 'colours', 'fig', 'ax'] + + default_options = { + 'atoms': None, + 'orbitals': None, + 'up': True, + 'down': True, + 'sum_atoms': True, + 'collapse_spin': False, + 'sum_orbitals': False, + 'palettes': [('qualitative', 'Dark2_8')], + 'colours': None, + 'fig': None, + 'ax': None + + + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + if not options['ax'] and not options['fig']: + fig, ax = btp.prepare_plot(options) + else: + fig, ax = options['fig'], options['ax'] + + species, *_ = dft.io.get_atoms(data['poscar']) + + pdos, options['dos_info'] = dft.io.read_pdos(data=data, options=options) # Extract projected DOS from DOSCAR, decomposed on individual atoms and orbitals Should yield list of N DataFrames where N is number of atoms in POSCAR + + + if not options['orbitals']: + options['orbitals'] = ['s', 'p1', 'p2', 'p3', 'd1', 'd2', 'd3', 'd4', 'd5'] if not options['sum_orbitals'] else ['s', 'p', 'd'] + + if not options['colours']: + colour_cycle = generate_colours(options=options) +# + #colours = [] + #for orbital in options['orbitals']: + # colours.append(next(colour_cycle)) +# + # else: + # colours = options['colours'] + + elif not isinstance(options['colours'], list): + new_colours = [] + for atom in options['atoms']: + new_colours.append([options['colours']]) + + options['colours'] = new_colours + + + print(options['colours']) + + + if not isinstance(options['orbitals'][0], list): + new_orbitals = [] + for atom in options['atoms']: + new_orbitals.append([options['orbitals']]) + + options['orbitals'] = new_orbitals + + + if options['atoms']: + for i, atom in enumerate(options['atoms']): + + if options['sum_atoms']: + for ind, specie in enumerate(species): + if specie == atom: + atom_index = ind + else: + atom_index = atom-1 + + + for j, orbital in enumerate(options['orbitals'][i]): + + colour = options['colours'][i][j] + + if options['dos_info']['spin_polarised']: + if options['up']: + pdos[atom_index].plot(x='Energy', y=orbital+'_u', ax=ax, c=colour) + + if options['down']: + pdos[atom_index].plot(x='Energy', y=orbital+'_d', ax=ax, c=colour) + else: + pdos[atom_index].plot(x='Energy', y=orbital, ax=ax, c=colour) + + + + btp.adjust_plot(fig=fig, ax=ax, options=options) + + return [pdos, ax, fig] + + + +def get_pdos_indices(poscar, atoms): + + species, atom_num, atoms_dict = dft.io.get_atoms(poscar) + + + + +def get_pdos(doscar='DOSCAR', nedos=301, headerlines=6, spin=True, adjust=True, manual_adjust=None): + + lines = dft.io.open_doscar(doscar) + + number_of_atoms = dft.io.get_number_of_atoms(doscar) + + if adjust: + e_fermi = dft.io.get_fermi_level(doscar) if not manual_adjust else manual_adjust + else: + e_fermi = 0 + + pdos = [] + + columns_non_spin = ["Energy", "s", "p_y", "p_z", "p_x", "d_xy", "d_yz", "d_z2-r2", "d_xz", "d_x2-y2"] + columns_spin = ["Energy", "s_up", "s_down", "p_y_up", "p_y_down", "p_z_up", "p_z_down", "p_x_up", "p_x_down", "d_xy_up", "d_xy_down", "d_yz_up", "d_yz_down", + "d_z2-r2_up", "d_z2-r2_down", "d_xz_up", "d_xz_down", "d_x2-y2_up", "d_x2-y2_down"] + + up = ['s_up', "p_y_up", "p_z_up", "p_x_up", "d_xy_up", "d_yz_up", "d_z2-r2_up", "d_xz_up", "d_x2-y2_up"] + down = ['s_down', "p_y_down", "p_z_down", "p_x_down", "d_xy_down", "d_yz_down", "d_z2-r2_down", "d_xz_down", "d_x2-y2_down"] + total = ["s", "p_y", "p_z", "p_x", "d_xy", "d_yz", "d_z2-r2", "d_xz", "d_x2-y2"] + + for i in range(1,number_of_atoms+1): + atom_dos = [] + + for j in range(headerlines+(nedos*i)+i,nedos+headerlines+(nedos*i)+i): + line = lines[j].strip() + values = line.split() + + for ind, value in enumerate(values): + values[ind] = float(value) + + values[0] = values[0] - e_fermi + atom_dos.append(values) + + + atom_df = pd.DataFrame(data=atom_dos, columns=columns_non_spin) if spin==False else pd.DataFrame(data=atom_dos, columns=columns_spin) + + if spin==True: + atom_df[["s_down"]] = -atom_df[["s_down"]] + atom_df[["p_y_down"]] = -atom_df[["p_y_down"]] + atom_df[["p_z_down"]] = -atom_df[["p_z_down"]] + atom_df[["p_x_down"]] = -atom_df[["p_x_down"]] + atom_df[["d_xy_down"]] = -atom_df[["d_xy_down"]] + atom_df[["d_yz_down"]] = -atom_df[["d_yz_down"]] + atom_df[["d_z2-r2_down"]] = -atom_df[["d_z2-r2_down"]] + atom_df[["d_xz_down"]] = -atom_df[["d_xz_down"]] + atom_df[["d_x2-y2_down"]] = -atom_df[["d_x2-y2_down"]] + + atom_df = atom_df.assign(total_up = atom_df[up].sum(axis=1)) + atom_df = atom_df.assign(total_down = atom_df[down].sum(axis=1)) + + elif spin==False: + atom_df = atom_df.assign(total = atom_df[total].sum(axis=1)) + + pdos.append(atom_df) + + return pdos + + + +def prepare_plot(options={}): + + rc_params = options['rc_params'] + format_params = options['format_params'] + + required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + + default_options = { + 'single_column_width': 8.3, + 'double_column_width': 17.1, + 'column_type': 'single', + 'width_ratio': '1:1', + 'aspect_ratio': '1:1', + 'compress_width': 1, + 'compress_height': 1, + 'upscaling_factor': 1.0, + 'dpi': 600, + } + + options = update_options(format_params, required_options, default_options) + + + # Reset run commands + plt.rcdefaults() + + # Update run commands if any is passed (will pass an empty dictionary if not passed) + update_rc_params(rc_params) + + width = determine_width(options) + height = determine_height(options, width) + width, height = scale_figure(options=options, width=width, height=height) + + fig, ax = plt.subplots(figsize=(width, height), dpi=options['dpi']) + + return fig, ax + + +def prepare_dos_plot(width=None, height=None, square=True, dpi=None, colour_cycle=('qualitative', 'Dark2_8'), energyunit='eV', dosunit='arb. u.', scale=1, pdos=None): + + if not pdos: + linewidth = 3*scale + else: + linewidth = 3 + + axeswidth = 3*scale + + plt.rc('lines', linewidth=linewidth) + plt.rc('axes', linewidth=axeswidth) + + if square: + if not width: + width = 20 + + if not height: + height = width + + + fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(width, height), facecolor='w', dpi=dpi) + + + return fig, ax + +def prettify_dos_plot(fig, ax, options): + + + required_options = ['plot_kind', 'flip_xy', 'hide_x_labels', 'hide_y_labels', 'xlabel', 'ylabel', 'xunit', 'yunit', 'xlim', 'ylim', 'x_tick_locators', 'y_tick_locators', 'y_tick_format', 'x_tick_format', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', + 'colours', 'palettes', 'title', 'legend', 'labels', 'label_colours', 'legend_position', 'legend_ncol', 'subplots_adjust', 'text'] + + default_options = { + 'plot_kind': 'PDOS', # DOS/PDOS/COOP/COHP + 'flip_xy': False, + 'hide_x_labels': False, # Whether x labels should be hidden + 'hide_x_ticklabels': False, + 'hide_x_ticks': False, + 'hide_y_labels': False, # whether y labels should be hidden + 'hide_y_ticklabels': False, + 'hide_y_ticks': False, + 'xlabel': 'Energy', + 'ylabel': 'DOS', + 'xunit': r'eV', # The unit of the x-values in the curve plot + 'yunit': r'a.u.', # The unit of the y-values in the curve and bar plots + 'xlim': None, + 'ylim': None, + 'x_tick_locators': [1, .5], # Major and minor tick locators + 'y_tick_locators': [1, .5], + 'y_tick_format': None, + 'x_tick_format': None, + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'title': None, + 'legend': True, + 'labels': None, + 'label_colours': None, + 'legend_position': ['upper center', (0.20, 0.90)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'legend_ncol': 1, + 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], + 'text': None + } + + + if 'plot_kind' in options.keys(): + if 'ylabel' not in options.keys(): + if options['plot_kind'] == 'DOS': + options['ylabel'] = 'DOS' + elif options['plot_kind'] == 'PDOS': + options['ylabel'] = 'PDOS' + elif options['plot_kind'] == 'COOP': + options['ylabel'] = 'COOP' + elif options['plot_kind'] == 'COHP': + options['ylabel'] = 'COHP' + + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + + if options['flip_xy']: + + # Switch all the x- and y-specific values + options = swap_values(dict=options, key1='xlim', key2='ylim') + options = swap_values(dict=options, key1='xunit', key2='yunit') + options = swap_values(dict=options, key1='xlabel', key2='ylabel') + options = swap_values(dict=options, key1='x_tick_locators', key2='y_tick_locators') + options = swap_values(dict=options, key1='hide_x_labels', key2='hide_y_labels') + + # Set labels on x- and y-axes + if not options['hide_y_labels']: + ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + else: + ax.set_ylabel('') + + + + if not options['hide_x_labels']: + ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]') + else: + ax.set_xlabel('') + + + # Hide x- and y- ticklabels + if options['hide_y_ticklabels']: + ax.tick_params(axis='y', direction='in', which='both', labelleft=False, labelright=False) + if options['hide_x_ticklabels']: + ax.tick_params(axis='x', direction='in', which='both', labelbottom=False, labeltop=False) + + + # Hide x- and y-ticks: + if options['hide_y_ticks']: + ax.tick_params(axis='y', direction='in', which='both', left=False, right=False) + if options['hide_x_ticks']: + ax.tick_params(axis='x', direction='in', which='both', bottom=False, top=False) + + + + # Set multiple locators + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + + ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) + ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + + # Change format of axis tick labels if specified: + + + + # Set title + if options['title']: + ax.set_title(options['title']) + + + if options['y_tick_format']: + ax.yaxis.set_major_formatter(FormatStrFormatter(options['y_tick_format'])) + if options['x_tick_format']: + ax.xaxis.set_major_formatter(FormatStrFormatter(options['x_tick_format'])) + + + # Create legend + + if ax.get_legend(): + ax.get_legend().remove() + + + if options['legend'] and options['labels']: + + + # Generate colours + if not options['colours']: + colour_cycle = generate_colours(palettes=options['palettes']) + + colours = [] + for label in options['labels']: + colours.append(next(colour_cycle)) + + + else: + colours = options['colours'] + + if options['label_colours']: + colours = options['label_colours'] + + # Create legend + patches = [] + for i, label in enumerate(options['labels']): + patches.append(mpatches.Patch(color=colours[i], label=label)) + + print(options['legend_ncol']) + + ax.legend(handles=patches, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1], frameon=False, ncol=options['legend_ncol']) + + + + # Adjust where the axes start within the figure. Default value is 10% in from the left and bottom edges. Used to make room for the plot within the figure size (to avoid using bbox_inches='tight' in the savefig-command, as this screws with plot dimensions) + plt.subplots_adjust(left=options['subplots_adjust'][0], bottom=options['subplots_adjust'][1], right=options['subplots_adjust'][2], top=options['subplots_adjust'][3]) + + + # If limits for x- and y-axes is passed, sets these. + if options['xlim']: + ax.set_xlim(options['xlim']) + + if options['ylim']: + ax.set_ylim(options['ylim']) + + + # Add custom text + if options['text']: + plt.text(x=options['text'][1][0], y=options['text'][1][1], s=options['text'][0]) + + + + if options['e_fermi']: + if options['flip_xy']: + ax.axhline(0, c='black', ls='dashed') + else: + ax.axvline(0, c='black', ls='dashed') + + if options['plot_kind'] == 'DOS' or options['plot_kind'] == 'PDOS': + if options['dos_info']['spin_polarised']: + if options['flip_xy']: + ax.axvline(0, c='black') + else: + ax.axhline(0, c='black') + elif options['plot_kind'] == 'COOP' or options['plot_kind'] == 'COHP': + if options['flip_xy']: + ax.axvline(0, c='black') + else: + ax.axhline(0, c='black') + + return fig, ax + + + +def plot_coop(plot_data, options): + ''' interactions = list with number of interaction (index + 1 of interactions list from read_coop)''' + + + required_options = ['plot_kind', 'mode', 'up', 'down', 'collapse', 'interactions', 'flip_xy', 'fill', 'colours', 'palettes'] + + default_options = { + 'plot_kind': 'COOP', + 'mode': 'individual', + 'fill': False, + 'up': True, + 'down': True, + 'collapse': False, + 'interactions': None, + 'palettes': [('qualitative', 'Dark2_8')], + 'colours': None, + 'flip_xy': False + + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + + fig, ax = prepare_plot(options=options) + + coopcar, coop_interactions = dft.io.read_coop(plot_data=plot_data, options=options) + + + + if not options['colours']: + colour_cycle = generate_colours(palettes=options['palettes']) + + colours = [] + for interaction in range(len(coop_interactions)): + colours.append(next(colour_cycle)) + + else: + colours = options['colours'] + + + # If interactions has been specified + if options['interactions']: + + # Make interactions into a list of lists for correct looping below + if type(options['interactions'][0]) != list: + interactions_list = [options['interactions']] + else: + interactions_list = options['interactions'] + + for ind, interactions in enumerate(interactions_list): + + # Determine which columns to plot if collapse is enabled + if options['collapse']: + to_plot = [2*(i-1)+3 for i in interactions] + + # Make sum column for plotting if sum mode is enabled + if options['mode'] == 'sum': + coopcar["sum"] = coopcar.iloc[:, to_plot].sum(axis=1) + to_plot = ['sum'] + + + # Make mean column for plotting if mean mode is enabeld + elif options['mode'] == 'avg' or options['mode'] == 'average' or options['mode'] == 'mean': + coopcar["mean"] = coopcar.iloc[:, to_plot].mean(axis=1) + to_plot = ['mean'] + + # Determine which columns to plot if collapse is disabled and both up and down should be plotted + elif options['up'] and options['down']: + to_plot_up = [2*(i-1)+3 for i in interactions] + to_plot_down = [2*(i-1)+5 +2*len(coop_interactions) for i in interactions] + to_plot = to_plot_up + to_plot_down + + if options['mode'] == 'sum': + coopcar["sum_up"] = coopcar.iloc[:, to_plot_up].sum(axis=1) + coopcar["sum_down"] = coopcar.iloc[:, to_plot_down].sum(axis=1) + to_plot = ['sum_up', 'sum_down'] + + elif options['mode'] == 'avg' or options['mode'] == 'average' or options['mode'] == 'mean': + coopcar["mean_up"] = coopcar.iloc[:, to_plot_up].mean(axis=1) + coopcar["mean_down"] = coopcar.iloc[:, to_plot_down].mean(axis=1) + to_plot = ['mean_up', 'mean_down'] + + # Determine which columns to plot if collapse is disabled and only up should be plotted + elif options['up']: + to_plot = [2*(i-1)+3 for i in interactions] + + if options['mode'] == 'sum': + coopcar["sum_up"] = coopcar.iloc[:, to_plot].sum(axis=1) + to_plot = ['sum_up'] + + elif options['mode'] == 'avg' or options['mode'] == 'average' or options['mode'] == 'mean': + coopcar["mean_up"] = coopcar.iloc[:, to_plot].mean(axis=1) + to_plot = ['mean_up'] + + + # Determine which columns to plot if collapse is disabled and only down should be plotted + elif options['down']: + to_plot = [2*(i-1)+5 +2*len(coop_interactions) for i in interactions] + + if options['mode'] == 'sum': + coopcar["sum_down"] = coopcar.iloc[:, to_plot].sum(axis=1) + to_plot = ['sum_down'] + + elif options['mode'] == 'avg' or options['mode'] == 'average' or options['mode'] == 'mean': + coopcar["mean_down"] = coopcar.iloc[:, to_plot].mean(axis=1) + to_plot = ['mean_down'] + + + # Plot all columns as decided above + for j, column in enumerate(to_plot): + if options['fill']: + ax.fill_between(coopcar["Energy"], coopcar[column], 0, where=coopcar[column]>0, color=colours[ind]) + ax.fill_between(coopcar["Energy"], coopcar[column], 0, where=coopcar[column]<0, color=colours[ind+1]) + + else: + if options['mode'] == "individual": + colour = colours[j] + else: + colour = colours[ind] + + + if options['flip_xy']: + coopcar.plot(y='Energy', x=column, ax=ax, color=colour) + else: + coopcar.plot(x='Energy', y=column, ax=ax, color=colour) + + prettify_dos_plot(fig=fig, ax=ax, options=options) + + return coopcar + + + +def prettify_coop_plot(fig, ax, energyunit='eV', dosunit='arb. u.', xlim=None, ylim=None, title=None, hide_ylabels=False, hide_xlabels=False, hide_yvals=False, hide_xvals=False, flip_xy=False, pad_bottom=None, scale=1, colours=None, atoms=None, pdos=False, width=None, height=None, e_fermi=False, adjust=False, legend=False, labels=None, label_colours=None, xpad=0, ypad=0): + + # Set sizes of ticks, labes etc. + ticksize = 30*scale + labelsize = 30*scale + legendsize = 30*scale + titlesize = 30*scale + + linewidth = 3*scale + axeswidth = 3*scale + majorticklength = 20*scale + minorticklength = 10*scale + + plt.xticks(fontsize=ticksize) + plt.yticks(fontsize=ticksize) + + if flip_xy: + + # Set labels on x- and y-axes + if not hide_ylabels: + if ypad: + ax.set_ylabel('Energy [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + else: + ax.set_ylabel('Energy [{}]'.format(energyunit), size=labelsize) + + if pdos: + if xpad: + ax.set_xlabel('COOP [{}]'.format(dosunit), size=labelsize, labelpad=xpad) + else: + ax.set_xlabel('COOP [{}]'.format(dosunit), size=labelsize) + + else: + if width >= 10: + if xpad: + ax.set_xlabel('COOP [{}]'.format(dosunit), size=labelsize, labelpad=xpad) + else: + ax.set_xlabel('COOP [{}]'.format(dosunit), size=labelsize) + + else: + if xpad: + ax.set_xlabel('COOP [{}]'.format(dosunit), size=labelsize, labelpad=xpad) + else: + ax.set_xlabel('COOP [{}]'.format(dosunit), size=labelsize) + + ax.tick_params(axis='y', direction='in', which='major', right=True, length=majorticklength, width=linewidth) + ax.tick_params(axis='y', direction='in', which='minor', right=True, length=minorticklength, width=linewidth) + + if hide_yvals: + ax.tick_params(axis='y', labelleft=False) + + ax.tick_params(axis='x', direction='in', which='major', bottom=False, labelbottom=False) + + ax.yaxis.set_major_locator(MultipleLocator(1)) + ax.yaxis.set_minor_locator(MultipleLocator(.5)) + + + + else: + # Set labels on x- and y-axes + if adjust: + if xpad: + ax.set_xlabel('E - E$_F$ [{}]'.format(energyunit), size=labelsize, labelpad=xpad) + else: + ax.set_xlabel('E - E$_F$ [{}]'.format(energyunit), size=labelsize) + + + else: + if xpad: + ax.set_xlabel('Energy [{}]'.format(energyunit), size=labelsize, labelpad=xpad) + else: + ax.set_xlabel('Energy [{}]'.format(energyunit), size=labelsize) + + + if height < 10: + if ypad: + ax.set_ylabel('COOP [{}]'.format(dosunit), size=labelsize, labelpad=ypad) + else: + ax.set_ylabel('COOP [{}]'.format(dosunit), size=labelsize) + + else: + if ypad: + ax.set_ylabel('Crystal orbital overlap population [{}]'.format(dosunit), size=labelsize, labelpad=ypad) + else: + ax.set_ylabel('Crystal orbital overlap population [{}]'.format(dosunit), size=labelsize) + + + ax.tick_params(axis='x', direction='in', which='major', bottom=True, top=True, length=majorticklength, width=linewidth) + ax.tick_params(axis='x', direction='in', which='minor', bottom=True, top=True, length=minorticklength, width=linewidth) + + + ax.tick_params(axis='y', which='major', direction='in', right=True, left=True, labelleft=True, length=majorticklength, width=linewidth) + ax.tick_params(axis='y', which='minor', direction='in', right=True, left=True, length=minorticklength, width=linewidth) + + if hide_ylabels: + ax.set_ylabel('') + if hide_xlabels: + ax.set_xlabel('') + if hide_yvals: + ax.tick_params(axis='y', which='both', labelleft=False) + if hide_xvals: + ax.tick_params(axis='x', which='both', labelbottom=False) + + + if ylim: + yspan = ylim[1] - ylim[0] + yloc = np.round(yspan / 4, 2) + + ax.yaxis.set_major_locator(MultipleLocator(yloc)) + ax.yaxis.set_minor_locator(MultipleLocator(yloc/2)) + + + + ax.xaxis.set_major_locator(MultipleLocator(1)) + ax.xaxis.set_minor_locator(MultipleLocator(.5)) + + + + plt.xlim(xlim) + plt.ylim(ylim) + + + if title: + ax.set_title(title, size=40) + + + + if legend: + patches = [] + + if label_colours: + colours=label_colours + + for ind, label in enumerate(labels): + patches.append(mpatches.Patch(color=colours[ind], label=label)) + + fig.legend(handles=patches, loc='upper right', ncol=len(labels), bbox_to_anchor=(0.8, 0.45), fontsize=legendsize/1.25, frameon=False) + + #bbox_to_anchor=(1.20, 0.91) + + + + if pad_bottom is not None: + bigax = fig.add_subplot(111) + bigax.set_facecolor([1,1,1,0]) + bigax.spines['top'].set_visible(False) + bigax.spines['bottom'].set_visible(True) + bigax.spines['left'].set_visible(False) + bigax.spines['right'].set_visible(False) + bigax.tick_params(labelcolor='w', color='w', direction='in', top=False, bottom=True, left=False, right=False, labelleft=False, pad=pad_bottom) + + + + if xpad: + ax.tick_params(axis='x', pad=xpad) + + if ypad: + ax.tick_params(axis='y', pad=ypad) + + if e_fermi: + if flip_xy: + plt.axhline(0, lw=linewidth, c='black', ls='dashed') + else: + plt.axvline(0, lw=linewidth, c='black', ls='dashed') + + + plt.axhline(0, lw=linewidth, c='black') + + return fig, ax + + + + +def get_unique_atoms(interactions): + ''' Get all the unique atoms involved in the interactions from the COOP-calculation + + Input: + interactions: list of interactions that comes as output from read_coop() + + Outut: + unique_atoms: list of unique atoms in the interactions list''' + + unique_atoms = [] + + for interaction in interactions: + + atoms = interaction.split('->') + + for atom in atoms: + if atom not in unique_atoms: + unique_atoms.append(atom) + + + unique_atoms.sort() + + return unique_atoms + + +def get_interactions_involving(interactions, targets): + ''' Get the indicies (+1) of all the interactions involving target. This list can be used as input to plot_coop(), as it is + then formatted the way that function accepts these interactions. + + Input: + interactions: list of interactions as output from read_coop() + target: the particular atom that should be involved in the interactions contained in the output list + + Output: + target_interactions: Indices (+1) of all the interactions involving target atom.''' + + target_interactions = [] + appended_interactions = [] + + + if type(targets) == list: + for target in targets: + for ind, interaction in enumerate(interactions): + if target in interaction.split('->') and interaction not in appended_interactions: + target_interactions.append(ind+1) + appended_interactions.append(interaction) + + else: + for ind, interaction in enumerate(interactions): + if targets in interaction.split('->'): + target_interactions.append(ind+1) + + + return target_interactions + + + + +def update_rc_params(rc_params): + ''' Update all passed run commands in matplotlib''' + + if rc_params: + for key in rc_params.keys(): + plt.rcParams.update({key: rc_params[key]}) + + + +def update_options(options, required_options, default_options): + ''' Update all passed options''' + + + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + + + return options + + + +def determine_width(options): + + conversion_cm_inch = 0.3937008 # cm to inch + + if options['column_type'] == 'single': + column_width = options['single_column_width'] + elif options['column_type'] == 'double': + column_width = options['double_column_width'] + + column_width *= conversion_cm_inch + + + width_ratio = [float(num) for num in options['width_ratio'].split(':')] + + + width = column_width * width_ratio[0]/width_ratio[1] + + + return width + + +def determine_height(options, width): + + aspect_ratio = [float(num) for num in options['aspect_ratio'].split(':')] + + height = width/(aspect_ratio[0] / aspect_ratio[1]) + + return height + + +def scale_figure(options, width, height): + width = width * options['upscaling_factor'] * options['compress_width'] + height = height * options['upscaling_factor'] * options['compress_height'] + + return width, height + + + +def swap_values(dict, key1, key2): + + key1_val = dict[key1] + dict[key1] = dict[key2] + dict[key2] = key1_val + + return dict + + + +def generate_colours(options: dict): + + if not isinstance(options['palettes'], list): + options['palettes'] = [options['palettes']] + + # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. + colour_collection = [] + + for palette in options['palettes']: + mod = importlib.import_module("palettable.colorbrewer.%s" % palette[0]) + colour = getattr(mod, palette[1]).mpl_colors + colour_collection = colour_collection + colour + + colour_cycle = itertools.cycle(colour_collection) + + + return colour_cycle \ No newline at end of file diff --git a/nafuma/dft/io.py b/nafuma/dft/io.py new file mode 100644 index 0000000..0c32451 --- /dev/null +++ b/nafuma/dft/io.py @@ -0,0 +1,545 @@ +import pandas as pd +import matplotlib.pyplot as plt +import numpy as np + +import linecache + +import nafuma.auxillary as aux + + +def open_doscar(doscar='DOSCAR'): + with open(doscar, 'r') as dos: + lines = dos.readlines() + + return lines + +def open_poscar(poscar='POSCAR'): + with open(poscar, 'r') as pos: + lines = pos.readlines() + + return lines + +def open_outcar(outcar='OUTCAR'): + with open(outcar, 'r') as out: + lines = out.readlines() + + return lines + +def get_number_of_atoms(doscar='DOSCAR'): + lines = open_doscar(doscar) + + return int(lines[0].strip().split()[0]) + +def get_atoms(poscar): + + with open(poscar, 'r') as poscar: + lines = poscar.readlines() + + atoms = lines[5].split() + atom_num = lines[6].split() + + + atom_num = [int(num) for num in atom_num] + + atoms_dict = {} + + for ind, atom in enumerate(atoms): + atoms_dict[atom] = atom_num[ind] + + return atoms, atom_num, atoms_dict + + +def get_fermi_level(doscar='DOSCAR', headerlines=6): + lines = open_doscar(doscar) + + return float(lines[headerlines-1].strip().split()[3]) + + +def get_valence_electron_count(outcar='OUTCAR', poscar='POSCAR'): + lines = open_outcar(outcar) + + atoms, atoms_dict = get_atoms(poscar) + n_atoms = len(atoms) + + for line in lines: + line = line.strip() + if line[0:4] == "ZVAL": + valence_electrons = line.split()[-n_atoms:] + break + + return valence_electrons + + + + +def read_coop(data={}, options={}): + ''' Reads a COOPCAR.lobster file and prepares the DataFrame for further data handling. + + Input: + path: The path to the COOPCAR.lobster-file + + Output: + coopcar: A DataFrame containing the COOP data + interactions: A list of interactions''' + + + required_options = ['collapse'] + + default_options = { + 'collapse': False + } + + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + interactions = determine_interactions(data['coopcar']) + + coopcar = pd.read_csv(data['coopcar'], skiprows=3+len(interactions), header=None, delim_whitespace=True) + + spin_polarised = determine_spin_polarisation(coopcar, interactions) + + + # Create list of column names + # If the run is spin polarised, add all up states first and then all down states + if spin_polarised: + columns = ['Energy', 'avg_up', 'avg_int_up'] + for i in range(1, len(interactions)+1): + columns += ['interaction{}_up'.format(i), 'interaction{}_int_up'.format(i)] + columns += ['avg_down', 'avg_int_down'] + for i in range(1, len(interactions)+1): + columns += ['interaction{}_down'.format(i), 'interaction{}_int_down'.format(i)] + + # Otherwise just + else: + columns = ['Energy', 'avg', 'avg_int'] + for i in range(1, len(interactions)+1): + columns += ['interaction_{}'.format(i), 'interaction_{}_int'.format(i)] + + + + + coopcar.columns = columns + + + if options['collapse']: + + columns_collapsed = ['Energy'] + + for column in columns: + if column.split('_')[0] not in columns_collapsed: + columns_collapsed.append(''.join(column.split('_')[:-1])) + columns_collapsed.append(''.join(column.split('_')[:-1])+'_int') + + + for column in columns_collapsed: + if column != 'Energy': + coopcar[column] = coopcar[column+'_up'] + coopcar[column+'_down'] + + columns.remove('Energy') + coopcar.drop(columns, axis=1, inplace=True) + coopcar.columns = columns_collapsed + + + + + + return coopcar, interactions + + +def read_cohp(path, flip_sign=False): + ''' Reads a COHPCAR.lobster file and prepares the DataFrame for further data handling. + + Input: + path: The path to the COOPCAR.lobster-file + flip_sign: Boolean value to determine whether all COHP-values should be flipped (that is, return -COHP) + + Output: + coopcar: A DataFrame containing the COOP data + interactions: A list of interactions''' + + interactions = determine_interactions(path) + + cohpcar = pd.read_csv(path, skiprows=3+len(interactions), header=None, delim_whitespace=True) + + spin_polarised = determine_spin_polarisation(cohpcar, interactions) + + + # Create list of column names + # If the run is spin polarised, add all up states first and then all down states + if spin_polarised: + columns = ['Energy', 'avg_up', 'avg_up_int'] + for i in range(1, len(interactions)+1): + columns += ['interaction_{}_up'.format(i), 'interaction_{}_up_int'.format(i)] + columns += ['avg_down', 'avg_down_int'] + for i in range(1, len(interactions)+1): + columns += ['interaction_{}_down'.format(i), 'interaction_{}_up_int'.format(i)] + + # Otherwise just + else: + columns = ['Energy', 'avg', 'avg_int'] + for i in range(1, len(interactions)+1): + columns += ['interaction_{}'.format(i), 'interaction_{}_int'.format(i)] + + + cohpcar.columns = columns + + if flip_sign: + columns = columns[1:] + + for column in columns: + cohpcar[column] = -cohpcar[column] + + return cohpcar, interactions + + +def determine_interactions(path): + ''' Determines the number of interactions present in the COOPCAR.lobster file. + + Input: + path: The path to the COOPCAR.lobster-file + + Output: + interactions: A list of strings with the interactions in the COOPCAR.lobster file''' + + + with open(path, 'r') as coop: + lines = coop.readlines() + + + interactions = [] + for line in lines: + if line[0:2] == 'No': + interactions.append(line.split(':')[-1].split('(')[0]) + + + return interactions + + + +def determine_spin_polarisation(coopcar, interactions): + ''' Determines whether a COOPCAR.lobster file is spin polarised or not. + + Input: + coopcar: A DataFrame containing the COOP data + interactions: A list of interactions obtained from the determine_interactions function + + Output: + spin_polarised: Boolean value to indicate whether or not the COOP data is spin polarised''' + + number_of_interactions = len(interactions) + + spin_polarised = True if coopcar.shape[1] == 4*number_of_interactions + 5 else False + + return spin_polarised + + + + +def read_dos(path, flip_down=True): + + dos_info = get_doscar_information(path) + + with open(path, 'r') as doscar: + count = 0 + raw_data = [] + + while count < dos_info["NEDOS"] + 6: + if count >= 6: + data_line = [float(x) for x in doscar.readline().split()] + raw_data.append(data_line) + + else: + doscar.readline() + + count += 1 + + + dos = pd.DataFrame(raw_data) + + if dos_info["spin_polarised"]: + header_names = ['energy', 'total_up', 'total_down', 'total_integrated_up', 'total_integrated_down'] + else: + header_names = ['energy', 'total', 'total_integrated'] + + + dos.columns = header_names + + if dos_info["spin_polarised"] and flip_down: + dos["total_down"] = -dos["total_down"] + dos["total_integrated_down"] = -dos["total_integrated_down"] + + return dos + + +def read_pdos(data: dict, options={}): + + required_options = ['flip_down', 'sum_atoms', 'sum_orbitals', 'collapse_spin', 'adjust', 'manual_adjust', 'normalise', 'normalise_unit_cell', 'normalisation_factor'] + + default_options = { + 'flip_down': True, + 'sum_atoms': False, + 'sum_orbitals': False, + 'collapse_spin': False, + 'adjust': False, + 'manual_adjust': None, + 'normalise': False, + 'normalise_unit_cell': False, + 'normalisation_factor': None + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + # Get information about the DOSCAR-file (ENMIN, ENMAX, NEDOS, EFERMI and spin_polarised) + dos_info = get_doscar_information(data['doscar']) + + # Get information from the POSCAR-file (or CONTCAR) (which species, number of atoms per specie and a dictionary matching the two) + species, atom_num, atoms_dict = get_atoms(data['poscar']) + + + # Open DOSCAR and read all lines + with open(data['doscar'], 'r') as file: + doscar_data = file.readlines() + + # Make list of all individual atoms + atoms = [] + for specie in species: + for i in range(0,atoms_dict[specie]): + atoms.append(specie) + + + # Loop through all the atoms and make a DataFrame for each atom. + + pdos_full = [] + for ind, atom in enumerate(atoms): + + # Define line to start reading + start = int((1 + ind) * dos_info["NEDOS"] + 6 + (ind * 1)) + + + + add_d_orbitals = False + add_p_orbitals = False + # Check if d-orbitals are included (if DOSCAR comes from LOBSTER), otherwise they will have to be added to make the DataFrames the same size + if dos_info['kind'] == 'LOBSTER': + # Extract basis sets from each atom + basis_sets = doscar_data[start].split(';')[-1] + + if not 'd' in basis_sets: + add_d_orbitals = True + + if basis_sets.count('p') != 6: + add_p_orbitals = True + + + + # Start reading lines into a list and convert to DataFrame and cast all entries as float + pdos_atom = [] + for i in range(1,dos_info["NEDOS"]+1): + pdos_atom.append(doscar_data[start+i].split()) + + + pdos_atom = pd.DataFrame(pdos_atom) + pdos_atom = pdos_atom.astype(float) + + + + + # Give the columns names and add second set of p-orbitals, d-orbitals, or both if they don't exist in the data + if add_p_orbitals: + if dos_info['spin_polarised']: + pdos_atom.columns = ['Energy', 's_u', 's_d', 'p1_u', 'p1_d', 'p2_u', 'p2_d', 'p3_u', 'p3_d'] + pdos_atom[['2p1_u', '2p1_d', '2p2_u', '2p2_d', '2p3_u', '2p3_d']] = 0 + + if add_d_orbitals: + pdos_atom[['d1_u', 'd1_d', 'd2_u', 'd2_d', 'd3_u', 'd3_d', 'd4_u', 'd4_d', 'd5_u', 'd5_d']] = 0 + pdos_atom['tot_u'] = pdos_atom[['s_u', 'p1_u', 'p2_u', 'p3_u', '2p1_u', '2p2_u', '2p3_u', 'd1_u', 'd2_u', 'd3_u', 'd4_u', 'd5_u']].sum(axis=1) + pdos_atom['tot_d'] = pdos_atom[['s_d', 'p1_d', 'p2_d', 'p3_d', '2p1_d', '2p2_d', '2p3_d', 'd1_d', 'd2_d', 'd3_d', 'd4_d', 'd5_d']].sum(axis=1) + + else: + pdos_atom['tot_u'] = pdos_atom[['s_u', 'p1_u', 'p2_u', 'p3_u', '2p1_u', '2p2_u', '2p3_u', 'd1_u', 'd2_u', 'd3_u', 'd4_u', 'd5_u']].sum(axis=1) + pdos_atom['tot_d'] = pdos_atom[['s_d', 'p1_d', 'p2_d', 'p3_d', '2p1_d', '2p2_d', '2p3_d', 'd1_d', 'd2_d', 'd3_d', 'd4_d', 'd5_d']].sum(axis=1) + + + + elif add_d_orbitals: + if dos_info["spin_polarised"]: + pdos_atom.columns = ['Energy', 's_u', 's_d', 'p1_u', 'p1_d', 'p2_u', 'p2_d', 'p3_u', 'p3_d'] + pdos_atom[['d1_u', 'd1_d', 'd2_u', 'd2_d', 'd3_u', 'd3_d', 'd4_u', 'd4_d', 'd5_u', 'd5_d']] = 0 + pdos_atom['tot_u'] = pdos_atom[['s_u', 'p1_u', 'p2_u', 'p3_u', '2p1_u', '2p2_u', '2p3_u', 'd1_u', 'd2_u', 'd3_u', 'd4_u', 'd5_u']].sum(axis=1) + pdos_atom['tot_d'] = pdos_atom[['s_d', 'p1_d', 'p2_d', 'p3_d', '2p1_d', '2p2_d', '2p3_d', 'd1_d', 'd2_d', 'd3_d', 'd4_d', 'd5_d']].sum(axis=1) + + + else: + pdos_atom.columns = ['Energy', 's', 'p1', 'p2', 'p3'] + pdos_atom[['d1', 'd2', 'd3', 'd4', 'd5']] = 0 + pdos_atom['tot'] = pdos_atom[['s', 'p1', 'p2', 'p3', '2p1', '2p2', '2p3', 'd1', 'd2', 'd3', 'd4', 'd5']].sum(axis=1) + + else: + if dos_info["spin_polarised"]: + if dos_info['kind'] == 'LOBSTER': + pdos_atom.columns = ['Energy', 's_u', 's_d', 'p1_u', 'p1_d', 'p2_u', 'p2_d', 'p3_u', 'p3_d', '2p1_u', '2p1_d', '2p2_u', '2p2_d', '2p3_u', '2p3_d', 'd1_u', 'd1_d', 'd2_u', 'd2_d', 'd3_u', 'd3_d', 'd4_u', 'd4_d', 'd5_u', 'd5_d'] + pdos_atom['tot_u'] = pdos_atom[['s_u', 'p1_u', 'p2_u', 'p3_u', '2p1_u', '2p2_u', '2p3_u','d1_u', 'd2_u', 'd3_u', 'd4_u', 'd5_u']].sum(axis=1) + pdos_atom['tot_d'] = pdos_atom[['s_d', 'p1_d', 'p2_d', 'p3_d', '2p1_d', '2p2_d', '2p3_d', 'd1_d', 'd2_d', 'd3_d', 'd4_d', 'd5_d']].sum(axis=1) + + elif dos_info['kind'] == 'VASP': + pdos_atom.columns = ['Energy', 's_u', 's_d', 'p1_u', 'p1_d', 'p2_u', 'p2_d', 'p3_u', 'p3_d', 'd1_u', 'd1_d', 'd2_u', 'd2_d', 'd3_u', 'd3_d', 'd4_u', 'd4_d', 'd5_u', 'd5_d'] + pdos_atom['tot_u'] = pdos_atom[['s_u', 'p1_u', 'p2_u', 'p3_u', 'd1_u', 'd2_u', 'd3_u', 'd4_u', 'd5_u']].sum(axis=1) + pdos_atom['tot_d'] = pdos_atom[['s_d', 'p1_d', 'p2_d', 'p3_d', 'd1_d', 'd2_d', 'd3_d', 'd4_d', 'd5_d']].sum(axis=1) + + else: + pdos_atom.columns = ['Energy', 's', 'p1', 'p2', 'p3', '2p1', '2p2', '2p3', 'd1', 'd2', 'd3', 'd4', 'd5'] + pdos_atom['tot'] = pdos_atom[['s', 'p1', 'p2', 'p3', '2p1', '2p2', '2p3', 'd1', 'd2', 'd3', 'd4', 'd5']].sum(axis=1) + + + + # Change the sign of the spin down columns if flip_down is True + if options['flip_down'] and dos_info['spin_polarised'] and not options['collapse_spin']: + down = [orbital for orbital in pdos_atom.columns if '_d' in orbital] + + for orbital in down: + pdos_atom[orbital] = -pdos_atom[orbital] + + + if options['manual_adjust']: + pdos_atom["Energy"] = pdos_atom["Energy"] - options['manual_adjust'] + + + + pdos_full.append(pdos_atom) + + + # If sum_atoms is True, all DataFrames of a given specie will be added together + if options['sum_atoms']: + + # Initalise a new emtpy list that will become our pdos_full + pdos_full_sum_atoms = [] + + + start = 0 + # Loop through each specie so that each specie will get exactly one DataFrame + for specie in species: + # Initialise with first DataFrame of the specie + atom_pdos_summed = pdos_full[start] + + # Loop over DOSes and add if there's a match for specie + for ind, pdos in enumerate(pdos_full): + if atoms[ind] == specie and ind != start: + atom_pdos_summed = atom_pdos_summed + pdos + + + # Divide the Energy by the number of DataFrames added to get back to the original value of the Energy + atom_pdos_summed["Energy"] = atom_pdos_summed["Energy"] / atoms_dict[specie] + + + if options['normalise']: + + if dos_info["spin_polarised"]: + columns = atom_pdos_summed.columns + #columns = ['s_u', 's_d', 'p1_u', 'p1_d', 'p2_u', 'p2_d', 'p3_u', 'p3_d', '2p1_u', '2p1_d', '2p2_u', '2p2_d', '2p3_u', '2p3_d', 'd1_u', 'd1_d', 'd2_u', 'd2_d', 'd3_u', 'd3_d', 'd4_u', 'd4_d', 'd5_u', 'd5_d', 'tot_u', 'tot_d'] + for column in columns: + atom_pdos_summed[column] = atom_pdos_summed[column] / atoms_dict[specie] + + + # Append the new DataFrame for a given specie to the list + pdos_full_sum_atoms.append(atom_pdos_summed) + + start += atoms_dict[specie] + + + # Rename the list + pdos_full = pdos_full_sum_atoms + + + # If collapse_spin is True for a spin polarised DOSCAR, the up and down channels of each orbital will be added together. + if options['collapse_spin'] and dos_info['spin_polarised']: + + pdos_full_spin_collapsed = [] + for pdos in pdos_full: + temp_pdos = pd.DataFrame() + + temp_pdos["Energy"] = pdos["Energy"] + + temp_pdos['s'] = pdos[['s_u', 's_d']].sum(axis=1) + + temp_pdos['p1'] = pdos[['p1_u', 'p1_d']].sum(axis=1) + temp_pdos['p2'] = pdos[['p2_u', 'p2_d']].sum(axis=1) + temp_pdos['p3'] = pdos[['p3_u', 'p3_d']].sum(axis=1) + + temp_pdos['2p1'] = pdos[['2p1_u', '2p1_d']].sum(axis=1) + temp_pdos['2p2'] = pdos[['2p2_u', '2p2_d']].sum(axis=1) + temp_pdos['2p3'] = pdos[['2p3_u', '2p3_d']].sum(axis=1) + + temp_pdos['d1'] = pdos[['d1_u', 'd1_d']].sum(axis=1) + temp_pdos['d2'] = pdos[['d2_u', 'd2_d']].sum(axis=1) + temp_pdos['d3'] = pdos[['d3_u', 'd3_d']].sum(axis=1) + temp_pdos['d4'] = pdos[['d4_u', 'd4_d']].sum(axis=1) + temp_pdos['d5'] = pdos[['d5_u', 'd5_d']].sum(axis=1) + + temp_pdos['tot'] = pdos[['tot_u', 'tot_d']].sum(axis=1) + + pdos_full_spin_collapsed.append(temp_pdos) + + + pdos_full = pdos_full_spin_collapsed + dos_info['spin_polarised'] = False + + + + # If sum_orbitals is True, all columns belonging to a particular set of orbitals will be added together. + if options['sum_orbitals']: + pdos_full_sum_orbitals = [] + + for pdos in pdos_full: + temp_pdos = pd.DataFrame() + + temp_pdos["Energy"] = pdos["Energy"] + + if dos_info['spin_polarised']: + temp_pdos['s_u'] = pdos['s_u'] + temp_pdos['s_d'] = pdos['s_d'] + + temp_pdos['p_u'] = pdos[['p1_u', 'p2_u', 'p3_u']].sum(axis=1) + temp_pdos['p_d'] = pdos[['p1_d', 'p2_d', 'p3_d']].sum(axis=1) + + if len(pdos_full[0].columns) == 25: + temp_pdos['2p_u'] = pdos[['2p1_u', '2p2_u', '2p3_u']].sum(axis=1) + temp_pdos['2p_d'] = pdos[['2p1_d', '2p2_d', '2p3_d']].sum(axis=1) + + temp_pdos['d_u'] = pdos[['d1_u', 'd2_u', 'd3_u', 'd4_u', 'd5_u']].sum(axis=1) + temp_pdos['d_d'] = pdos[['d1_d', 'd2_d', 'd3_d', 'd4_d', 'd5_d']].sum(axis=1) + + temp_pdos['tot_u'] = pdos['tot_u'] + temp_pdos['tot_d'] = pdos['tot_d'] + + else: + temp_pdos['s'] = pdos['s'] + temp_pdos['p'] = pdos[['p1', 'p2', 'p3']].sum(axis=1) + temp_pdos['2p'] = pdos[['2p1', '2p2', '2p3']].sum(axis=1) + temp_pdos['d'] = pdos[['d1', 'd2', 'd3', 'd4', 'd5']].sum(axis=1) + temp_pdos['tot'] = pdos['tot'] + + pdos_full_sum_orbitals.append(temp_pdos) + + pdos_full = pdos_full_sum_orbitals + + + + + return pdos_full, dos_info + + +def get_doscar_information(path): + ''' Reads information from the DOSCAR''' + + kind = 'LOBSTER' if 'LOBSTER' in linecache.getline(path, 5) else 'VASP' + dos_info_raw = linecache.getline(path, 6).split() + spin_polarised = True if len(linecache.getline(path, 7).split()) == 5 else False + + + dos_info = {'ENMIN': float(dos_info_raw[0]), 'ENMAX': float(dos_info_raw[1]), 'NEDOS': int(dos_info_raw[2]), 'EFERMI': float(dos_info_raw[3]), 'spin_polarised': spin_polarised, 'kind': kind} + + return dos_info + + +#def get_bader_charges(poscar='POSCAR', acf='ACF.dat'): + diff --git a/nafuma/dft/structure.py b/nafuma/dft/structure.py new file mode 100644 index 0000000..04584cb --- /dev/null +++ b/nafuma/dft/structure.py @@ -0,0 +1,906 @@ +import math +import re +import pandas as pd +import numpy as np +from scipy.optimize import curve_fit + +import matplotlib.pyplot as plt +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) +import importlib +import matplotlib.patches as mpatches +import matplotlib.lines as mlines +from mpl_toolkits.axisartist.axislines import Subplot +from cycler import cycler +import itertools + +from ase import Atoms +from ase.io.trajectory import Trajectory +from ase import io +from ase.units import kJ +from ase.eos import EquationOfState +import os +import os.path + + +def read_eos_data(path, options): + ''' Reads volume and energy data from a energy-volume run and fits the data to an equation of state. Outputs a list with one pandas DataFrame containing the data points from the DFT-calculations, + one DataFrame containing the fitted curve data points and one dictionary with equilibrium volume, equilibrium energy and bulk modulus in GPa + + path: Path to the folder containing the energ.dat and POSCAR files. energ.dat must have two columns with volumes in the first, energy in the second separated by whitespace. + atoms_per_fu: Number of atoms per formula unit. Used to scale the values to be comparable with other calculations that may have a different sized unit cell. + eos: Type of equation of state to fit to. Same keywords as the ones used in ASE, as it simply calls ASE to fit the equation of state. + ''' + + required_options = ['atoms_per_fu', 'reference', 'eos'] + + default_options = { + 'atoms_per_fu': -1, # Scaling factor to output energy per f.u. + 'reference': 0, # Whether the energy should be relative to some reference energy (typically lowest energy) + 'eos': 'birchmurnaghan', # what type of EoS curve to fit the data to. Options: murnaghan, birch, birchmurnaghan, vinet, pouriertarantola + } + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + + # Make paths for the energ.dat and POSCAR files. + energ_path = os.path.join(path, 'energ.dat') + poscar_path = os.path.join(path, 'POSCAR') + + # Read POSCAR and calculate the scale factor to give values per formula unit + at = io.read(poscar_path) + + if options['atoms_per_fu'] == -1: + scale_factor = 1 + else: + scale_factor = options['atoms_per_fu'] / len(at) + + # Get the label + label = os.path.basename(path) + + # Reads the energ.dat file and structures the data into a pandas DataFrame. Then scales the values according to the scale factor. + dft_df = pd.read_csv(energ_path, delim_whitespace=True, header=None) + dft_df.columns = ['Configuration', 'Volume', 'Energy'] + dft_df['Energy'] = dft_df['Energy'] * scale_factor + dft_df['Volume'] = dft_df['Volume'] * scale_factor + + + dft_df["Energy"] = dft_df["Energy"] - options['reference'] # subtracts a reference energy if provided. THis value defaults to 0, so will not do anything if not provided. + + # Fit data to Equation of State using ASEs EquationOfState object. Makes a DataFrame out of the data points of the fitted curve. Also makes a ditionary of the equilibrium constants, + #then packages everything in a list which is returned by the function. + eos = EquationOfState(dft_df['Volume'].values, dft_df['Energy'].values, eos=options['eos']) + v0, e0, B = eos.fit() + + eos_df = pd.DataFrame(data={'Volume': eos.getplotdata()[4], 'Energy': eos.getplotdata()[5]}) + + equilibrium_constants = {'v0': v0, 'e0': e0,'B': B/kJ * 1.0e24} + + data = [dft_df, eos_df, equilibrium_constants, label] + + return data + + +def read_eos_datas(path, options): + + + required_options = ['subset', 'sort_by'] + + default_options = { + 'subset': None, # list with directory names of what you want to include + 'sort_by': 'e0', # whether the data should be sorted or not - relevant for bar plots, but also for the order of the entries in the legend in the EoScruve plot + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + # If a subset of directories is not specified, will create a list of all directories in the path given. + if not options['subset']: + dirs = [dir for dir in os.listdir(path) if os.path.isdir(os.path.join(path, dir)) and dir[0] != '.'] + else: + dirs = options['subset'] + + + datas = [] + + + # Loop through all subdirectories and reads the data from these. Also appends the name of the directory to the list that is returned from the plot_eos_data() function + for dir in dirs: + subdir = os.path.join(path, dir) + data = read_eos_data(subdir, options) + datas.append(data) + + + # Sorts the data if sort is enabled. + if options['sort_by']: + datas = sort_data(datas, options['sort_by']) + + + return datas + + +def get_summarised_data(path, options): + + datas = read_eos_datas(path=path, options=options) + + summary = [] + for data in datas: + summary.append([data[3], data[2]['e0'], data[2]['v0'], data[2]['B']]) + + df = pd.DataFrame(summary) + df.columns = ['Label', 'E0', 'V0', 'B'] + + emin = df["E0"].min() + + df["dE0"] = df["E0"] - emin + + # Rearranging the columns + df = df[['Label', 'E0', 'dE0', 'V0', 'B']] + + return df + + +def plot_eos_data(path, options): + ''' Plots the data from the energy-volume curve runs. Allows plotting of just the energy-volume curves, a bar plot showing the equilibrium energies or both. + + path: path to where the data is located. It should point to a directory with subdirectories for each structure to be plotted. Inside each of these subdirectories there should be an energ.dat and a POSCAR file. + atoms_per_fu: Number of atoms per formula unit. Used to scale the values to be comparable with other calculations that may have a different sized unit cell. + dirs: List of directory names if only a subset of all available datasets is to be plotted. Defaults to None, and will thus get data from all subdirectories. + eos: Type of equation of state to fit to. Same keywords as the ones used in ASE, as it simply calls ASE to fit the equation of state. + width: Width of the total figure. Defaults to None, which will again default to width=20. + height: Height of the total figure. Defaults to None, which will again will default to height= width / phi where phi is the golden ratio. + dpi: Dots per inch of the figure. Defaults to pyplot's default + colour_cycles: List of tuples with sets of colours for the palettable colour collection. Defaults to two sets of in total 20 colours. Used for giving different colours to energy-volume curves. + energyunit: The energy unit. Defaults to eV per formula unit. Only used on the axis labels. + volumeunit: The volume unit. Defaults to Å^3. Only used on the axis labels. + xlim: Limits of the x-axes. List of min and max. If mode = both is used, has to contain two lists for each of the plots. As the x-limits for a bar plot is nonesense, should just contain a list with a NoneType. + ylim: Limits of the y-axes. List of min and max. If mode = both is used, has to contain two lists for each of the plots. + sort: Whether or not to sort the data from lowest to highest equilibrium energy. Defaults to True. + sort_by: What to sort by if sort is enabled. Defaults to e0. Other options: v0 = equilibrium volumes, B = bulk moduli. Alphabetical order sorting is not implemented. + mode: Determines what to plot. Defaults to energy-volume curves ('curves'). Other options: 'bars', bar-plot of equilibrium energies. 'both', both energy-volume curves and bar plots are plotted side-by-side. + highlight: Takes a list, either of booleans to highlight certain bars (must be the same length as the number of data sets). Alternatively can contain only names of the datasets to highlight. Defaults to None.''' + + + required_options = ['plot_kind', 'highlight', + 'reference', + 'eos', 'sort_by', + 'curve_colours', + 'bar_colours', + 'marker_cycle', + 'ylim', + 'legend_map', + 'rc_params', + 'legend'] + + + default_options = { + 'plot_kind': 'EoScurve', # EoScurve or EoSbars + 'highlight': None, # list with directory names (or Boolean array) of which bars to highlight. Only relevant to EoSbars + 'reference': 0, # Whether the energy should be relative to some reference energy (typically lowest energy) + 'eos': 'birchmurnaghan', # what type of EoS curve to fit the data to. Options: murnaghan, birch, birchmurnaghan, vinet, pouriertarantola + 'sort_by': 'e0', # whether the data should be sorted or not - relevant for bar plots, but also for the order of the entries in the legend in the EoScruve plot + 'curve_colours': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], # a set of two colour cycles from the palettable package. Requires many colours for the EoScurve plot + 'bar_colours': [('qualitative', 'Dark2_3'), ('qualitative', 'Pastel2_3')], # set of two colour cycles from the palettable package. Extracts first element of each to make the highlighted and subdued colours respectively. Should be replaced in future by explicit passing of colours + 'marker_cycle': ('o', '*', '^', 'v', 'd', 'H', '8', '>', 'P', 'X'), # marker styles for the EoScurve plot + 'ylim': None, # y-limits (ist) + 'legend': True, + 'legend_map': None, # a dictionary with mappings between the folder names and what should appear in the legend + 'rc_params': None # dictionary of run commands to update plot style + } + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + # Create path to the data + datas = read_eos_datas(path=path, options=options) + + + ### PLOT THE ENERGY-VOLUME CURVES + if options['plot_kind'] == 'EoScurve': + + # Fetches a figure and axes object from the prepare_plot() function + fig, ax = prepare_plot(options=options) + + # Make an cyclic iterable of markers to be used for the calculated data points. + marker_cycle = itertools.cycle(options['marker_cycle']) + + + # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. + colour_collection = [] + for cycle in options['curve_colours']: + mod = importlib.import_module("palettable.colorbrewer.%s" % cycle[0]) + colour = getattr(mod, cycle[1]).mpl_colors + colour_collection = colour_collection + colour + + colour_cycle = itertools.cycle(colour_collection) + + labels = [] + colours = [] + markers = [] + + + # For each of the data sets, extracts the data and plots them. + for data in datas: + dft_df, eos_df, label = data[0], data[1], data[3] + + + # If ylim is passed, only plot those that have a minimum energy below the max ylim parameter + if options['ylim']: + plot = True if dft_df["Energy"].min() < options['ylim'][1] else False + else: + plot = True + + if plot: + if options['label_map']: + labels.append(options['label_map'][label]) + + colours.append(next(colour_cycle)) + markers.append(next(marker_cycle)) + + dft_df.plot.scatter(x=1, y=2, ax=ax, marker=markers[-1], color=colours[-1], s=20) + eos_df.plot(x=0, y=1, ax=ax, color=colours[-1], label='_', ls='--') + + + if options['legend']: + options['legend_content'] = [labels, colours, markers] + + + ### PLOT THE BAR PLOTS + elif options['plot_kind'] == 'EoSbars': + + # Fetches a figure and axes object from the prepare_plot() function + fig, ax = prepare_plot(options=options) + + e0 = [] + labels = [] + colours = [] + + # Pick out colour for highlighting (NB! These colours are not passed as arguments, but could be in future) + + bar_colours = [] + for cycle in options['bar_colours']: + mod = importlib.import_module("palettable.colorbrewer.%s" % cycle[0]) + bar_colours.append(getattr(mod, cycle[1]).mpl_colors[0]) + + + # Loops through the datasets, picks out equilibrium volume and labels and sets colours according to the whether the highlight option is used or not. + for data in datas: + + if options['ylim']: + plot = True if data[2]['e0'] < options['ylim'][1] else False + else: + plot = True + + if plot: + + # Adds 100 if plotting in relative mode. The bases of the bar plots are sunk by 100 during plotting + adjustment = 100 if options['reference'] != 0 else 100 + print(adjustment) + + e0.append(data[2]['e0']+adjustment) + print(e0[-1]) + labels.append(options['label_map'][data[3]]) + + if options['highlight'] is not None: + if data[3] in options['highlight']: + colours.append(bar_colours[0]) + else: + colours.append(bar_colours[1]) + + elif options['highlight'] is not None and type(options['highlight'][0] == str): + if labels[-1] in options['highlight']: + colours.append(bar_colours[0]) + else: + colours.append(bar_colours[1]) + + else: + colours.append(bar_colours[0]) + + # Makes the bar plot. + bottom = -100 if options['reference'] != 0 else 0 + plt.bar(range(len(e0)), e0, color=colours, bottom=bottom) + plt.xticks(range(len(e0)), labels, rotation=90) + + + fig, ax = prettify_plot(fig=fig, ax=ax, options=options) + + + + +def sort_data(datas, sort_by='e0'): + ''' Bubble sort algorithm to sort the data sets''' + + l = len(datas) + + for i in range(0, l): + for j in range(0, l-i-1): + if datas[j][2]['{}'.format(sort_by)] > datas[j+1][2]['{}'.format(sort_by)]: + temp = datas[j] + datas[j] = datas[j+1] + datas[j+1] = temp + + return datas + + + +def prepare_plot(options={}): + + # Reset run commands + plt.rcdefaults() + + # Update run commands if any is passed + if 'rc_params' in options.keys(): + update_rc_params(options['rc_params']) + + + + required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + default_options = { + 'single_column_width': 8.3, + 'double_column_width': 17.1, + 'column_type': 'single', + 'width_ratio': '1:1', + 'aspect_ratio': '1:1', + 'compress_width': 1, + 'compress_height': 1, + 'upscaling_factor': 1.0, + 'dpi': 600} + + options = update_options(options, required_options, default_options) + + width = determine_width(options) + height = determine_height(options, width) + width, height = scale_figure(options=options, width=width, height=height) + + fig, ax = plt.subplots(figsize=(width, height), dpi=options['dpi']) + + return fig, ax + + + + + + +def update_rc_params(rc_params): + ''' Update all passed run commands in matplotlib''' + + if rc_params: + for key in rc_params.keys(): + plt.rcParams.update({key: rc_params[key]}) + + + +def update_options(options, required_options, default_options): + ''' Update all passed options''' + + + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + + return options + + +def determine_width(options): + + conversion_cm_inch = 0.3937008 # cm to inch + + if options['column_type'] == 'single': + column_width = options['single_column_width'] + elif options['column_type'] == 'double': + column_width = options['double_column_width'] + + column_width *= conversion_cm_inch + + + width_ratio = [float(num) for num in options['width_ratio'].split(':')] + + + width = column_width * width_ratio[0]/width_ratio[1] + + + return width + + +def determine_height(options, width): + + aspect_ratio = [float(num) for num in options['aspect_ratio'].split(':')] + + height = width/(aspect_ratio[0] / aspect_ratio[1]) + + return height + +def scale_figure(options, width, height): + width = width * options['upscaling_factor'] * options['compress_width'] + height = height * options['upscaling_factor'] * options['compress_height'] + + return width, height + + + +def prepare_plot_old(width=None, height=None, dpi=None, energyunit='eV', volumeunit=r'Å$^3$', mode='curves', width_ratio=[1, 1], square=True, pad_bottom=None, scale=1, format_params=None): + '''Prepares pyplot figure and axes objects.''' + + + + linewidth = 3*scale + axeswidth = 3*scale + + plt.rc('lines', linewidth=linewidth) + plt.rc('axes', linewidth=axeswidth) + + + if square: + if not width: + width = 20 + + height = width + + + else: + if not width: + width = 20 + + + if not height: + golden_ratio = (math.sqrt(5) - 1) / 2 + height = width*golden_ratio + + + + if mode == 'curves': + + fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(width, height), facecolor='w', dpi=dpi) + + + if mode == 'bars': + + fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(width, height), facecolor='w', dpi=dpi) + + + if mode == 'both': + + fig, ax = plt.subplots(1, 2, figsize=(width, height), gridspec_kw={'width_ratios': width_ratio}) + + + return fig, ax + + +def prettify_plot(fig, ax, options): + '''Prepares pyplot figure and axes objects.''' + + required_options = ['plot_kind', 'hide_x_labels', 'hide_y_labels', 'xunit', 'yunit', 'legend_content', 'legend_position', 'x_tick_locators', 'y_tick_locators', 'tick_directions', 'subplots_adjust', 'xlim', 'ylim'] + + default_options = { + 'plot_kind': 'EoScurve', # EoScurve or EoSbars + 'hide_x_labels': False, # Whether x labels should be hidden + 'hide_y_labels': False, # whether y labels should be hidden + 'xunit': r'Å$^3$', # The unit of the x-values in the curve plot + 'yunit': r'eV f.u.$^{-1}$', # The unit of the y-values in the curve and bar plots + 'xlim': None, + 'ylim': None, + 'legend_content': None, + 'legend_position': ['upper center', (1.10, 0.90)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'x_tick_locators': [10, 5], # Major and minor tick locators + 'y_tick_locators': [.1, .05], # Major and minor tick locators + 'tick_directions': 'in', # in or out + 'subplots_adjust': [0.1, 0.1, 0.9, 0.9] + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + + if options['plot_kind'] == 'EoScurve': + + # Set labels on x- and y-axes + ax.set_xlabel('Volume [{}]'.format(options['xunit'])) + + if not options['hide_y_labels']: + ax.set_ylabel('Energy [{}]'.format(options['yunit'])) + else: + ax.set_ylabel('') + ax.tick_params(labelleft=False) + + + ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) + ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + + if ax.get_legend(): + ax.get_legend().remove() + + + if options['legend']: + labels = options['legend_content'][0] + colours = options['legend_content'][1] + markers = options['legend_content'][2] + + entries = [] + + for i in range(len(options['legend_content'][0])): + entries.append(mlines.Line2D([], [], label=labels[i], color=colours[i], marker=markers[i], linestyle='None')) + + + fig.legend(handles=entries, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1], frameon=False) + + + + + if options['plot_kind'] == 'EoSbars': + + if not options['hide_y_labels']: + ax.set_ylabel('Energy [{}]'.format(options['yunit'])) + + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + + ax.tick_params(axis='x', which='minor', bottom=False, top=False) + + + + + # Adjust where the axes start within the figure. Default value is 10% in from the left and bottom edges. Used to make room for the plot within the figure size (to avoid using bbox_inches='tight' in the savefig-command, as this screws with plot dimensions) + plt.subplots_adjust(left=options['subplots_adjust'][0], bottom=options['subplots_adjust'][1], right=options['subplots_adjust'][2], top=options['subplots_adjust'][3]) + + + # If limits for x- and y-axes is passed, sets these. + if options['xlim'] is not None: + ax.set_xlim(options['xlim']) + + if options['ylim'] is not None: + ax.set_ylim(options['ylim']) + + + return fig, ax + + + +def prettify_plot_old(fig, ax, energyunit='eV', volumeunit=r'Å$^3$', mode='curves', legend_content=None, pad_bottom=None, scale=1, hide_ylabels=False, xpad=None, ypad=None): + '''Prepares pyplot figure and axes objects.''' + + # Set sizes of ticks, labes etc. + ticksize = 30*scale + labelsize = 30*scale + legendsize = 15*scale + titlesize = 30*scale + + linewidth = 3*scale + axeswidth = 3*scale + markersize = 15*scale + majorticklength = 20*scale + minorticklength = 10*scale + + xpad = 4 if not xpad else xpad + ypad = 4 if not ypad else ypad + + + if mode == 'curves': + + # Set labels on x- and y-axes + ax.set_xlabel('Volume [{}]'.format(volumeunit), size=labelsize, labelpad=xpad) + + if not hide_ylabels: + ax.set_ylabel('Energy [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + else: + ax.set_ylabel('') + + # Set tick parameters + ax.tick_params(axis='both', direction='in', which='major', length=majorticklength, width=axeswidth, right=True, top=True, labelsize=ticksize) + ax.tick_params(axis='both', direction='in', which='minor', length=minorticklength, width=axeswidth, right=True, top=True, labelsize=ticksize) + + ax.tick_params(axis='x', pad=xpad) + ax.tick_params(axis='y', pad=ypad) + + if hide_ylabels: + ax.tick_params(labelleft=False) + + plt.xticks(fontsize=ticksize) + plt.yticks(fontsize=ticksize) + + + ax.xaxis.set_major_locator(MultipleLocator(10)) + ax.xaxis.set_minor_locator(MultipleLocator(5)) + + ax.yaxis.set_major_locator(MultipleLocator(.1)) + ax.yaxis.set_minor_locator(MultipleLocator(.05)) + + + ax.get_legend().remove() + if legend_content: + patches = [] + labels = legend_content[0] + colours = legend_content[1] + markers = legend_content[2] + + entries = [] + + for ind, label in enumerate(legend_content[0]): + entries.append(mlines.Line2D([], [], color=colours[ind], marker=markers[ind], linestyle='None', + markersize=markersize, label=labels[ind])) + + #patches.append(mpatches.Patch(color=colours[ind], label=labels[ind])) + + + fig.legend(handles=entries, loc='upper center', bbox_to_anchor=(1.10, 0.90), fontsize=legendsize, frameon=False) + + if pad_bottom is not None: + bigax = fig.add_subplot(111) + bigax.set_facecolor([1,1,1,0]) + bigax.spines['top'].set_visible(False) + bigax.spines['bottom'].set_visible(True) + bigax.spines['left'].set_visible(False) + bigax.spines['right'].set_visible(False) + bigax.tick_params(labelcolor='w', color='w', direction='in', top=False, bottom=True, left=False, right=False, labelleft=False, pad=pad_bottom) + + if mode == 'bars': + + + ax.tick_params(axis='both', direction='in', which='major', length=majorticklength, width=axeswidth, right=True, top=True) + ax.tick_params(axis='both', direction='in', which='minor', length=minorticklength, width=axeswidth, right=True, top=True) + + if not hide_ylabels: + ax.set_ylabel('Energy [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + + ax.yaxis.set_major_locator(MultipleLocator(.1)) + ax.yaxis.set_minor_locator(MultipleLocator(.05)) + + ax.tick_params(axis='x', pad=xpad) + ax.tick_params(axis='y', pad=ypad) + + plt.xticks(fontsize=ticksize) + plt.yticks(fontsize=ticksize) + + if pad_bottom is not None: + bigax = fig.add_subplot(111) + bigax.set_facecolor([1,1,1,0]) + bigax.spines['top'].set_visible(False) + bigax.spines['bottom'].set_visible(True) + bigax.spines['left'].set_visible(False) + bigax.spines['right'].set_visible(False) + bigax.tick_params(labelcolor='w', color='w', direction='in', top=False, bottom=True, left=False, right=False, labelleft=False, pad=pad_bottom) + + if mode == 'both': + + # Set labels on x- and y-axes + ax[0].set_xlabel('Volume [{}]'.format(volumeunit), size=labelsize, labelpad=xpad) + ax[0].set_ylabel('Energy [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + + # Set tick parameters + ax[0].tick_params(axis='both', direction='in', which='major', length=majorticklength, width=axeswidth, right=True, left=True, top=True, labelsize=ticksize) + ax[0].tick_params(axis='both', direction='in', which='minor', length=minorticklength, width=axeswidth, right=True, left=True, top=True, labelsize=ticksize) + + ax[0].tick_params(axis='x', pad=xpad) + ax[0].tick_params(axis='y', pad=ypad) + + ax[0].xaxis.set_major_locator(MultipleLocator(10)) + ax[0].xaxis.set_minor_locator(MultipleLocator(5)) + + ax[0].yaxis.set_major_locator(MultipleLocator(.1)) + ax[0].yaxis.set_minor_locator(MultipleLocator(.05)) + + plt.xticks(fontsize=ticksize) + plt.yticks(fontsize=ticksize) + + + ax[1].yaxis.set_major_locator(MultipleLocator(.2)) + ax[1].yaxis.set_minor_locator(MultipleLocator(.1)) + ax[1].yaxis.set_label_position('right') + ax[1].yaxis.tick_right() + ax[1].set_ylabel('Energy [{}]'.format(energyunit), size=labelsize, ypad=ypad) + ax[1].tick_params(axis='both', direction='in', which='major', length=majorticklength, width=axeswidth, left=True, right=True, top=True) + ax[1].tick_params(axis='both', direction='in', which='minor', length=minorticklength, width=axeswidth, left=True, right=True, top=True) + + ax[1].tick_params(axis='x', pad=xpad) + ax[1].tick_params(axis='y', pad=ypad) + + + plt.xticks(fontsize=ticksize) + plt.yticks(fontsize=ticksize) + + return fig, ax + + + +def parabola(V, a, b, c): + """parabola polynomial function + + this function is used to fit the data to get good guesses for + the equation of state fits + + a 4th order polynomial fit to get good guesses for + was not a good idea because for noisy data the fit is too wiggly + 2nd order seems to be sufficient, and guarantees a single minimum""" + + + E = (a * V**2) + (b * V) + c + + return E + + +def murnaghan(V, E0, V0, B0, BP): + 'From PRB 28,5480 (1983' + + E = E0 + ((B0 * V) / BP) * (((V0 / V)**BP) / (BP - 1) + 1) - ((V0 * B0) / (BP - 1)) + return E + + +def birch(V, E0, V0, B0, BP): + """ + From Intermetallic compounds: Principles and Practice, Vol. I: Principles + Chapter 9 pages 195-210 by M. Mehl. B. Klein, D. Papaconstantopoulos + paper downloaded from Web + + case where n=0 + """ + + E = (E0 + + 9 / 8 * B0 * V0 * ((V0 / V)**(2 / 3) - 1)**2 + + 9 / 16 * B0 * V0 * (BP - 4) * ((V0 / V)**(2 / 3) - 1)**3) + return E + + +def birchmurnaghan(V, E0, V0, B0, BP): + """ + BirchMurnaghan equation from PRB 70, 224107 + Eq. (3) in the paper. Note that there's a typo in the paper and it uses + inversed expression for eta. + """ + + eta = (V0 / V)**(1 / 3) + + E = E0 + 9 * B0 * V0 / 16 * (eta**2 - 1)**2 * (6 + BP * (eta**2 - 1) - 4 * eta**2) + + return E + + +def vinet(V, E0, V0, B0, BP): + 'Vinet equation from PRB 70, 224107' + + eta = (V / V0)**(1 / 3) + + E = (E0 + 2 * B0 * V0 / (BP - 1)**2 * + (2 - (5 + 3 * BP * (eta - 1) - 3 * eta) * + np.exp(-3 * (BP - 1) * (eta - 1) / 2))) + + return E + +def pouriertarantola(V, E0, V0, B0, BP): + 'Pourier-Tarantola equation from PRB 70, 224107' + + eta = (V / V0)**(1 / 3) + squiggle = -3 * np.log(eta) + + E = E0 + B0 * V0 * squiggle**2 / 6 * (3 + squiggle * (BP - 2)) + return E + + + +def get_initial_guesses(volume, energy): + + p = np.polyfit(volume, energy, deg=2) + + a, b, c = p[0], p[1], p[2] + + # Estimated from dE/dV = 2aV0 + b => V0 = -b / 2a + v0 = -b / (2*a) + + # Estimated by evaluating a parabola with a, b and c values at V = V0 + e0 = parabola(v0, a, b, c) + + # Estimated form B0 ~ V0 * d^2E / dV^2. d^2E / dV^2 = 2a. + b0 = 2 * a * v0 + + # Just a reasonable starting value + bp = 4 + + + return [e0, v0, b0, bp] + + + +def fit_eos_curve(volume, energy, p0, eos): + + eos_dict = {'murnaghan': murnaghan, 'birch': birch, 'birchmurnaghan': birchmurnaghan, 'vinet': vinet, 'pouriertarantola': pouriertarantola} + + func = eos_dict[eos] + + popt, pcov = curve_fit(func, volume, energy, p0) + + E0, V0, B0, BP = popt[0], popt[1], popt[2], popt[3] + + return [E0, V0, B0, BP] + + + + +def get_plotdata(volume, energy, equilibrium_values, eos): + + eos_dict = {'murnaghan': murnaghan, 'birch': birch, 'birchmurnaghan': birchmurnaghan, 'vinet': vinet, 'pouriertarantola': pouriertarantola} + + V = np.linspace(volume.min(), volume.max(), 100) + + E0, V0, B0, BP = equilibrium_values[0], equilibrium_values[1], equilibrium_values[2], equilibrium_values[3] + + print(E0, V0, B0, BP) + + func = eos_dict[eos] + + print(func) + + E = func(V, E0, V0, B0, BP) + + return E, V + + +def get_atoms(poscar): + + with open(poscar, 'r') as poscar: + lines = poscar.readlines() + + atoms = lines[5].split() + atom_num = lines[6].split() + + + atom_num = [int(num) for num in atom_num] + + atoms_dict = {} + + for ind, atom in enumerate(atoms): + atoms_dict[atom] = atom_num[ind] + + return atoms, atom_num, atoms_dict + + + +def get_equilibrium_data(path, atoms_per_formula_unit, eos=None): + + + if not eos: + eos = 'murnaghan' + + + dirs = [os.path.join(path, dir) for dir in os.listdir(path)] + + + + data = [] + + for dir in dirs: + atoms, atom_num, atoms_dict = get_atoms(os.path.join(dir, 'POSCAR')) + scaling_factor = sum(atom_num) / atoms_per_formula_unit + + label = dir.split('/')[-1] + + dft_df = pd.read_csv(os.path.join(dir, 'energ.dat'), header=None, delim_whitespace=True) + dft_df.columns = ['Volume', 'Energy'] + + volume = dft_df["Volume"].to_numpy() / scaling_factor + energy = dft_df["Energy"].to_numpy() / scaling_factor + + p0 = get_initial_guesses(volume, energy) + + equilibrium_constants = fit_eos_curve(volume, energy, p0, eos) + e0, v0, b0, bp = equilibrium_constants[0], equilibrium_constants[1], equilibrium_constants[2], equilibrium_constants[3] + + data.append([label, e0, v0, b0/kJ*1e24, bp]) + + + df = pd.DataFrame(data) + df.columns = ['Label', 'E0', 'V0', 'B0', 'Bp'] + df.sort_values(by='E0', ascending=True, inplace=True) + df.reset_index(inplace=True) + + E_min = df['E0'].min() + + df['dE'] = df['E0'] - E_min + + df = df[['Label', 'E0', 'dE', 'V0', 'B0', 'Bp']] + + + return df + + + + From 02367581ce895e2388e17d2ecb7c77c08f718309 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 20 Sep 2022 20:23:22 +0200 Subject: [PATCH 286/355] Initial add of phonon scripts --- nafuma/dft/__init__.py | 2 +- nafuma/dft/phonons.py | 1702 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1703 insertions(+), 1 deletion(-) create mode 100644 nafuma/dft/phonons.py diff --git a/nafuma/dft/__init__.py b/nafuma/dft/__init__.py index db87b5d..7aa3743 100644 --- a/nafuma/dft/__init__.py +++ b/nafuma/dft/__init__.py @@ -1 +1 @@ -from . import electrons, io, structure \ No newline at end of file +from . import electrons, io, structure, phonons \ No newline at end of file diff --git a/nafuma/dft/phonons.py b/nafuma/dft/phonons.py new file mode 100644 index 0000000..c62b560 --- /dev/null +++ b/nafuma/dft/phonons.py @@ -0,0 +1,1702 @@ +import re +from this import d +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd + +import subprocess +import os +import shutil + +from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) +from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition, + mark_inset) +import importlib +import matplotlib.patches as mpatches +from matplotlib.lines import Line2D +from cycler import cycler +import itertools + + +def get_atoms(path='.'): + + poscar = os.path.join(path, 'POSCAR') + + with open(poscar, 'r') as poscar: + lines = poscar.readlines() + + atoms = lines[5].split() + atom_num = lines[6].split() + + + atom_num = [int(num) for num in atom_num] + + return atoms, atom_num + +def get_dimensions(path='.'): + + poscar = os.path.join(path, 'POSCAR') + sposcar = os.path.join(path, 'SPOSCAR') + + + with open(poscar, 'r') as poscar: + + lines_pos = poscar.readlines() + + + with open(sposcar, 'r') as sposcar: + + lines_spos = sposcar.readlines() + + + + a_p, b_p, c_p = lines_pos[2].split(), lines_pos[3].split(), lines_pos[4].split() + a_s, b_s, c_s = lines_spos[2].split(), lines_spos[3].split(), lines_spos[4].split() + + lattice_params_poscar, lattice_params_sposcar = [a_p, b_p, c_p], [a_s, b_s, c_s] + + + + poscar_new = [] + sposcar_new = [] + + for lp_p, lp_s in zip(lattice_params_poscar, lattice_params_sposcar): + lp_p = np.sqrt(float(lp_p[0])**2 + float(lp_p[1])**2 + float(lp_p[2])**2) + lp_s = np.sqrt(float(lp_s[0])**2 + float(lp_s[1])**2 + float(lp_s[2])**2) + + poscar_new.append(lp_p) + sposcar_new.append(lp_s) + + + dim = [int(lp_s/lp_p) for lp_s, lp_p in zip(sposcar_new, poscar_new)] + + return dim + + +def read_band(band_dir): + ''' Reads a band file as written by the function write_phonon_bands() into a pandas DataFrame and returns this. Contains two columns: k-points (the "distance" output in the band.yaml-file by phonopy) and frequencies. + + Input: + band_dir: the path to the band-file. + + Output: + band: pandas DataFrame containing frequencies of the band along the k-point path specified in the phonopy calculation''' + + + # Read the band into a pandas DataFrame + band = pd.read_csv(band_dir, delim_whitespace=True, header=None, names=['kpt', 'frequency']) + + return band + + +def read_kpoints(kpoints_dir): + ''' Reads a VASP KPOINTS-file in line mode. Returns two lists: special_points_coords, containing the coordinates of the special points in k-space and special_points_labels, the names of these special points. + Requires a KPOINTS-file that is in line mode with special points indicated with a "!". + + Input: + kpoints_dir: the path to the KPOINTS-file + + Output: + special_points_coords: List of 3D coordinates of the k-space special points + special_points_labels: List of names of the k-space special points''' + + + # Open the KPOINTS-file and read it line by line, appending each line with a "!" to the special_points list. + special_points = [] + + with open(kpoints_dir) as kpoints: + lines = kpoints.readlines() + + for line in lines: + if '!' in line: + special_points.append(line) + + + # Go through the special points to separate them into the coordinate and the label for each special point into special_points_coords and special_points_labels respectively. + special_points_coords = [] + special_points_labels = [] + + for special_point in special_points: + if len(special_point.split()) == 5: + special_points_coords.append(special_point.split()[0:3]) + special_points_labels.append(special_point.split()[-1]) + + + return special_points_coords, special_points_labels + + + + + +def get_kpoints_ticks(band): + ''' Finds the coordinates for the special points in the 1D-projection given by phonopy (the parameter 'distance' in the band.yaml file). This is to determine the placement of labels and vertical lines in the bandstructure plot. + + Input: + band: the path to a band_XX.dat file. Should not matter which one is passed here. + + Output: + kpts_ticks: A list of coordinates corresponding to the special points.''' + + band = np.genfromtxt(band) + + kpts_ticks = [] + + # Append the first point + kpts_ticks.append(0.) + + # Go through all data points - where the x-value repeats, a k-point tick is appended to the list + for j in np.arange(np.shape(band)[0]-1): + if band[j,0]==band[j+1,0]: + kpts_ticks.append(band[j,0]) + + # Append the last point + kpts_ticks.append(max(band[:,0])) + + + return kpts_ticks + +def get_kpoints_labels(special_points_labels): + ''' Takes the raw special point labels from read_kpoints() and writes them in a way to be used in the bandstructure plots. + Where there is a discontinuity in the path, the label is separated with a |. + + Input: + special_points_labels: A list of special points as directly read from the KPOINTS-file by read_kpoints() + + Ouput: + labels: A list of labels suitable to pass as x-ticks during plotting of the bandstructure plots.''' + + + # Loop through the raw special points labels list following a set of rules, to extract the labels suitable for plotting + labels = [] + + for ind, label in enumerate(special_points_labels): + + # Add the first label as this will be a separate special point + if ind == 0: + label = '${}$'.format(label) if (label[0] == '\\') else label + labels.append(label) + + # Add the last label, as this will also be a separate special point (or will it? Must change this if that is not always the case) + elif ind == len(special_points_labels)-1: + label = '${}$'.format(label) if (label[0] == '\\') else label + labels.append(label) + + # Skip every second entry, as they will repeat due to the way the KPOINTS-file is constructed + elif ind%2 != 0: + continue + + # Add label if it's continuous (i.e. if the current and previous points are the same), add "previous|current" if discontinuous (i.e. if they are not the same) + else: + if label == special_points_labels[ind-1]: + label = '${}$'.format(label) if (label[0] == '\\') else label # If the special point has a greek letter, such as the gamma point, makes sure that the label is enclosed in $ to be rendered correctly. + labels.append(label) + + else: + label = '${}$'.format(label) if (label[0] == '\\') else label # If the special point has a greek letter, such as the gamma point, makes sure that the label is enclosed in $ to be rendered correctly. + previous_label = special_points_labels[ind-1] + previous_label = '${}$'.format(previous_label) if previous_label[0] == '\\' else previous_label # If the special point has a greek letter, such as the gamma point, makes sure that the label is enclosed in $ to be rendered correctly. + + labels.append("{}|{}".format(previous_label, label)) + + + return labels + + +def read_phonon_dos(dos_path): + ''' Reads the phonon density of states from a total_dos.dat file as written by phonopy. This file will be generated by the function calculate_phonon_dos() as well as this calls phonopy to calculate the density of states. + + Input: + dos_path: the path to the total_dos.dat file. Must include the filename + + Output: + df: pandas DataFrame containing the contents of the total_dos.dat file. Two columns, "Frequency" and "DOS". ''' + + df = pd.read_csv(dos_path, header=None, skiprows=1, delim_whitespace=True) + df.columns = ['Frequency', 'DOS'] + + return df + + +def read_phonon_pdos(path, normalise=False, poscar=None): + ''' Reads the phonon density of states from a total_dos.dat file as written by phonopy. This file will be generated by the function calculate_phonon_dos() as well as this calls phonopy to calculate the density of states. + + Input: + dos_path: the path to the total_dos.dat file. Must include the filename + + Output: + df: pandas DataFrame containing the contents of the total_dos.dat file. Two columns, "Frequency" and "DOS". ''' + + df = pd.read_csv(path, index_col=0) + + if normalise and poscar: + atoms, atom_num = get_atoms(poscar) + + + for atom, num in zip(atoms, atom_num): + df[atom] = df[atom] / num + + + return df + + +def write_phonopy_band_path(special_points_coords): + ''' Writes the band path used by phonopy to calculate the bandstructure from the raw information as extracted by read_kpoints(). + + Input: + special_points_coords: list of coordinates for the special points as read by read_kpoints(), that reads a VASP KPOINTS.bands file. + + Output: + phonopy_band_path: ''' + + coords = [] + + for ind, coord in enumerate(special_points_coords): + + # Add the first label + if ind == 0: + coord = "{} {} {} ".format(coord[0], coord[1], coord[2]) + coords.append(coord) + + # Add the last label + elif ind == len(special_points_coords)-1: + coord = "{} {} {}".format(coord[0], coord[1], coord[2]) + coords.append(coord) + + # Skip every second entry + elif ind%2 != 0: + continue + + # Add label if it's continuous, add "previous|current" if discontinuous + else: + if coord == special_points_coords[ind-1]: + coord = "{} {} {} ".format(coord[0], coord[1], coord[2]) + coords.append(coord) + + else: + first_coord = "{} {} {}".format(special_points_coords[ind-1][0], special_points_coords[ind-1][1], special_points_coords[ind-1][2]) + second_coord = "{} {} {} ".format(coord[0], coord[1], coord[2]) + coords.append("{}, {} ".format(first_coord, second_coord)) + + + phonopy_band_path = '' + + for coord in coords: + phonopy_band_path = phonopy_band_path + coord + + return phonopy_band_path + + +def write_mesh_conf(atoms, dim, mesh, dos_range=None, pdos=False, atom_num=None, tmax=None): + + atom_str = 'ATOM_NAME = ' + for atom in atoms: + atom_str += atom + " " + + dim_str = 'DIM = ' + for d in dim: + dim_str += str(d) + " " + + mesh_str = 'MP = ' + for m in mesh: + mesh_str += str(m) + " " + + dos_str = 'DOS_RANGE = ' + for d in dos_range: + dos_str += str(d) + " " + + if tmax: + tmax_str = f'TMAX = {tmax}' + + if pdos: + pdos_str = 'PDOS =' + + atoms_sum = 0 + for ind, atom in enumerate(atoms): + for i in range(1,atom_num[ind]+1): + pdos_str += " {}".format(i+atoms_sum) + + # Add comma after numbers unless it's the last entry + if ind != len(atom_num)-1: + pdos_str += ',' + + atoms_sum = atoms_sum + atom_num[ind] + + + with open('mesh.conf', 'w') as conf: + + conf.write(atom_str + '\n' + dim_str + '\n' + mesh_str) + + if tmax: + conf.write('\n' + tmax_str) + + if dos_range: + conf.write('\n' + dos_str) + + if pdos: + conf.write('\n' + pdos_str) + + conf.write('\n' + "WRITE_MESH = .FALSE.") + + +def write_band_conf(atoms, dim, mesh, band, band_points=None): + + atom_str = 'ATOM_NAME = ' + for atom in atoms: + atom_str += atom + " " + + dim_str = 'DIM = ' + for d in dim: + dim_str += str(d) + " " + + mesh_str = 'MP = ' + for m in mesh: + mesh_str += str(m) + " " + + band_str = 'BAND = ' + band + + if band_points: + band_points_str = "BAND_POINTS = " + band_points + + + with open('band.conf', 'w') as conf: + + if not band_points: + conf.write(atom_str + '\n' + dim_str + '\n' + mesh_str + '\n' + band_str) + + else: + conf.write(atom_str + '\n' + dim_str + '\n' + mesh_str + '\n' + band_str + '\n' + band_points_str) + + + +def calculate_phonon_dos(path, atoms, dim, mesh, dos_range=None): + + cwd = os.getcwd() + os.chdir(path) + + write_mesh_conf(atoms, dim, mesh, dos_range=dos_range) + + subprocess.call('phonopy -ps mesh.conf >> phonopy_output.dat', shell=True) + + + # Make folder and move output in there + os.mkdir('total_dos') + shutil.move('total_dos.pdf', 'total_dos/total_dos.pdf') + shutil.move('total_dos.dat', 'total_dos/total_dos.dat') + shutil.move('mesh.conf', 'total_dos/mesh.conf') + shutil.move('phonopy.yaml', 'total_dos/phonopy.yaml') + shutil.move('phonopy_output.dat', 'total_dos/phonopy_output.dat') + + os.chdir(cwd) + + + + + +def calculate_phonon_pdos(path, atoms, dim, mesh, dos_range=None, atom_num=None, order=None): + ''' Calculate the projected phonon DOS. Calls function to write mesh.conf file, and then cleans up the output by summing all the individual contributions per atom to the same species.''' + + cwd = os.getcwd() + os.chdir(path) + + write_mesh_conf(atoms, dim, mesh, dos_range=dos_range, pdos=True, atom_num=atom_num) + + subprocess.call('phonopy -ps mesh.conf >> phonopy_output.dat', shell=True) + + + df = pd.read_csv('projected_dos.dat', delim_whitespace=True, skiprows=1, header=None, dtype=float) + + + # Loop over the columns and add according to "atoms" and "atom_num" lists + + atoms_sum = 0 + for atom, num in zip(atoms, atom_num): + df[atom] = df[1+atoms_sum] + + for i in range(2+atoms_sum, atoms_sum+num+1): + df[atom] = df[atom] + df[i] + + + + atoms_sum += num + + + + # Remove all other columns, and rename the first column to "Frequency" + df.drop(df.iloc[:, 1:atoms_sum+1], inplace = True, axis = 1) + df.rename(columns = {0: "Frequency"}, inplace=True) + + + # If a list is passed to order, this will change the order of the atoms: + + if order: + df_temp = pd.DataFrame() + df_temp["Frequency"] = df["Frequency"] + + for atom in order: + df_temp[atom] = df[atom] + + df = df_temp + + + # Save the cleaned up DataFrame to file. + df.to_csv('projected_dos_clean.dat') + + + + # Make folder and move output in there + os.mkdir('projected_dos') + shutil.move('partial_dos.pdf', 'projected_dos/partial_dos.pdf') + shutil.move('projected_dos.dat', 'projected_dos/projected_dos.dat') + shutil.move('projected_dos_clean.dat', 'projected_dos/projected_dos_clean.dat') + shutil.move('mesh.conf', 'projected_dos/mesh.conf') + shutil.move('phonopy.yaml', 'projected_dos/phonopy.yaml') + shutil.move('phonopy_output.dat', 'projected_dos/phonopy_output.dat') + + + + os.chdir(cwd) + + + +def calculate_thermal_properties(path, atoms, dim, mesh, dos_range=None, tmax=None): + + cwd = os.getcwd() + os.chdir(path) + + write_mesh_conf(atoms, dim, mesh, dos_range=dos_range, tmax=tmax) + + subprocess.call('phonopy -t mesh.conf >> phonopy_output.dat', shell=True) + + with open('phonopy_output.dat', 'r') as f: + lines = f.readlines() + + + data = [] + for ind, line in enumerate(lines): + + if line.split(): + if "#" in line.split()[0]: + j = 1 + while lines[ind+j].split(): + data.append(lines[ind+j].split()) + j += 1 + + + + df = pd.DataFrame(data) + df.columns = ['T', 'F', 'S', 'C_v', 'E'] + + df.to_csv('thermal_properties.dat') + + + + + #Make folder and move output in there + os.mkdir('thermal_properties') + shutil.move('thermal_properties.yaml', 'thermal_properties/thermal_properties.yaml') + shutil.move('thermal_properties.dat', 'thermal_properties/thermal_properties.dat') + shutil.move('mesh.conf', 'thermal_properties/mesh.conf') + shutil.move('phonopy.yaml', 'thermal_properties/phonopy.yaml') + shutil.move('phonopy_output.dat', 'thermal_properties/phonopy_output.dat') + + + os.chdir(cwd) + +def calculate_phonon_bandstructure(path, atoms, dim, mesh, kpoints='KPOINTS.bands', band_points=None): + + cwd = os.getcwd() + os.chdir(path) + + + kpoints_coords, kpoints_labels = read_kpoints(kpoints) + + band = write_phonopy_band_path(kpoints_coords) + + write_band_conf(atoms, dim, mesh, band, band_points=band_points) + + subprocess.call('phonopy band.conf >> phonopy_output.dat', shell=True) + + write_phonon_bands() + + + os.mkdir('dispersion_relation') + + shutil.move('band.conf', 'dispersion_relation/band.conf') + shutil.move('band.yaml', 'dispersion_relation/band.yaml') + shutil.move('bands', 'dispersion_relation/bands') + shutil.move('mesh.yaml', 'dispersion_relation/mesh.yaml') + shutil.move('phonopy.yaml', 'dispersion_relation/phonopy.yaml') + shutil.move('phonopy_output.dat', 'dispersion_relation/phonopy_output.dat') + + os.chdir(cwd) + +def write_phonon_bands(band='band.yaml'): + + with open(band, 'r') as f: + lines = f.readlines() + + + kpoints = [] + frequencies = [] + + for line in lines: + if 'distance' in line: + kpoints.append(line.split()[-1]) + + if 'frequency' in line: + frequencies.append(line.split()[-1]) + + + number_of_kpoints = len(kpoints) + number_of_bands = len(frequencies) / number_of_kpoints + + if not os.path.isdir('bands'): + os.mkdir('bands') + + os.chdir('bands') + + for i in range(int(number_of_bands)): + + with open('band_{}.dat'.format(i+1), 'w') as b: + for ind, kpoint in enumerate(kpoints): + if ind == len(kpoints)-1: + b.write("{} {}".format(kpoint, frequencies[ind*int(number_of_bands)+i])) + else: + b.write("{} {}\n".format(kpoint, frequencies[ind*int(number_of_bands)+i])) + + + os.chdir('../') + + +def plot_phonon_dos(dos_path='total_dos.dat', options={}): + + + required_options = ['xlim', 'ylim', 'flip_xy', 'colours', 'palettes', 'rc_params', 'format_params'] + + + default_options = { + 'xlim': None, # x-limits + 'ylim': None, # y-limits + 'flip_xy': False, # Whether to flip what is plotted on the x- and y-axes respectively. Default is False and plots frequency along x-axis and density of states along y-axis. + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'format_params': {}, + 'rc_params': {} + } + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + fig, ax = prepare_plot(options=options) + + dos = read_phonon_dos(dos_path=dos_path) + + if not options['xlim']: + options['xlim'] = [dos["Frequency"].min(), dos["Frequency"].max()] + + if not options['ylim']: + options['ylim'] = [dos["DOS"].min(), dos["DOS"].max()*1.1] + + + if not options['colours']: + colours = generate_colours(palette=options['palette']) + else: + colours = itertools.cycle(options['colours']) + + if options['flip_xy']: + dos.plot(x='DOS', y='Frequency', ax=ax, color=colours[0]) + + else: + dos.plot(x='Frequency', y='DOS', ax=ax, color=colours[0]) + + + options['plot_kind'] = 'DOS' + fig, ax = prettify_dos_plot(fig=fig, ax=ax, options=options) + + ax.get_legend().remove() + + + return fig, ax + + + + +def prepare_plot(options={}): + + rc_params = options['rc_params'] + format_params = options['format_params'] + + required_options = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi'] + + default_options = { + 'single_column_width': 8.3, + 'double_column_width': 17.1, + 'column_type': 'single', + 'width_ratio': '1:1', + 'aspect_ratio': '1:1', + 'compress_width': 1, + 'compress_height': 1, + 'upscaling_factor': 1.0, + 'dpi': 600, + } + + options = update_options(format_params, required_options, default_options) + + + # Reset run commands + plt.rcdefaults() + + # Update run commands if any is passed (will pass an empty dictionary if not passed) + update_rc_params(rc_params) + + width = determine_width(options) + height = determine_height(options, width) + width, height = scale_figure(options=options, width=width, height=height) + + fig, ax = plt.subplots(figsize=(width, height), dpi=options['dpi']) + + return fig, ax + + +def plot_phonon_pdos(path='projected_dos_clean.dat', options={}): + + + required_options = ['xlim', 'ylim', 'flip_xy', 'colours', 'palettes', 'normalise', 'poscar', 'atoms', 'rc_params', 'format_params'] + + + default_options = { + 'xlim': None, # x-limits + 'ylim': None, # y-limits + 'flip_xy': False, # Whether to flip what is plotted on the x- and y-axes respectively. Default is False and plots frequency along x-axis and density of states along y-axis. + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'normalise': False, + 'poscar': None, + 'atoms': [], + 'format_params': {}, + 'rc_params': {} + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + dos = read_phonon_pdos(path=path, normalise=options['normalise'], poscar=options['poscar']) + + fig, ax = prepare_plot(options=options) + + if not options['xlim']: + options['xlim'] = [dos["Frequency"].min(), dos["Frequency"].max()] + + if not options['ylim'] and options['atoms']: + ymin = 0 + ymax = 0 + + + for atom in options['atoms']: + if dos[atom].min() < ymin: + ymin = dos[atom].min() + + if dos[atom].max() > ymax: + ymax = dos[atom].max() + + options['ylim'] = [ymin, ymax*1.1] + + + if not options['colours']: + colours = generate_colours(palette=options['palette']) + else: + colours = itertools.cycle(options['colours']) + + for ind, atom in enumerate(options['atoms']): + + if options['flip_xy']: + dos.plot(x=atom, y='Frequency', ax=ax, color=next(colours)) + + else: + dos.plot(x='Frequency', y=atom, ax=ax, color=next(colours)) + + + options['plot_kind'] = 'PDOS' + prettify_dos_plot(fig=fig, ax=ax, options=options) + + return fig, ax + + + + +def plot_phonon_bandstructure(band_folder='bands', kpoints='KPOINTS.bands', options={}, title=None, xlim=None, ylim=None, pad_bottom=None, scale=1, square=True, width=None, height=None, dpi=None, rotation=None, xpad=None, ypad=None): + + + # Get the special points labels + kpoint_coords, kpoint_labels = read_kpoints(kpoints) + kpoint_labels = get_kpoints_labels(kpoint_labels) + + + + # Get current folder and change into the folder containing bands + cwd = os.getcwd() + os.chdir(band_folder) + + band_paths = [band for band in os.listdir() if os.path.isfile(band) and band[0:4] == 'band'] + + # Get the location of the special points along the x-axis + kpoint_ticks = get_kpoints_ticks(band_paths[0]) + + bands = [] + for band_path in band_paths: + bands.append(read_band(band_path)) + + + + fig, ax = prepare_plot(options=options) + + mod = importlib.import_module("palettable.colorbrewer.%s" % 'qualitative') + colour = getattr(mod, 'Dark2_3').mpl_colors[0] + + kpt_min = None + kpt_max = None + freq_min = None + freq_max = None + + for band in bands: + if kpt_min == None or band["kpt"].min() < kpt_min: + kpt_min = band["kpt"].min() + if kpt_max == None or band["kpt"].max() > kpt_max: + kpt_max = band["kpt"].max() + + if freq_min == None or band["frequency"].min() < freq_min: + freq_min = band["frequency"].min() + if freq_max == None or band["frequency"].max() > freq_max: + freq_max = band["frequency"].max() + + band.plot('kpt', 'frequency', ax=ax, color=colour) + + + if not xlim: + xlim = [kpt_min, kpt_max] + if not ylim: + ylim = [freq_min-freq_max*0.1, freq_max+freq_max*0.1] + + ax.get_legend().remove() + + prettify_plot(fig=fig, ax=ax, special_points_labels=kpoint_labels, special_points_coords=kpoint_ticks, xlim=xlim, ylim=ylim, title=title, pad_bottom=pad_bottom, scale=scale, rotation=rotation, xpad=xpad, ypad=ypad) + + os.chdir(cwd) + + +def prepare_plot_old(width=None, height=None, square=True, dpi=None, colour_cycle=('qualitative', 'Dark2_8'), temperatureunit='K', energyunit='eV f.u.$^{-1}$', scale=1): + + linewidth = 3*scale + axeswidth = 3*scale + + plt.rc('lines', linewidth=linewidth) + plt.rc('axes', linewidth=axeswidth) + + if square: + if not width: + width = 20 + + if not height: + height = width + + + fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(width, height), facecolor='w', dpi=dpi) + + return fig, ax + + +def prettify_plot(fig, ax, frequencyunit='THz', special_points_coords=None, special_points_labels=None, xlim=None, ylim=None, title=None, pad_bottom=None, scale=1, rotation=None, xpad=None, ypad=None): + + + # Set sizes of ticks, labes etc. + ticksize = 30*scale + labelsize = 30*scale + legendsize = 30*scale + titlesize = 30*scale + + linewidth = 3*scale + axeswidth = 3*scale + majorticklength = 20*scale + minorticklength = 10*scale + + + # Set labels on x- and y-axes + if ypad: + ax.set_ylabel('Frequency [{}]'.format(frequencyunit), size=labelsize, labelpad=ypad) + else: + ax.set_ylabel('Frequency [{}]'.format(frequencyunit), size=labelsize) + + + ax.set_xlabel('') + + + + + ax.tick_params(axis='y', direction='in', which='major', right=True, length=10, width=0.5) + ax.tick_params(axis='y', direction='in', which='minor', right=True, length=5, width=0.5) + + ax.tick_params(axis='x', direction='in', which='major', bottom=False) + + + + + ax.yaxis.set_major_locator(MultipleLocator(5)) + ax.yaxis.set_minor_locator(MultipleLocator(2.5)) + + + plt.xticks(fontsize=ticksize) + plt.yticks(fontsize=ticksize) + + + + # Set tick parameters + if special_points_coords: + for coord in special_points_coords: + plt.axvline(coord, color='black', linestyle='--', linewidth=0.5) + + + + plt.xticks(ticks=special_points_coords, labels=special_points_labels, rotation=rotation) + + + + if xlim: + plt.xlim(xlim) + + if ylim: + plt.ylim(ylim) + + + if title: + ax.set_title(title, size=40) + + if pad_bottom is not None: + bigax = fig.add_subplot(111) + bigax.set_facecolor([1,1,1,0]) + bigax.spines['top'].set_visible(False) + bigax.spines['bottom'].set_visible(True) + bigax.spines['left'].set_visible(False) + bigax.spines['right'].set_visible(False) + bigax.tick_params(labelcolor='w', color='w', direction='in', top=False, bottom=True, left=False, right=False, labelleft=False, pad=pad_bottom) + + if xpad: + ax.tick_params(axis='x', pad=xpad) + if ypad: + ax.tick_params(axis='y', pad=ypad) + + return fig, ax + + +def prettify_dos_plot(fig, ax, options, frequencyunit='THz', dosunit='a.u.', xlim=None, ylim=None, title=None, hide_ylabels=False, flip_xy=False, pad_bottom=None, scale=1, pdos=False, colours=None, atoms=None, xpad=None, ypad=None): + + + required_options = ['plot_kind', 'flip_xy', 'hide_x_labels', 'hide_y_labels', 'xlabel', 'ylabel', 'xunit', 'yunit', 'xlim', 'ylim', 'x_tick_locators', 'y_tick_locators', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', + 'colours', 'palettes', 'title', 'legend', 'legend_position', 'subplots_adjust', 'text'] + + default_options = { + 'plot_kind': 'DOS', # DOS or PDOS + 'flip_xy': False, + 'hide_x_labels': False, # Whether x labels should be hidden + 'hide_x_ticklabels': False, + 'hide_x_ticks': False, + 'hide_y_labels': False, # whether y labels should be hidden + 'hide_y_ticklabels': False, + 'hide_y_ticks': False, + 'xlabel': 'Frequency', + 'ylabel': 'DOS', + 'xunit': r'THz', # The unit of the x-values in the curve plot + 'yunit': r'a.u.', # The unit of the y-values in the curve and bar plots + 'xlim': None, + 'ylim': None, + 'x_tick_locators': [5, 2.5], # Major and minor tick locators + 'y_tick_locators': [10, 5], + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'title': None, + 'legend': True, + 'legend_position': ['upper center', (0.20, 0.90)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], + 'text': None + } + + + if 'plot_kind' in options.keys(): + if 'ylabel' not in options.keys(): + if options['plot_kind'] == 'DOS': + options['ylabel'] = 'Density of states' + elif options['plot_kind'] == 'PDOS': + options['ylabel'] = 'PDOS' + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + + if options['flip_xy']: + + # Switch all the x- and y-specific values + options = swap_values(dict=options, key1='xlim', key2='ylim') + options = swap_values(dict=options, key1='xunit', key2='yunit') + options = swap_values(dict=options, key1='xlabel', key2='ylabel') + options = swap_values(dict=options, key1='x_tick_locators', key2='y_tick_locators') + options = swap_values(dict=options, key1='hide_x_labels', key2='hide_y_labels') + + # Set labels on x- and y-axes + if not options['hide_y_labels']: + ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + else: + ax.set_ylabel('') + + + + if not options['hide_x_labels']: + ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]') + else: + ax.set_xlabel('') + + + # Hide x- and y- ticklabels + if options['hide_y_ticklabels']: + ax.tick_params(axis='y', direction='in', which='both', labelleft=False, labelright=False) + if options['hide_x_ticklabels']: + ax.tick_params(axis='x', direction='in', which='both', labelbottom=False, labeltop=False) + + + # Hide x- and y-ticks: + if options['hide_y_ticks']: + ax.tick_params(axis='y', direction='in', which='both', left=False, right=False) + if options['hide_x_ticks']: + ax.tick_params(axis='x', direction='in', which='both', bottom=False, top=False) + + + + # Set multiple locators + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + + ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) + ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + + + # Set title + if options['title']: + ax.set_title(options['title']) + + + # Generate colours + if not options['colours']: + colours = generate_colours(palette=options['palette']) + else: + colours = itertools.cycle(options['colours']) + + + + # Create legend + + if ax.get_legend(): + ax.get_legend().remove() + + + if options['legend']: + if options['plot_kind'] == 'PDOS' and options['atoms']: + + # Create legend + patches = [] + for atom in options['atoms']: + patches.append(mpatches.Patch(color=next(colours), label=atom)) + + fig.legend(handles=patches, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1], frameon=False) + + + + # Adjust where the axes start within the figure. Default value is 10% in from the left and bottom edges. Used to make room for the plot within the figure size (to avoid using bbox_inches='tight' in the savefig-command, as this screws with plot dimensions) + plt.subplots_adjust(left=options['subplots_adjust'][0], bottom=options['subplots_adjust'][1], right=options['subplots_adjust'][2], top=options['subplots_adjust'][3]) + + + # If limits for x- and y-axes is passed, sets these. + if options['xlim'] is not None: + ax.set_xlim(options['xlim']) + + if options['ylim'] is not None: + ax.set_ylim(options['ylim']) + + + # Add custom text + if options['text']: + plt.text(x=options['text'][1][0], y=options['text'][1][1], s=options['text'][0]) + + return fig, ax + + + + + +def read_thermal_properties(path, number_of_formula_units=None, convert=True): + + kJ = 6.2415064799632E+21 + Na = 6.0221415E+23 + + thermal_properties = pd.read_csv(path, skiprows=1, index_col=0) + thermal_properties.columns = ['T', 'F', 'S', 'Cv', 'E'] + + + if convert: + thermal_properties.F = thermal_properties.F / Na * kJ + thermal_properties.S = thermal_properties.S / Na * kJ + thermal_properties.Cv = thermal_properties.Cv / Na * kJ + thermal_properties.E = thermal_properties.E / Na * kJ + + if number_of_formula_units: + thermal_properties.F = thermal_properties.F / number_of_formula_units + thermal_properties.S = thermal_properties.S / number_of_formula_units + thermal_properties.Cv = thermal_properties.Cv / number_of_formula_units + thermal_properties.E = thermal_properties.E / number_of_formula_units + + + + return thermal_properties + + + +def plot_thermal_properties(path, number_of_formula_units=None, convert=True): + + thermal_properties = read_thermal_properties(path=path, number_of_formula_units=number_of_formula_units, convert=convert) + + thermal_properties.plot(x='T', y=['F', 'S', 'Cv', 'E']) + + + + + +def get_adjusted_energies(paths, equilibrium_energies, options={}): + + + required_options = ['plot_kind', 'reference', 'number_of_formula_units', 'xlim', 'ylim', 'flip_xy', 'colours', 'palettes', 'normalise', 'poscar', 'atoms', 'rc_params', 'format_params'] + + + default_options = { + 'plot_kind': 'absolute', + 'reference': 0, + 'number_of_formula_units': None, + 'xlim': None, # x-limits + 'ylim': None, # y-limits + 'flip_xy': False, # Whether to flip what is plotted on the x- and y-axes respectively. Default is False and plots frequency along x-axis and density of states along y-axis. + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'normalise': False, + 'poscar': None, + 'atoms': [], + 'format_params': {}, + 'rc_params': {} + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + dfs = [] + + if not options['number_of_formula_units']: + options['number_of_formula_units'] = [None for i in range(len(paths))] + + for ind, path in enumerate(paths): + df = read_thermal_properties(path, options['number_of_formula_units'][ind]) + dfs.append(df) + + + for ind, df in enumerate(dfs): + df["adjusted_energy"] = equilibrium_energies[ind] + df["F"] + + + if options['plot_kind'] == 'difference': + for ind, df in enumerate(dfs): + df["reference_energy"] = dfs[options['reference']]["adjusted_energy"] + df["difference_energy"] = df["adjusted_energy"] - df["reference_energy"] + + + if options['plot_kind'] == 'relative': + for ind, df in enumerate(dfs): + df["reference_energy"] = dfs[options['reference']]["adjusted_energy"].iloc[0] + df["relative_energy"] = df["adjusted_energy"] - df["reference_energy"] + + + return dfs + + + +def find_low_energy_structures_at_extremas(dfs): + + energy_low_T = -1 + low_T_ind = -1 + energy_high_T = -1 + high_T_ind = -1 + + for ind, df in enumerate(dfs): + if low_T_ind == -1: + low_T_ind = ind + energy_low_T = df['adjusted_energy'].loc[df['T'] == df['T'].min()].values[0] + + elif df['adjusted_energy'].loc[df['T'] == df['T'].min()].values[0] < energy_low_T: + low_T_ind = ind + energy_low_T = df['adjusted_energy'].loc[df['T'] == df['T'].min()].values[0] + + if high_T_ind == -1: + high_T_ind = ind + energy_high_T = df['adjusted_energy'].loc[df['T'] == df['T'].max()].values[0] + + elif df['adjusted_energy'].loc[df['T'] == df['T'].max()].values[0] < energy_high_T: + high_T_ind = ind + energy_high_T = df['adjusted_energy'].loc[df['T'] == df['T'].max()].values[0] + + + + return [low_T_ind, high_T_ind] + +def find_intersection(dfs, ind1, ind2): + + intersection = -1 + + for T in dfs[0]['T']: + + if dfs[ind2]['adjusted_energy'].loc[dfs[ind2]['T'] == T].values[0] < dfs[ind1]['adjusted_energy'].loc[dfs[ind1]['T'] == T].values[0]: + intersection = T + break + + + return intersection + + + + + +def plot_adjusted_energies(paths, equilibrium_energies, options={}): + + + ''' This function plots the adjusted total energies of a set of structures given a set of thermal properties calculated using phonopy. + + paths: List of paths (strings) to the .csv-files with thermal properties. + equilibrium_energies: List of equilibrium energies (floats) of pristine calculations + labels: List of labels (strings) to be shown in the plot + mode: Whether to plot as a difference plot ("difference_plot") or absolute units ("absolute"). Defaults to absolute + difference_reference: Index of which structure should serve as the reference. Defaults to 0. + number_of_formula_units: List of number of formula units per unit cell (int, float) to scale the data properly. Defaults to None, meaning to scaling. + width: Width of the plot. Defaults to None, meaning standard width is used. + width: Height of the plot. Defaults to None, meaning standard height is used. + dpi: Dots per inch. Defaults to None, meaning standard dpi is used. + colour_cycle: Tuple with type of colour scheme from the colorbrewer: http://jiffyclub.github.io/palettable/colorbrewer/ + temperatureunit: The unit to plot the temperature in. Only K implemented so far. + energyunit: The unit to plot the energy in. Only eV per f.u. impleneted so far. + inset: Whether or not there should be an inset. This is not very well implemented, and may cause issues. Defaults to False. + inset_lims: The x-limits of the inset. Defaults to None, meaning it will just try to figure it out itself. + ''' + + required_options = ['plot_kind', 'reference', 'number_of_formula_units', 'labels', 'xlim', 'ylim', 'colours', 'palettes', 'linestyles', 'rc_params', 'format_params', 'inset_xlim', 'inset_ylim', 'draw_intersection_main', 'draw_intersection_inset', 'intersection_indices', 'intersection_lw'] + + + default_options = { + 'plot_kind': 'absolute', + 'reference': 0, + 'number_of_formula_units': None, + 'labels': None, + 'xlim': None, # x-limits + 'ylim': None, # y-limits + 'inset_xlim': None, + 'inset_ylim': None, + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'linestyles': ['solid', 'dotted', 'dashed'], + 'format_params': {}, + 'rc_params': {}, + 'draw_intersection_main': False, + 'draw_intersection_inset': False, + 'intersection_indices': None, + 'intersection_lw': None, + } + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + energy_dfs = get_adjusted_energies(paths=paths, equilibrium_energies=equilibrium_energies, options=options) + + fig, ax = prepare_plot(options=options) + + + if not options['labels']: + options['labels'] = ['_' for i in range(len(paths))] + + + if not options['colours']: + colours = generate_colours(palettes=options['palettes']) + else: + colours = itertools.cycle(options['colours']) + + + linestyles = itertools.cycle(options['linestyles']) + + for df in energy_dfs: + if options['plot_kind'] == 'difference': + df.plot(x='T', y='difference_energy', ax=ax, ls=next(linestyles), c=next(colours)) + elif options['plot_kind'] == 'relative': + df.plot(x='T', y='relative_energy', ax=ax, ls=next(linestyles), c=next(colours)) + elif options['plot_kind'] == 'absolute': + df.plot(x='T', y='adjusted_energy', ax=ax, ls=next(linestyles), c=next(colours)) + + ax.set_xlim([int(df["T"].min()), int(df["T"].max())]) + + + + + + fig, ax = prettify_thermal_plot(fig=fig, ax=ax, options=options) + + + + if options['inset_xlim']: + inset_ax = prepare_inset_axes(ax, options) + + if not options['colours']: + colours = generate_colours(palettes=options['palettes']) + else: + colours = itertools.cycle(options['colours']) + + + linestyles = itertools.cycle(options['linestyles']) + + + for df in energy_dfs: + if options['plot_kind'] =='absolute': + y = 'adjusted_energy' + elif options['plot_kind'] == 'relative': + y = 'relative_energy' + elif options['plot_kind'] == 'difference': + y = 'difference_energy' + + df.loc[(df["T"] >= options['inset_xlim'][0]) & (df["T"] <= options['inset_xlim'][1])].plot(x='T', y=y, ax=inset_ax, ls=next(linestyles), c=next(colours)) + inset_ax.set_xlim([options['inset_xlim'][0], options['inset_xlim'][1]]) + + if options['inset_ylim']: + inset_ax.set_ylim([options['inset_ylim'][0], options['inset_ylim'][1]]) + + inset_ax.get_legend().remove() + inset_ax.set_xlabel('') + + + if options['draw_intersection_main'] or options['draw_intersection_inset']: + + if not options['intersection_indices']: + options['intersection_indices'] = find_low_energy_structures_at_extremas(energy_dfs) + + if not options['intersection_lw']: + options['intersection_lw'] = plt.rcParams['lines.linewidth'] + + intersection = find_intersection(energy_dfs, options['intersection_indices'][0], options['intersection_indices'][1]) + + if options['draw_intersection_main']: + ax.axvline(x=intersection, ls='dashed', c='black', lw=options['intersection_lw']) + if options['draw_intersection_inset']: + inset_ax.axvline(x=intersection, ls='dashed', c='black', lw=options['intersection_lw']) + + + return fig, ax + + + + + +def prepare_thermal_plot(width=None, height=None, dpi=None, colour_cycle=('qualitative', 'Dark2_8'), temperatureunit='K', energyunit='eV f.u.$^{-1}$', scale=1): + + linewidth = 3*scale + axeswidth = 3*scale + + plt.rc('lines', linewidth=linewidth) + plt.rc('axes', linewidth=axeswidth) + + if not width: + width = 20 + + if not height: + height = width + + + fig = plt.figure(figsize=(width, height), facecolor='w', dpi=dpi) + ax = plt.gca() + + # Set colour cycle + mod = importlib.import_module("palettable.colorbrewer.%s" % colour_cycle[0]) + colors = getattr(mod, colour_cycle[1]).mpl_colors + ax.set_prop_cycle(cycler('color', colors)) + + return fig, ax + + + +def prettify_thermal_plot(fig, ax, options): + + required_options = ['plot_kind', 'hide_x_labels', 'hide_y_labels', 'rotation_x_ticks', 'rotation_y_ticks', 'xlabel', 'ylabel', 'xunit', 'yunit', 'xlim', 'ylim', 'x_tick_locators', 'y_tick_locators', 'hide_x_ticks', 'hide_y_ticks', 'hide_x_ticklabels', 'hide_y_ticklabels', + 'colours', 'palettes', 'title', 'legend', 'legend_position', 'subplots_adjust', 'text'] + + default_options = { + 'plot_kind': 'absolute', # absolute, relative, difference + 'hide_x_labels': False, # Whether x labels should be hidden + 'hide_x_ticklabels': False, + 'hide_x_ticks': False, + 'rotation_x_ticks': 0, + 'hide_y_labels': False, # whether y labels should be hidden + 'hide_y_ticklabels': False, + 'hide_y_ticks': False, + 'rotation_y_ticks': 0, + 'xlabel': 'Temperature', + 'ylabel': 'Energy', + 'xunit': r'K', # The unit of the x-values in the curve plot + 'yunit': r'eV f.u.$^{-1}$', # The unit of the y-values in the curve and bar plots + 'xlim': None, + 'ylim': None, + 'x_tick_locators': [100, 50], # Major and minor tick locators + 'y_tick_locators': [10, 5], + 'labels': None, + 'colours': None, + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], + 'title': None, + 'legend': True, + 'legend_position': ['upper center', (0.20, 0.90)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], + 'text': None + } + + + if 'plot_kind' in options.keys(): + if 'ylabel' not in options.keys(): + if options['plot_kind'] == 'absolute': + options['ylabel'] = 'Energy' + elif options['plot_kind'] == 'relative': + options['ylabel'] = 'Relative energy' + elif options['plot_kind'] == 'difference': + options['ylabel'] = 'Energy difference' + + if 'y_tick_locators' not in options.keys(): + if options['plot_kind'] == 'absolute' or options['plot_kind'] == 'relative': + options['y_tick_locators'] = [1, 0.5] + elif options['plot_kind'] == 'difference': + options['y_tick_locators'] = [0.1, 0.05] + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + # Set labels on x- and y-axes + if not options['hide_y_labels']: + ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + else: + ax.set_ylabel('') + + if not options['hide_x_labels']: + ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]') + else: + ax.set_xlabel('') + + + # Set multiple locators + ax.yaxis.set_major_locator(MultipleLocator(options['y_tick_locators'][0])) + ax.yaxis.set_minor_locator(MultipleLocator(options['y_tick_locators'][1])) + + ax.xaxis.set_major_locator(MultipleLocator(options['x_tick_locators'][0])) + ax.xaxis.set_minor_locator(MultipleLocator(options['x_tick_locators'][1])) + + # Hide x- and y- ticklabels + if options['hide_y_ticklabels']: + ax.tick_params(axis='y', direction='in', which='both', labelleft=False, labelright=False) + else: + plt.xticks(rotation=options['rotation_x_ticks']) + #ax.set_xticklabels(ax.get_xticks(), rotation = options['rotation_x_ticks']) + + if options['hide_x_ticklabels']: + ax.tick_params(axis='x', direction='in', which='both', labelbottom=False, labeltop=False) + else: + pass + #ax.set_yticklabels(ax.get_yticks(), rotation = options['rotation_y_ticks']) + + + # Hide x- and y-ticks: + if options['hide_y_ticks']: + ax.tick_params(axis='y', direction='in', which='both', left=False, right=False) + if options['hide_x_ticks']: + ax.tick_params(axis='x', direction='in', which='both', bottom=False, top=False) + + + + + + + # Set title + if options['title']: + ax.set_title(options['title']) + + + + + + # Create legend + + if ax.get_legend(): + ax.get_legend().remove() + + + if options['legend']: + + + # Make palette and linestyles from original parameters + if not options['colours']: + colours = generate_colours(palettes=options['palettes']) + else: + colours = itertools.cycle(options['colours']) + + + linestyles = itertools.cycle(options['linestyles']) + + # Create legend + custom_lines = [] + active_labels = [] + + for label in options['labels']: + + + # Discard next linestyle and colour if label is _ + if label == '_': + _ = next(colours) + _ = next(linestyles) + + else: + custom_lines.append(Line2D([0], [0], color=next(colours), ls=next(linestyles))) + active_labels.append(label) + + + + ax.legend(custom_lines, active_labels, frameon=False, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1]) + #fig.legend(handles=patches, loc=options['legend_position'][0], bbox_to_anchor=options['legend_position'][1], frameon=False) + + + + # Adjust where the axes start within the figure. Default value is 10% in from the left and bottom edges. Used to make room for the plot within the figure size (to avoid using bbox_inches='tight' in the savefig-command, as this screws with plot dimensions) + plt.subplots_adjust(left=options['subplots_adjust'][0], bottom=options['subplots_adjust'][1], right=options['subplots_adjust'][2], top=options['subplots_adjust'][3]) + + + # If limits for x- and y-axes is passed, sets these. + if options['xlim'] is not None: + ax.set_xlim(options['xlim']) + + if options['ylim'] is not None: + ax.set_ylim(options['ylim']) + + + # Add custom text + if options['text']: + plt.text(x=options['text'][1][0], y=options['text'][1][1], s=options['text'][0]) + + return fig, ax + + + +def prettify_thermal_plot_old(fig, ax, options, colour_cycle=('qualitative', 'Dark2_8'), temperatureunit='K', energyunit='eV f.u.$^{-1}$', mode='absolute', scale=1, linestyles=None, labels=None, xpad=None, ypad=None): + + # Set sizes of ticks, labes etc. + ticksize = 30*scale + labelsize = 30*scale + legendsize = 30*scale + titlesize = 30*scale + + linewidth = 3*scale + axeswidth = 3*scale + majorticklength = 20*scale + minorticklength = 10*scale + + xpad = 4 if not xpad else xpad + ypad = 4 if not ypad else ypad + + + # Set labels on x- and y-axes + ax.set_xlabel('Temperature [{}]'.format(temperatureunit), size=labelsize, labelpad=xpad) + if mode == 'absolute': + ax.set_ylabel('Total energy [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + elif mode == 'difference_plot': + ax.set_ylabel('Energy difference [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + + elif mode == 'relative': + ax.set_ylabel('Relative energy [{}]'.format(energyunit), size=labelsize, labelpad=ypad) + + # Set tick parameters + ax.tick_params(axis='x', direction='in', which='major', top=True, length=majorticklength, width=axeswidth, pad=xpad) + ax.tick_params(axis='x', direction='in', which='minor', top=True, length=minorticklength, width=axeswidth) + + ax.tick_params(axis='y', direction='in', which='major', right=True, length=majorticklength, width=axeswidth, pad=ypad) + ax.tick_params(axis='y', direction='in', which='minor', right=True, length=minorticklength, width=axeswidth) + + + ax.xaxis.set_major_locator(MultipleLocator(100)) + ax.xaxis.set_minor_locator(MultipleLocator(50)) + + if mode == 'absolute': + ax.yaxis.set_major_locator(MultipleLocator(1)) + ax.yaxis.set_minor_locator(MultipleLocator(0.5)) + + elif mode == 'difference_plot': + ax.yaxis.set_major_locator(MultipleLocator(0.1)) + ax.yaxis.set_minor_locator(MultipleLocator(0.05)) + + + + if labels: + + custom_lines = [] + + if not linestyles: + linestyles = ['solid', 'dotted', 'dashed'] + + mod = importlib.import_module("palettable.colorbrewer.%s" % colour_cycle[0]) + palette = getattr(mod, colour_cycle[1]).mpl_colors + palette = itertools.cycle(palette) + + colours = [] + for label in labels: + colours.append(next(palette)) + + + patches = [] + for ind, label in enumerate(labels): + custom_lines.append(Line2D([0], [0], color=colours[ind], lw=linewidth, ls=linestyles[ind])) + + + ax.legend(custom_lines, labels, fontsize=labelsize, frameon=False) + + + plt.xticks(fontsize=ticksize, rotation=45) + plt.yticks(fontsize=ticksize) + + + return fig, ax + + +def prepare_inset_axes(parent_ax, options): + + required_options = ['hide_inset_x_labels','hide_inset_x_ticklabels', 'hide_inset_x_ticks', 'rotation_inset_x_ticks', 'hide_inset_y_labels', 'hide_inset_y_ticklabels', 'hide_inset_y_ticks', 'rotation_inset_y_ticks', + 'inset_x_tick_locators', 'inset_y_tick_locators', 'inset_position', 'legend_position'] + + default_options = { + 'hide_inset_x_labels': False, # Whether x labels should be hidden + 'hide_inset_x_ticklabels': False, + 'hide_inset_x_ticks': False, + 'rotation_inset_x_ticks': 0, + 'hide_inset_y_labels': False, # whether y labels should be hidden + 'hide_inset_y_ticklabels': False, + 'hide_inset_y_ticks': False, + 'rotation_inset_y_ticks': 0, + 'inset_x_tick_locators': [100, 50], # Major and minor tick locators + 'inset_y_tick_locators': [10, 5], + 'inset_position': [0.1,0.1,0.3,0.3], + 'legend_position': ['upper center', (0.20, 0.90)] # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + } + + + options = update_options(options=options, required_options=required_options, default_options=default_options) + + + # Create a set of inset Axes: these should fill the bounding box allocated to + # them. + inset_ax = plt.axes([0, 0, 2, 2]) + # Manually set the position and relative size of the inset axes within ax1 + ip = InsetPosition(parent_ax, options['inset_position']) + inset_ax.set_axes_locator(ip) + + mark_inset(parent_ax, inset_ax, loc1=2, loc2=4, fc='none', ec='black') + + inset_ax.xaxis.set_major_locator(MultipleLocator(options['inset_x_tick_locators'][0])) + inset_ax.xaxis.set_minor_locator(MultipleLocator(options['inset_x_tick_locators'][1])) + + + inset_ax.yaxis.set_major_locator(MultipleLocator(options['inset_y_tick_locators'][0])) + inset_ax.yaxis.set_minor_locator(MultipleLocator(options['inset_y_tick_locators'][1])) + + + + + return inset_ax + + + +def update_rc_params(rc_params): + ''' Update all passed run commands in matplotlib''' + + if rc_params: + for key in rc_params.keys(): + plt.rcParams.update({key: rc_params[key]}) + + + +def update_options(options, required_options, default_options): + ''' Update all passed options''' + + + for option in required_options: + if option not in options.keys(): + options[option] = default_options[option] + + + + return options + + + +def determine_width(options): + + conversion_cm_inch = 0.3937008 # cm to inch + + if options['column_type'] == 'single': + column_width = options['single_column_width'] + elif options['column_type'] == 'double': + column_width = options['double_column_width'] + + column_width *= conversion_cm_inch + + + width_ratio = [float(num) for num in options['width_ratio'].split(':')] + + + width = column_width * width_ratio[0]/width_ratio[1] + + + return width + + +def determine_height(options, width): + + aspect_ratio = [float(num) for num in options['aspect_ratio'].split(':')] + + height = width/(aspect_ratio[0] / aspect_ratio[1]) + + return height + + +def scale_figure(options, width, height): + width = width * options['upscaling_factor'] * options['compress_width'] + height = height * options['upscaling_factor'] * options['compress_height'] + + return width, height + + + +def swap_values(dict, key1, key2): + + key1_val = dict[key1] + dict[key1] = dict[key2] + dict[key2] = key1_val + + return dict + + + +def generate_colours(palettes): + + # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. + colour_collection = [] + for palette in palettes: + mod = importlib.import_module("palettable.colorbrewer.%s" % palette[0]) + colour = getattr(mod, palette[1]).mpl_colors + colour_collection = colour_collection + colour + + colour_cycle = itertools.cycle(colour_collection) + + + return colour_cycle \ No newline at end of file From caca4e0c1fa995448bdd0c33837072e05b895902 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 20 Sep 2022 20:24:16 +0200 Subject: [PATCH 287/355] Allow for plotting EoS even when one fails --- nafuma/dft/structure.py | 63 ++++++++++++++++++++++++++++------------- 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/nafuma/dft/structure.py b/nafuma/dft/structure.py index 04584cb..20f2994 100644 --- a/nafuma/dft/structure.py +++ b/nafuma/dft/structure.py @@ -3,6 +3,7 @@ import re import pandas as pd import numpy as np from scipy.optimize import curve_fit +import warnings import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator) @@ -21,6 +22,9 @@ from ase.eos import EquationOfState import os import os.path +import nafuma.auxillary as aux +import nafuma.plotting as btp + def read_eos_data(path, options): ''' Reads volume and energy data from a energy-volume run and fits the data to an equation of state. Outputs a list with one pandas DataFrame containing the data points from the DFT-calculations, @@ -70,15 +74,21 @@ def read_eos_data(path, options): # Fit data to Equation of State using ASEs EquationOfState object. Makes a DataFrame out of the data points of the fitted curve. Also makes a ditionary of the equilibrium constants, #then packages everything in a list which is returned by the function. eos = EquationOfState(dft_df['Volume'].values, dft_df['Energy'].values, eos=options['eos']) - v0, e0, B = eos.fit() - eos_df = pd.DataFrame(data={'Volume': eos.getplotdata()[4], 'Energy': eos.getplotdata()[5]}) + try: + v0, e0, B = eos.fit() + eos_df = pd.DataFrame(data={'Volume': eos.getplotdata()[4], 'Energy': eos.getplotdata()[5]}) - equilibrium_constants = {'v0': v0, 'e0': e0,'B': B/kJ * 1.0e24} + equilibrium_constants = {'v0': v0, 'e0': e0,'B': B/kJ * 1.0e24} - data = [dft_df, eos_df, equilibrium_constants, label] - - return data + data = [dft_df, eos_df, equilibrium_constants, label] + + return data + + except: + warnings.warn(f'WARNING: Unable to fit EoS curve for {label}') + + return [None, None, None, label] def read_eos_datas(path, options): @@ -107,7 +117,9 @@ def read_eos_datas(path, options): for dir in dirs: subdir = os.path.join(path, dir) data = read_eos_data(subdir, options) - datas.append(data) + + if isinstance(data[0], pd.DataFrame): + datas.append(data) # Sorts the data if sort is enabled. @@ -160,12 +172,17 @@ def plot_eos_data(path, options): highlight: Takes a list, either of booleans to highlight certain bars (must be the same length as the number of data sets). Alternatively can contain only names of the datasets to highlight. Defaults to None.''' + + # FIXME A lot of refactoring required to tidy this up + required_options = ['plot_kind', 'highlight', 'reference', 'eos', 'sort_by', - 'curve_colours', - 'bar_colours', - 'marker_cycle', + 'colours', + 'xlabel', 'ylabel', + 'xunit', 'yunit', + 'palettes', + 'markers', 'ylim', 'legend_map', 'rc_params', @@ -178,9 +195,11 @@ def plot_eos_data(path, options): 'reference': 0, # Whether the energy should be relative to some reference energy (typically lowest energy) 'eos': 'birchmurnaghan', # what type of EoS curve to fit the data to. Options: murnaghan, birch, birchmurnaghan, vinet, pouriertarantola 'sort_by': 'e0', # whether the data should be sorted or not - relevant for bar plots, but also for the order of the entries in the legend in the EoScruve plot - 'curve_colours': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], # a set of two colour cycles from the palettable package. Requires many colours for the EoScurve plot - 'bar_colours': [('qualitative', 'Dark2_3'), ('qualitative', 'Pastel2_3')], # set of two colour cycles from the palettable package. Extracts first element of each to make the highlighted and subdued colours respectively. Should be replaced in future by explicit passing of colours - 'marker_cycle': ('o', '*', '^', 'v', 'd', 'H', '8', '>', 'P', 'X'), # marker styles for the EoScurve plot + 'colours': None, + 'xlabel': 'Volume', 'ylabel': 'Energy', + 'xunit': 'Å$^3$', 'yunit': 'eV', + 'palettes': [('qualitative', 'Dark2_8'), ('qualitative', 'Paired_12')], # a set of two colour cycles from the palettable package. Requires many colours for the EoScurve plot + 'markers': ('o', '*', '^', 'v', 'd', 'H', '8', '>', 'P', 'X'), # marker styles for the EoScurve plot 'ylim': None, # y-limits (ist) 'legend': True, 'legend_map': None, # a dictionary with mappings between the folder names and what should appear in the legend @@ -198,15 +217,15 @@ def plot_eos_data(path, options): if options['plot_kind'] == 'EoScurve': # Fetches a figure and axes object from the prepare_plot() function - fig, ax = prepare_plot(options=options) + fig, ax = btp.prepare_plot(options=options) # Make an cyclic iterable of markers to be used for the calculated data points. - marker_cycle = itertools.cycle(options['marker_cycle']) + marker_cycle = itertools.cycle(options['markers']) # Creates a list of all the colours that is passed in the colour_cycles argument. Then makes cyclic iterables of these. colour_collection = [] - for cycle in options['curve_colours']: + for cycle in options['palettes']: mod = importlib.import_module("palettable.colorbrewer.%s" % cycle[0]) colour = getattr(mod, cycle[1]).mpl_colors colour_collection = colour_collection + colour @@ -239,16 +258,18 @@ def plot_eos_data(path, options): dft_df.plot.scatter(x=1, y=2, ax=ax, marker=markers[-1], color=colours[-1], s=20) eos_df.plot(x=0, y=1, ax=ax, color=colours[-1], label='_', ls='--') - + options['labels'] = labels + if options['legend']: options['legend_content'] = [labels, colours, markers] + ### PLOT THE BAR PLOTS elif options['plot_kind'] == 'EoSbars': # Fetches a figure and axes object from the prepare_plot() function - fig, ax = prepare_plot(options=options) + fig, ax = btp.prepare_plot(options=options) e0 = [] labels = [] @@ -257,7 +278,7 @@ def plot_eos_data(path, options): # Pick out colour for highlighting (NB! These colours are not passed as arguments, but could be in future) bar_colours = [] - for cycle in options['bar_colours']: + for cycle in options['palettes']: mod = importlib.import_module("palettable.colorbrewer.%s" % cycle[0]) bar_colours.append(getattr(mod, cycle[1]).mpl_colors[0]) @@ -301,7 +322,9 @@ def plot_eos_data(path, options): plt.xticks(range(len(e0)), labels, rotation=90) - fig, ax = prettify_plot(fig=fig, ax=ax, options=options) + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + return datas, fig, ax From 4fd68eab55bbe2e09d738f8404b7df2611ac671b Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 20 Sep 2022 20:24:32 +0200 Subject: [PATCH 288/355] Add plotting of refinement data --- nafuma/xrd/plot.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 4ed6d7a..3b3c6bd 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -926,3 +926,79 @@ def make_animation(data: dict, options={}): shutil.rmtree('tmp') + + +def plot_refinement(data, options={}): + + + required_options = ['diff_offset', 'index', 'title', 'xlim', 'r_wp', 'r_exp', 'wp'] + + default_options = { + 'diff_offset': .10, + 'index': -1, + 'title': None, + 'xlim': None, + 'r_wp': True, + 'r_exp': False, + 'wp': False, + } + + options = aux.update_options(options=options, default_options=default_options, required_options=required_options) + + df = pd.read_csv(data['path'], delim_whitespace=True, header=None) + df.columns = ['2th', 'Yobs', 'Ycalc', 'diff'] + df['diff'] = df['diff'] - options['diff_offset']*(df['Yobs'].max() - df['Yobs'].min()) + + + if not isinstance(data['results'], list): + data['results'] = [data['results']] + + results = { + 'vol': [], + 'mass': [], + 'wp': [], + 'a': [], + 'b': [], + 'c': [], + 'alpha': [], + 'beta': [], + 'gamma': [] + } + + for result in data['results']: + result = xrd.refinement.read_results(path=result) + + r_wp = result['r_wp'].iloc[options['index']] + r_exp = result['r_exp'].iloc[options['index']] + + for attr in results.keys(): + results[attr].append(result[attr].iloc[options['index']]) + + fig, ax = plt.subplots(figsize=(20,10)) + + df.plot(x='2th', y='Yobs', kind='scatter', ax=ax, c='black', marker='$\u25EF$') + df.plot(x='2th', y='Ycalc', ax=ax, c='red') + df.plot(x='2th', y='diff', ax=ax) + + if options['r_wp']: + ax.text(x=0.7*df['2th'].max(), y=0.7*df['Yobs'].max(), s='R$_{wp}$ = '+f'{r_wp}', fontsize=20) + + if options['r_exp']: + ax.text(x=0.70*df['2th'].max(), y=0.60*df['Yobs'].max(), s='R$_{exp}$ = '+f'{r_exp}', fontsize=20) + + + if options['wp']: + for i, (result, label) in enumerate(zip(data['results'], options['labels'])): + ax.text(x=0.7*df['2th'].max(), y=(0.9-0.1*i)*df['Yobs'].max(), s=f'{label}: {np.round(float(results["wp"][i]), 2)}%', fontsize=20) + + if options['title']: + ax.set_title(options['title'], size=30) + + if options['xlim']: + ax.set_xlim(options['xlim']) + else: + ax.set_xlim([df['2th'].min(), df['2th'].max()]) + + ax.tick_params(which='both', labelleft=False, left=False, labelsize=20, direction='in') + ax.set_ylabel('Intensity [arb. u.]', size=20) + ax.set_xlabel('2$\\theta$ [$^{\circ}$]', size=20) \ No newline at end of file From 006cddaaa3ad39dd699cfd270064663343986441 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 20 Sep 2022 20:25:18 +0200 Subject: [PATCH 289/355] Add new params and add define statements --- nafuma/xrd/refinement.py | 99 +++++++++++++++++++++++++++++++++++----- 1 file changed, 88 insertions(+), 11 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index a8e27ff..4fe4f78 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -16,7 +16,7 @@ import nafuma.auxillary as aux def make_initial_inp(data: dict, options={}): - required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'topas_options', 'background', 'capillary', 'start', 'finish', 'exclude'] + required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'topas_options', 'background', 'capillary', 'th2_offset', 'fit_peak_width', 'start', 'finish', 'exclude'] default_options = { 'filename': 'start.inp', @@ -27,10 +27,12 @@ def make_initial_inp(data: dict, options={}): 'topas_options': {}, 'background': 7, 'capillary': False, + 'th2_offset': False, + 'fit_peak_width': False, 'interval': [None, None], # Start and finish values that TOPAS should refine on. Overrides 'start' and 'finish' 'start': None, # Start value only. Overridden by 'interval' if this is set. 'finish': None, # Finish value only. Overridden by 'interval' if this is set. - 'exlude': [] # Excluded regions. List of lists. + 'exclude': [] # Excluded regions. List of lists. } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -70,6 +72,23 @@ def make_initial_inp(data: dict, options={}): write_str(fout=fout, data=data, options=options, index=i) + with open(options['filename'], 'r') as fin: + + lines = [] + for statement in data['define_statements']: + lines.append(f'\'#define {statement}\n') + + lines.append('\n') + + lines += fin.readlines() + + + with open(options['filename'], 'w') as fout: + for line in lines: + fout.write(line) + + + def write_xdd(fout, data, options): @@ -108,7 +127,13 @@ def write_xdd(fout, data, options): fout.write('\t'+snippets['wavelength'].format(data['wavelength'])+'\n') fout.write('\t'+snippets['lp_factor'].format(topas_options['lp_factor'])+'\n') - fout.write('\t'+snippets['zero_error']+'\n') + + if options['th2_offset']: + fout.write('\n') + for line in snippets['th2_offset']: + fout.write('\t'+line+'\n') + else: + fout.write('\t'+snippets['zero_error']+'\n') if options['capillary']: fout.write('\n') @@ -139,6 +164,9 @@ def write_xdd(fout, data, options): def write_params(fout, data, options, index=0): + # List to include #define-statements at top of INP-file + data['define_statements'] = [] + atoms = read_cif(data['str'][index]) if 'labels' in data.keys(): label = data['labels'][index] @@ -161,6 +189,7 @@ def write_params(fout, data, options, index=0): # WRITE LATTICE PARAMETERS # If start_values is defined: fout.write(f'#ifdef start_values_{label}\n') + data['define_statements'].append(f'start_values_{label}') lpa = f'local !lpa_{label} {a} ;: {a}' lpb = f'local !lpb_{label} {b} ;: {b}' lpc = f'local !lpc_{label} {c} ;: {c}' @@ -228,6 +257,12 @@ def write_params(fout, data, options, index=0): def write_str(fout, data, options, index=0): + import nafuma.xrd as xrd + basedir = os.path.dirname(xrd.__file__) + + with open (os.path.join(basedir, 'snippets.json'), 'r') as snip: + snippets = json.load(snip) + atoms = read_cif(data['str'][index]) if 'labels' in data.keys(): label = data['labels'][index] @@ -256,18 +291,24 @@ def write_str(fout, data, options, index=0): fout.write('\n') fout.write('\t\tscale @ 1.0\n') fout.write('\t\tr_bragg 1.0\n\n') + if options['fit_peak_width']: + fout.write('\t\t'+snippets['fit_peak_width']+'\n\n') fout.write(f'\t\t#ifdef crystallite_size_gaussian_{label}\n') + data['define_statements'].append(f'crystallite_size_gaussian_{label}') fout.write(f'\t\tCS_G(csgc_{label}_XXXX)\n') fout.write('\t\t#endif\n') fout.write(f'\t\t#ifdef crystallite_size_lorentzian_{label}\n') + data['define_statements'].append(f'crystallite_size_lorentzian_{label}') fout.write(f'\t\tCS_L(cslc_{label}_XXXX)\n') fout.write('\t\t#endif\n\n') fout.write(f'\t\t#ifdef strain_gaussian_{label}\n') + data['define_statements'].append(f'strain_gaussian_{label}') fout.write(f'\t\tStrain_G(sgc_{label}_XXXX)\n') fout.write('\t\t#endif\n') fout.write(f'\t\t#ifdef strain_lorentzian_{label}\n') + data['define_statements'].append(f'strain_lorentzian_{label}') fout.write(f'\t\tStrain_L(slc_{label}_XXXX)\n') fout.write('\t\t#endif\n\n') @@ -277,10 +318,16 @@ def write_str(fout, data, options, index=0): for atom in atoms['atoms'].keys(): - specie = '{}{}'.format( + if 'oxidation_states' in data.keys(): + specie = '{}{}'.format( atoms["atoms"][atom]["_atom_site_type_symbol"], data['oxidation_states'][index][atoms["atoms"][atom]["_atom_site_type_symbol"]] - ) + ) + else: + specie = '{}'.format( + atoms["atoms"][atom]["_atom_site_type_symbol"] + ) + site = f'site {atom}' x = f'\t\tx = x_{atom}_{label}' @@ -317,11 +364,11 @@ def write_output(fout, data, options, index=0): fout.write('#ifdef output\n') - fout.write(f'\t\tOut_Riet({options["save_dir"]}/{label}_XXXX_riet.xy)\n') - fout.write(f'\t\tOut_CIF_STR({options["save_dir"]}/{label}_XXXX.cif)\n') - fout.write(f'\t\tOut_CIF_ADPs({options["save_dir"]}/{label}_XXXX.cif)\n') - fout.write(f'\t\tOut_CIF_Bonds_Angles({options["save_dir"]}/{label}_XXXX.cif)\n') - fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file({options["save_dir"]}/{label}_XXXX_hkl.dat)\n') + fout.write(f'\t\tOut_Riet({options["save_dir"]}/{filename}_{label}_XXXX_riet.xy)\n') + fout.write(f'\t\tOut_CIF_STR({options["save_dir"]}/{filename}_{label}_XXXX.cif)\n') + fout.write(f'\t\tOut_CIF_ADPs({options["save_dir"]}/{filename}_{label}_XXXX.cif)\n') + fout.write(f'\t\tOut_CIF_Bonds_Angles({options["save_dir"]}/{filename}_{label}_XXXX.cif)\n') + fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file({options["save_dir"]}/{filename}_{label}_XXXX_hkl.dat)\n') fout.write('\n') @@ -381,6 +428,20 @@ def write_output(fout, data, options, index=0): fout.write('\n') fout.write('\t\tOut_String("\\n")\n') + + if options['fit_peak_width']: + fout.write(f'out {options["save_dir"]}/{filename}_peak_width.dat\n') + fout.write('\t\tOut_String("prm !ad = \t\t")\n') + fout.write('\t\tOut(ad)') + fout.write('\tOut_String(";\\n")\n') + fout.write('\t\tOut_String("prm !bd = \t\t")\n') + fout.write('\t\tOut(bd)') + fout.write('\tOut_String(";\\n")\n') + fout.write('\t\tOut_String("prm !cd = \t\t")\n') + fout.write('\t\tOut(cd)') + fout.write('\tOut_String(";")\n') + + fout.write('#endif') fout.write('\n\n') @@ -785,7 +846,23 @@ def refine(data: dict, options={}): def read_results(path): - results = pd.read_csv(path, delim_whitespace=True, index_col=0) + results = pd.read_csv(path, delim_whitespace=True, index_col=0, header=None) + + atoms = int((results.shape[1] - 15) / 5) + + headers = [ + 'r_wp', 'r_exp', 'r_p', 'r_p_dash', 'r_exp_dash', 'gof', + 'vol', 'mass', 'wp', + 'a', 'b', 'c', 'alpha', 'beta', 'gamma', + ] + + + labels = ['x', 'y', 'z', 'occ', 'beq'] + for i in range(atoms): + for label in labels: + headers.append(f'atom_{i+1}_{label}') + + results.columns = headers return results From 07f304ce9759622b344612ab2345bf55759ecd0f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 20 Sep 2022 20:25:30 +0200 Subject: [PATCH 290/355] Add new snippets --- nafuma/xrd/snippets.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nafuma/xrd/snippets.json b/nafuma/xrd/snippets.json index 5b0f97d..00266bd 100644 --- a/nafuma/xrd/snippets.json +++ b/nafuma/xrd/snippets.json @@ -3,12 +3,19 @@ "capillary": [ "local !packing_density {} min 0.1 max 1.0 'typically 0.2 to 0.5", "local !capdia {} 'capillary diameter in mm", - "local !linab = Get(mixture_MAC) Get(mixture_density_g_on_cm3);: 'in cm-1", + "local !linab = Get(mixture_MAC) Get(mixture_density_g_on_cm3);: 100 'in cm-1", "local muR = (capdia/20)*linab*packing_density;", "Cylindrical_I_Correction(muR)" ], "gauss_fwhm": "gauss_fwhm = Sqrt({} Cos(2 * Th)^4 + {} Cos(2 * Th)^2 + {});", "lp_factor": "LP_Factor({}) 'change the LP correction or lh value if required", "wavelength": "lam ymin_on_ymax 0.0001 la 1.0 lo {} lh 0.1", - "zero_error": "Zero_Error(zero, 0)" + "zero_error": "Zero_Error(zero, 0)", + "th2_offset": [ + "prm zero \t\t\t0 \t\t\t\tmin = Max(Val - 20 Yobs_dx_at(X1), -100 Yobs_dx_at(X1)); max = Min(Val + 20 Yobs_dx_at(X2), 100 Yobs_dx_at(X2)); del = .01 Yobs_dx_at(X1); val_on_continue 0", + "prm cos_shift \t\t0 \t\t\t\tmin = Val-.8; max = Val+.8; del 0.001", + "prm sin_shift \t\t0 \t\t\t\tmin = Val-.8; max = Val+.8; del 0.001", + "th2_offset = (zero) + (cos_shift) Cos(Th) + (sin_shift) Sin(Th) ;" + ], + "fit_peak_width": "DC1( ad, 0, bd, 0, cd, 0)" } \ No newline at end of file From 8aa8164a803f3c28193bee5d1e633d5de87fa879 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 27 Sep 2022 10:19:33 +0200 Subject: [PATCH 291/355] Small adjustment to legend plotter --- nafuma/plotting.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 367b74c..74db433 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -6,6 +6,7 @@ import importlib import matplotlib.patches as mpatches from matplotlib.lines import Line2D import matplotlib.lines as mlines +import matplotlib.markers as mmarkers import itertools @@ -25,11 +26,13 @@ def prepare_plot(options={}): else: rc_params = {} + if 'format_params' in options.keys(): format_params = options['format_params'] else: format_params = {} + required_format_params = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', 'width', 'height', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi', 'nrows', 'ncols', 'grid_ratio_height', 'grid_ratio_width'] @@ -54,7 +57,7 @@ def prepare_plot(options={}): format_params = aux.update_options(format_params, required_format_params, default_format_params) - + # Reset run commands @@ -107,6 +110,7 @@ def adjust_plot(fig, ax, options): 'title', 'legend', 'legend_position', 'legend_ncol', 'subplots_adjust', + 'marker_edges', 'text'] default_options = { @@ -123,6 +127,7 @@ def adjust_plot(fig, ax, options): 'title': None, # Title of the plot 'legend': False, 'legend_position': ['lower center', (0.5, -0.1)], 'legend_ncol': 1, # Toggles on/off legend. Specifices legend position and the number of columns the legend should appear as. 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], # Adjustment of the Axes-object within the Figure-object. Fraction of the Figure-object the left, bottom, right and top edges of the Axes-object will start. + 'marker_edges': None, 'text': None # Text to show in the plot. Should be a list where the first element is the string, and the second is a tuple with x- and y-coordinates. Could also be a list of lists to show more strings of text. } @@ -229,7 +234,12 @@ def adjust_plot(fig, ax, options): _ = next(markers) else: - active_markers.append(mlines.Line2D([], [], markeredgecolor=next(colours), color=(1, 1, 1, 0), marker=next(markers))) + marker = next(markers) + if not marker: + active_markers.append(mlines.Line2D([], [], color=next(colours))) + else: + active_markers.append(mlines.Line2D([], [], markerfacecolor=next(colours), markeredgecolor=options['marker_edges'], markersize=10, color=(1,1,1,0), marker=marker)) + active_labels.append(label) From 61cc0343e01514ecb2e5514fa472e33a74d4696d Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 27 Sep 2022 10:19:51 +0200 Subject: [PATCH 292/355] Add POSCAR writer and str generator --- nafuma/dft/io.py | 213 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) diff --git a/nafuma/dft/io.py b/nafuma/dft/io.py index 0c32451..6f0a60e 100644 --- a/nafuma/dft/io.py +++ b/nafuma/dft/io.py @@ -2,6 +2,10 @@ import pandas as pd import matplotlib.pyplot as plt import numpy as np +import warnings + +import os + import linecache import nafuma.auxillary as aux @@ -543,3 +547,212 @@ def get_doscar_information(path): #def get_bader_charges(poscar='POSCAR', acf='ACF.dat'): + + +def write_poscar(data: dict, options={}): + + required_options = ['path', 'overwrite', 'scale'] + + default_options = { + 'path': './POSCAR.vasp', + 'overwrite': False, + 'scale': 1.0 + } + + options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + + + if os.path.isfile(options['path']) and not options['overwrite']: + warnings.warn(f"File {options['path']} already exists, and overwrite disabled. Quitting.") + return None + + + atom_nums = count_atoms(data) + + with open(options['path'], 'w') as fout: + + # Write first line + for atom in data['atoms']: + if atom_nums[atom] > 1: + fout.write(f'{atom}{atom_nums[atom]}') + else: + fout.write(f'{atom}') + + fout.write('\t - Automatically generated by the NAFUMA Python package.\n') + + # Write second line + fout.write(str(options['scale'])+'\n') + + # Write lattice parameters + # FIXME Now only writes cells without any angles + fout.write('\t{:<09} \t {:<09} \t {:<09}\n'.format( + str(data['lattice_parameters'][0]), + str(0.0), + str(0.0), + ) + ) + + fout.write('\t{:<09} \t {:<09} \t {:<09}\n'.format( + str(0.0), + str(data['lattice_parameters'][1]), + str(0.0), + ) + ) + + fout.write('\t{:<09} \t {:<09} \t {:<09}\n'.format( + str(0.0), + str(0.0), + str(data['lattice_parameters'][2]), + ) + ) + + + # Write atoms + for atom in data['atoms']: + fout.write(f'\t{atom}') + fout.write('\n') + + for atom in data['atoms']: + fout.write(f'\t{atom_nums[atom]}') + fout.write('\n') + + fout.write('Direct\n') + + + + + for atom in data['atoms']: + for label, coords in data['coordinates'].items(): + if atom in label: + fout.write('\t{:<09} \t {:<09} \t {:<09}\n'.format( + coords[0], + coords[1], + coords[2] + ) + ) + + + + +def apply_transformation(data, rotation=[[1, 0, 0], [0, 1, 0], [0, 0, 1]], translation=[0,0,0]): + + if not isinstance(rotation, np.ndarray): + rotation = np.array(rotation) + + if not rotation.shape == (3,3): + print("NOOOO!!!!") + + for label in data['coordinates'].keys(): + data['coordinates'][label] = rotation.dot(data['coordinates'][label]) + data['coordinates'][label] = translate_back(data['coordinates'][label]) + data['coordinates'][label] = data['coordinates'][label] + translation + + return data + + +def translate_back(coords): + + for i, coord in enumerate(coords): + if coord < 0: + while coords[i] < 0: + coords[i] = coords[i]+1 + + elif coord >= 1: + while coords[i] >= 1: + coords[i] = coords[i]-1 + + return coords + + +def count_atoms(data): + atom_nums = {} + for atom in data['atoms']: + atom_nums[atom] = 0 + + for label in data['coordinates'].keys(): + for atom in data['atoms']: + if atom in label: + atom_nums[atom] += 1 + + + return atom_nums + +def append_data(data, new_data): + + if not new_data: + return data + + if not data: + data = { + 'atoms': new_data['atoms'], + 'coordinates': {}, + 'lattice_parameters': new_data['lattice_parameters'] + } + + atom_num = count_atoms(data) + + new_coords = {} + + for label, coords in data['coordinates'].items(): + new_coords[label] = coords + + + extend_unit_cell = [0,0,0] + for label, coords in new_data['coordinates'].items(): + atom = ''.join(filter(str.isalpha, label)) + number = int(''.join(filter(str.isnumeric, label))) + new_number = number + atom_num[atom] + new_label = atom+str(new_number) + + new_coords[new_label] = coords + + for i, coord in enumerate(coords): + if coord > 1 and np.floor(coord) >= extend_unit_cell[i]: + extend_unit_cell[i] = np.floor(coord) + + data['coordinates'] = new_coords + + return data + + +def make_supercell(data, supercell): + + for i, param in enumerate(data['lattice_parameters']): + data['lattice_parameters'][i] = supercell[i] * param + + if supercell[i] > 0: + + for label, coords in data['coordinates'].items(): + data['coordinates'][label][i] = (1/supercell[i])*data['coordinates'][label][i] + + + + +def copy_data(data): + + new_data = {} + + new_data['atoms'] = list(data['atoms']) + new_data['coordinates'] = data['coordinates'].copy() + new_data['lattice_parameters'] = list(data['lattice_parameters']) + + return new_data + + +def unit_vector(vector): + """ Returns the unit vector of the vector. """ + return vector / np.linalg.norm(vector) + +def angle_between(v1, v2): + """ Returns the angle in radians between vectors 'v1' and 'v2':: + + >>> angle_between((1, 0, 0), (0, 1, 0)) + 1.5707963267948966 + >>> angle_between((1, 0, 0), (1, 0, 0)) + 0.0 + >>> angle_between((1, 0, 0), (-1, 0, 0)) + 3.141592653589793 + """ + v1_u = unit_vector(v1) + v2_u = unit_vector(v2) + return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0)) \ No newline at end of file From 6247ce24bf209ff52e4d0d129ce7bec578f713f0 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 27 Sep 2022 10:20:13 +0200 Subject: [PATCH 293/355] Initial add of EDS module --- nafuma/eds/__init__.py | 1 + nafuma/eds/io.py | 99 ++++++++++++++++++++++++++++++++++++++++++ nafuma/eds/plot.py | 60 +++++++++++++++++++++++++ 3 files changed, 160 insertions(+) create mode 100644 nafuma/eds/__init__.py create mode 100644 nafuma/eds/io.py create mode 100644 nafuma/eds/plot.py diff --git a/nafuma/eds/__init__.py b/nafuma/eds/__init__.py new file mode 100644 index 0000000..e0e052c --- /dev/null +++ b/nafuma/eds/__init__.py @@ -0,0 +1 @@ +from . import io, plot \ No newline at end of file diff --git a/nafuma/eds/io.py b/nafuma/eds/io.py new file mode 100644 index 0000000..8f269d6 --- /dev/null +++ b/nafuma/eds/io.py @@ -0,0 +1,99 @@ +from PIL import Image +import numpy as np +import cv2 + +def read_image(path, weight=None, colour=None, resize=None): + + img = np.array(Image.open(path)) + + if colour is not None: + img = change_colour(img, colour) + + if resize is not None: + img = resize_image(img, resize) + + if weight is not None: + img = scale_image(img, weight) + + return img + + +def scale_image(image, factor): + + + for i in range(0,image.shape[0]): + for j in range(0, image.shape[1]): + image[i][j][0] = image[i][j][0]*factor + image[i][j][1] = image[i][j][1]*factor + image[i][j][2] = image[i][j][2]*factor + + return image + + +def resize_image(image, factor): + + y, x = image.shape[0:2] + + new_y, new_x = int(y*factor), int(x*factor) + + image = image[:new_y, :new_x] + + res = cv2.resize(image, dsize=(x, y), interpolation=cv2.INTER_CUBIC) + + return res + + +def add_images(image1, image2): + + assert image1.shape == image2.shape + + compound_image = np.zeros((image1.shape[0], image1.shape[1], image1.shape[2])) + for i in range(image1.shape[0]): + for j in range(image1.shape[1]): + compound_image[i][j] = [0, 0, 0] + + compound_image[i][j][0] = int(int(image1[i][j][0]) + int(image2[i][j][0])) + compound_image[i][j][1] = int(int(image1[i][j][1]) + int(image2[i][j][1])) + compound_image[i][j][2] = int(int(image1[i][j][2]) + int(image2[i][j][2])) + + + + return compound_image + + + + +def get_colour(image): + + + colour = [0, 0, 0] + for i in range(image.shape[0]): + for j in range(image.shape[1]): + if image[i][j][0] > colour[0]: + colour[0] = image[i][j][0] + + if image[i][j][1] > colour[1]: + colour[1] = image[i][j][1] + + if image[i][j][2] > colour[2]: + colour[2] = image[i][j][2] + + colour = np.array(colour) + + return colour + + +def change_colour(image, new_colour): + + new_colour = np.array(new_colour) + + old_colour = get_colour(image) + + + for i in range(image.shape[0]): + for j in range(image.shape[1]): + factor = max(image[i][j]) / max(old_colour) + image[i][j] = new_colour.astype(float) * factor + + + return image diff --git a/nafuma/eds/plot.py b/nafuma/eds/plot.py new file mode 100644 index 0000000..9e634e4 --- /dev/null +++ b/nafuma/eds/plot.py @@ -0,0 +1,60 @@ +import nafuma.auxillary as aux +import nafuma.plotting as btp +import nafuma.eds.io as io + +import numpy as np + +def show_image(data, options={}): + + + default_options = { + 'hide_x_labels': True, + 'hide_y_labels': True, + 'hide_x_ticklabels': True, + 'hide_y_ticklabels': True, + 'hide_x_ticks': True, + 'hide_y_ticks': True, + 'colours': None, + 'show_image': True + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + + + if not isinstance(data['path'], list): + data['path'] = [data['path']] + + + if not 'image' in data.keys(): + + data['image'] = [None for _ in range(len(data['path']))] + + if not 'weights' in data.keys(): + data['weights'] = [1.0 for _ in range(len(data['path']))] + + if not options['colours']: + options['colours'] = [None for _ in range(len(data['path']))] + + for i, (path, weight, colour) in enumerate(zip(data['path'], data['weights'], options['colours'])): + data['image'][i] = io.read_image(path=path, weight=weight, colour=colour, resize=options['resize']) + + + images = [] + for i, image in enumerate(data['image']): + images.append(image) +# + final_image = np.mean(images, axis=0) / 255 + if len(data['path']) > 1: + data['image'].append(final_image) + + if options['show_image']: + fig, ax = btp.prepare_plot(options) + ax.imshow(final_image) + btp.adjust_plot(fig=fig, ax=ax, options=options) + + return data['image'], fig, ax + + else: + return data['image'], None, None + From 83f4f6a155f04375f1ad6f4b65085feeba764d5f Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 27 Sep 2022 10:21:39 +0200 Subject: [PATCH 294/355] Get HTXRD-plotter to a working state --- nafuma/xrd/plot.py | 68 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 3b3c6bd..e75dc9b 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -24,7 +24,7 @@ def plot_diffractogram(data, options={}): # Update options required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', 'offset_change', - 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'highlight', 'highlight_colours', 'interactive', 'rc_params', 'format_params', 'interactive_session_active'] + 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'highlight', 'highlight_colours', 'interactive', 'rc_params', 'format_params', 'interactive_session_active', 'plot_diff'] default_options = { 'x_vals': '2th', 'y_vals': 'I', @@ -51,6 +51,7 @@ def plot_diffractogram(data, options={}): 'interactive_session_active': False, 'rc_params': {}, 'format_params': {}, + 'plot_diff': False, } if 'offset_y' not in options.keys(): @@ -86,15 +87,25 @@ def plot_diffractogram(data, options={}): if not isinstance(data['wavelength'], list): data['wavelength'] = [data['wavelength'] for _ in range(len(data['path']))] + + + # LOAD DIFFRACTOGRAMS - for index in range(len(data['path'])): - diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) - - data['diffractogram'][index] = diffractogram - data['wavelength'][index] = wavelength + + if 'htxrd' in data.keys() and data['htxrd']: + data['diffractogram'], data['wavelength'] = xrd.io.read_htxrd(data=data, options=options, index=0) + + else: + for index in range(len(data['path'])): + diffractogram, wavelength = xrd.io.read_data(data=data, options=options, index=index) - # FIXME This is a quick fix as the image is not reloaded when passing multiple beamline datasets. Should probably be handled in io? - data['image'] = None + + data['diffractogram'][index] = diffractogram + data['wavelength'][index] = wavelength + + + # FIXME This is a quick fix as the image is not reloaded when passing multiple beamline datasets. Should probably be handled in io? + data['image'] = None # Sets the xlim if this has not been specified @@ -106,7 +117,12 @@ def plot_diffractogram(data, options={}): options['heatmap_loaded'] = True if options['heatmap']: - options['xlim'] = [options['heatmap_xlim'][0], options['heatmap_xlim'][1]] + xlim_start_frac, xlim_end_frac = options['xlim'][0] / data['diffractogram'][0][options['x_vals']].max(), options['xlim'][1] / data['diffractogram'][0][options['x_vals']].max() + options['xlim'] = [options['heatmap_xlim'][0]*xlim_start_frac, options['heatmap_xlim'][1]*xlim_end_frac] + + if options['heatmap_reverse']: + data['heatmap'] = data['heatmap'].iloc[::-1] + data['heatmap_yticklabels'] = data['heatmap_yticklabels'][::-1] # If data was already loaded, only do a check to see if the data is in a list or not, and if not, put it in one. This is because it will be looped over later. else: @@ -177,8 +193,12 @@ def plot_diffractogram(data, options={}): # Limit for when it is assumed that each diffractogram should have its own colour - after 8, the default colour palette is used up and starts a new. # FIXME Should probably allow for more than 8 if wanted - not a priority now - if len(data['path']) < 8: - colours = btp.generate_colours(options['palettes']) + if len(data['path']) <= 8: + if 'colours' in options.keys(): + colours = btp.generate_colours(options['colours'], kind='single') + + else: + colours = btp.generate_colours(options['palettes']) # Generates the colours of a list of scans to highlight is passed. options['highlight'] and options['highlight_colour'] must be of equal length. Entries in highlight can either be a list or a single number, @@ -254,6 +274,8 @@ def plot_diffractogram(data, options={}): options['ylabel'] = options['heatmap.ylabel'] if not options['yunit'] or options['yunit'] == options['diff.yunit']: options['yunit'] = options['heatmap.yunit'] + + ax.tick_params(axis='x', which='minor', bottom=False, top=False) ax.tick_params(axis='y', which='minor', left=False, right=False) @@ -269,7 +291,7 @@ def plot_diffractogram(data, options={}): if options['highlight']: for i, highlight in enumerate(options['highlight']): - if i < len(options['highlight'])-1 or len(options['highlight']) == 1: + if i < len(options['highlight']) or len(options['highlight']) == 1: ax.axhline(y=highlight[1], c=options['highlight_colours'][i], ls='--', lw=0.5) @@ -296,6 +318,14 @@ def plot_diffractogram(data, options={}): options['hide_y_ticklabels'] = True options['hide_y_ticks'] = True + + if options['plot_diff'] and len(data['path']) == 2: + diff = data['diffractogram'][0] + diff['I'] = diff['I'] - data['diffractogram'][1]['I'] + diff['I'] = diff['I'] - 0.5 + + diff.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours)) + # Adjust the plot to make it prettier fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) @@ -418,6 +448,7 @@ def generate_heatmap(data, options={}): options['heatmap_xlim'] = xlims + # Get temperatures if HTXRD-scans scan_numbers = [] @@ -463,6 +494,8 @@ def generate_heatmap(data, options={}): def determine_grid_layout(options): + #aspect_ratio = int(options['format_params']['aspect_ratio'].split(':')[0]) / int(options['format_params']['aspect_ratio'].split(':')[1]) + nrows = 1 if not options['reflections_indices'] else 2 if options['reflections_plot']: @@ -470,7 +503,9 @@ def determine_grid_layout(options): nrows += 1 options['format_params']['nrows'] = nrows - options['format_params']['grid_ratio_height'] = [1 for i in range(nrows-1)]+[10] + + if not 'grid_ratio_height' in options['format_params'].keys(): + options['format_params']['grid_ratio_height'] = [0.6 for i in range(nrows-1)]+[10] return options @@ -794,7 +829,8 @@ def plot_reflection_table(data, reflections_params, ax=None, options={}): 'label': None } - if 'colour' in data.keys(): + + if 'colour' in reflections_params.keys(): options['reflections_colour'] = reflections_params['colour'] if 'min_alpha' in reflections_params.keys(): options['min_alpha'] = reflections_params['min_alpha'] @@ -806,7 +842,6 @@ def plot_reflection_table(data, reflections_params, ax=None, options={}): options['wavelength'] = reflections_params['wavelength'] options = aux.update_options(options=options, required_options=required_options, default_options=default_options) - if not ax: _, ax = btp.prepare_plot(options) @@ -826,8 +861,7 @@ def plot_reflection_table(data, reflections_params, ax=None, options={}): rel_intensity = (intensity / intensities.max())*(1-options['min_alpha']) + options['min_alpha'] colour.append(rel_intensity) colours.append(colour) - - + ax.vlines(x=reflections, ymin=-1, ymax=1, colors=colours, lw=0.5) From f1ec9df9b48a3a33d6ae8a98655eb41d4b9938d9 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Tue, 27 Sep 2022 10:22:06 +0200 Subject: [PATCH 295/355] Add a bunch of stuff to refinement --- nafuma/xrd/refinement.py | 194 ++++++++++++++++++++++++++++++++++----- nafuma/xrd/snippets.json | 35 +++++-- 2 files changed, 202 insertions(+), 27 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 4fe4f78..1b33bc3 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -16,19 +16,25 @@ import nafuma.auxillary as aux def make_initial_inp(data: dict, options={}): - required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'topas_options', 'background', 'capillary', 'th2_offset', 'fit_peak_width', 'start', 'finish', 'exclude'] + required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'instrument', 'topas_options', 'background', 'capillary', 'th2_offset', 'fit_peak_width', 'simple_axial_model', 'TCHZ_Peak_Type', 'start', 'finish', 'exclude', 'magnetic_atoms', 'radiation', 'magnetic_space_group', 'interval'] default_options = { 'filename': 'start.inp', 'overwrite': False, 'save_results': False, 'save_dir': 'results/', + 'instrument': None, + 'radiation': 'synchrotron', + 'magnetic_space_group': None, + 'magnetic_atoms': [], 'include': [], # Any files that should be included with #include 'topas_options': {}, 'background': 7, 'capillary': False, 'th2_offset': False, 'fit_peak_width': False, + 'simple_axial_model': False, + 'TCHZ_Peak_Type': False, 'interval': [None, None], # Start and finish values that TOPAS should refine on. Overrides 'start' and 'finish' 'start': None, # Start value only. Overridden by 'interval' if this is set. 'finish': None, # Finish value only. Overridden by 'interval' if this is set. @@ -72,6 +78,16 @@ def make_initial_inp(data: dict, options={}): write_str(fout=fout, data=data, options=options, index=i) + if 'peak' in data.keys(): + + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + fout.write('\'PEAK PHASES') + fout.write('\n\'------------------------------------------------------------------------------------------------------------------------------------------\n') + + write_peaks(fout=fout, data=data, options=options, index=i) + + + with open(options['filename'], 'r') as fin: lines = [] @@ -124,8 +140,21 @@ def write_xdd(fout, data, options): # Write wavelength and LP-factor fout.write('\n\n') + + if options['instrument'] == 'RECX2': + options['radiation'] = 'MoKa' + for line in snippets['RECX2']: + fout.write('\t'+line+'\n') + + if options['radiation'] == 'synchrotron': + fout.write('\t'+snippets['synchrotron'].format(data['wavelength'])+'\n') + if options['radiation'] == 'neutron': + fout.write('\t'+snippets['neutron'][0].format(data['wavelength']) +'\n') + fout.write('\t'+snippets['neutron'][1] + '\n') + if options['radiation'] == 'MoKa': + for line in snippets['MoKa']: + fout.write('\t'+line+'\n') - fout.write('\t'+snippets['wavelength'].format(data['wavelength'])+'\n') fout.write('\t'+snippets['lp_factor'].format(topas_options['lp_factor'])+'\n') if options['th2_offset']: @@ -190,21 +219,21 @@ def write_params(fout, data, options, index=0): # If start_values is defined: fout.write(f'#ifdef start_values_{label}\n') data['define_statements'].append(f'start_values_{label}') - lpa = f'local !lpa_{label} {a} ;: {a}' - lpb = f'local !lpb_{label} {b} ;: {b}' - lpc = f'local !lpc_{label} {c} ;: {c}' + lpa = f'local !lpa_{label} = {a} ;: {a}' + lpb = f'local !lpb_{label} = {b} ;: {b}' + lpc = f'local !lpc_{label} = {c} ;: {c}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpa, lpb, lpc)) - lpal = f'local !lpal_{label} {alpha} ;: {alpha}' - lpbe = f'local !lpbe_{label} {beta} ;: {beta}' - lpga = f'local !lpga_{label} {gamma} ;: {gamma}' + lpal = f'local !lpal_{label} = {alpha} ;: {alpha}' + lpbe = f'local !lpbe_{label} = {beta} ;: {beta}' + lpga = f'local !lpga_{label} = {gamma} ;: {gamma}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpal, lpbe, lpga)) # Otherwise fout.write('\n') fout.write('#else\n') - lpa = f'local !lpa_{label} {a}' - lpb = f'local !lpb_{label} {b}' - lpc = f'local !lpc_{label} {c}' + lpa = f'local !lpa_{label} = {a} ;: {a}' + lpb = f'local !lpb_{label} = {b} ;: {b}' + lpc = f'local !lpc_{label} = {c} ;: {c}' fout.write('{: <55} {: <55} {: <55}\n'.format(lpa, lpb, lpc)) lpal = f'local !lpal_{label} {alpha}' lpbe = f'local !lpbe_{label} {beta}' @@ -224,6 +253,7 @@ def write_params(fout, data, options, index=0): # WRITE SITE PARAMETERS + mag = [] for site in sites: params = [] @@ -232,13 +262,30 @@ def write_params(fout, data, options, index=0): value = atoms["atoms"][site][attr].split("(")[0] value = value if value != '.' else 0. + + params.append('{: <20} {: <20}'.format(f'local !{attrs[attr]}_{site}_{label}', f' = {value} ;: {value}')) - #fout.write(f'local {attrs[attr]}_{site}_{label}\t\t =\t {value} \t ;= \t\t\t\t\t') + + if options['magnetic_space_group']: + for mag_atom in options['magnetic_atoms']: + if mag_atom in site: + mag.append(site) fout.write('{: <55} {: <55} {: <55} {: <55} {: <55}\n'.format(*params)) fout.write('\n') + if options['magnetic_space_group']: + mag = list(dict.fromkeys(mag)) # remove duplicates + for m in mag: + fout.write('{: <55} {: <55} {: <55}\n'.format( + f'local !ml_x_{m}_{label}_XXXX = 0 ;: 0', + f'local !ml_y_{m}_{label}_XXXX = 0 ;: 0', + f'local !ml_z_{m}_{label}_XXXX = 0 ;: 0' + )) + + fout.write('\n') + fout.write('{: <55} {: <55} {: <55} {: <55}\n'.format( f'local !csgc_{label}_XXXX = 200 ;: 200', f'local !cslc_{label}_XXXX = 200 ;: 200', @@ -277,7 +324,10 @@ def write_str(fout, data, options, index=0): fout.write('\tstr\n') fout.write(f'\t\tphase_name "{atoms["_chemical_name_common"]} ({atoms["_space_group_name_H-M_alt"]})"\n'.replace('\'', '')) - fout.write(f'\t\tspace_group {atoms["_space_group_IT_number"]}\n') + if options['magnetic_space_group']: + fout.write(f'\t\tmag_space_group {options["magnetic_space_group"]}\n') + else: + fout.write(f'\t\tspace_group {atoms["_space_group_IT_number"]}\n') fout.write(f'\t\ta = lpa_{label} ;\n') fout.write(f'\t\tb = lpb_{label} ;\n') fout.write(f'\t\tc = lpc_{label} ;\n') @@ -287,8 +337,11 @@ def write_str(fout, data, options, index=0): fout.write('\n') fout.write(f'\t\tcell_volume\t vol_{label}_XXXX {atoms["_cell_volume"]}\n') fout.write(f'\t\tcell_mass\t mass_{label}_XXXX 1\n') - fout.write(f'\t\tweight_percent\t wp_{label}_XXXX 100\n\n') - fout.write('\n') + fout.write(f'\t\tweight_percent\t wp_{label}_XXXX 100\n\n\n') + if options['simple_axial_model']: + fout.write(f'\t\t{snippets["Simple_Axial_Model"]}\n') + if options['TCHZ_Peak_Type']: + fout.write(f'\t\t{snippets["TCHZ_Peak_Type"]}\n\n\n') fout.write('\t\tscale @ 1.0\n') fout.write('\t\tr_bragg 1.0\n\n') if options['fit_peak_width']: @@ -342,6 +395,11 @@ def write_str(fout, data, options, index=0): fout.write('\t\t{: <9} {: <30}; {: <30}; {: <30}; {: <30}; {: <30};'.format(site, x, y, z, occ, beq)) fout.write('\n') + if options['magnetic_space_group']: + for mag_atom in options['magnetic_atoms']: + if mag_atom in atom: + fout.write('\t\t'+snippets['magnetic_moment_str'].format(atom, label, atom, label, atom, label)+'\n') + if options['save_results']: @@ -364,11 +422,13 @@ def write_output(fout, data, options, index=0): fout.write('#ifdef output\n') - fout.write(f'\t\tOut_Riet({options["save_dir"]}/{filename}_{label}_XXXX_riet.xy)\n') - fout.write(f'\t\tOut_CIF_STR({options["save_dir"]}/{filename}_{label}_XXXX.cif)\n') - fout.write(f'\t\tOut_CIF_ADPs({options["save_dir"]}/{filename}_{label}_XXXX.cif)\n') - fout.write(f'\t\tOut_CIF_Bonds_Angles({options["save_dir"]}/{filename}_{label}_XXXX.cif)\n') - fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file({options["save_dir"]}/{filename}_{label}_XXXX_hkl.dat)\n') + fout.write(f'\t\tOut_Riet({options["save_dir"]}/{filename}_{label}_riet.xy)\n') + fout.write(f'\t\tOut_CIF_STR({options["save_dir"]}/{filename}_{label}.cif)\n') + fout.write(f'\t\tOut_CIF_ADPs({options["save_dir"]}/{filename}_{label}.cif)\n') + fout.write(f'\t\tOut_CIF_Bonds_Angles({options["save_dir"]}/{filename}_{label}.cif)\n') + fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file({options["save_dir"]}/{filename}_{label}_hkl.dat)\n') + if options['magnetic_space_group']: + fout.write(f'\t\tOut_CIF_mag({options["save_dir"]}/{filename}_{label}_magnetic.cif)\n') fout.write('\n') @@ -446,7 +506,26 @@ def write_output(fout, data, options, index=0): fout.write('\n\n') +def write_peaks(fout, data, options, index=0): + import nafuma.xrd as xrd + basedir = os.path.dirname(xrd.__file__) + + with open(os.path.join(basedir, 'snippets.json'), 'r') as snip: + snippets = json.load(snip) + + + if not isinstance(data['peak'], list): + data['peak'] = [data['peak']] + + for peak in data['peak']: + fout.write('\t'+snippets['peak'][0]+'\n') + fout.write('\t\t'+snippets['peak'][1].format(peak)+'\n') + fout.write('\t\t'+snippets['peak'][2]+'\n') + fout.write('\t\t'+snippets['peak'][3]+'\n') + fout.write('\t\t'+snippets['peak'][4]+'\n') + + fout.write('\n\n') def read_cif(path): @@ -600,7 +679,7 @@ def make_big_inp(data: dict, options={}): aux.backup_file(filename=options['output'], backup_dir=options['backup_dir']) - runlist = os.path.join(os.path.dirname(options['output']), 'runlist.txt') + runlist = os.path.abspath(os.path.join(os.path.dirname(options['output']), f'runlist_{os.path.basename(options["output"]).split(".")[0]}.txt')) options['include'].append(runlist) with open(options['template'], 'r') as template: @@ -615,6 +694,8 @@ def make_big_inp(data: dict, options={}): for i, path in enumerate(data['path']): + path = os.path.abspath(path) + s = make_inp_entry(template=template, xdd=path, num=i, options=options) output.write(s) output.write('\n\n') @@ -745,6 +826,7 @@ def make_inp_entry(template: str, xdd: str, num: int, options: dict) -> str: # Replace diffractogram-path s = template.replace(temp_xdd, xdd).replace('XXXX', num_str) + s = s.replace(os.path.basename(temp_xdd).split('.')[0], os.path.basename(xdd).split('.')[0]) return s @@ -867,3 +949,73 @@ def read_results(path): return results + + +def fix_magnetic_cif(path): + with open(path, 'r') as f: + lines = f.readlines() + + for i, line in enumerate(lines): + if '_magnetic_space_group_symop_operation_timereversal' in line: # last line before symmetry operations + j = 1 + line = lines[i + j] + while len(line) > 1: + lines[i+j] = f'{j} ' + line + j += 1 + line = lines[i+j] + + + + with open(path, 'w') as f: + for line in lines: + f.write(line) + + + +def get_magnetic_moment(path): + with open(path, 'r') as f: + lines = f.readlines() + + for i, line in enumerate(lines): + if '_magnetic_atom_site_moment_crystalaxis_mz' in line: # last line before magnetic moments + j = 1 + line = lines[i+j] + + magnetic_moments = {} + while line: + magnetic_moments[line.split()[0]] = (float(line.split()[1]), float(line.split()[2]), float(line.split()[3])) + + j += 1 + + if i+j < len(lines): + line = lines[i+j] + else: + break + + + + return magnetic_moments + + +def get_magnetic_moment_magnitude(magmom): + magnitudes = {} + for site in magmom.keys(): + magnitudes[site] = np.sqrt(magmom[site][0]**2+magmom[site][1]**2+magmom[site][2]**2) + + return magnitudes + + +def strip_labels_from_inp(path, save_path=None): + + if not save_path: + save_path = path + + with open(path, 'r') as fin: + lines = fin.readlines() + + for i, line in enumerate(lines): + lines[i] = line.replace('_XXXX', '') + + with open(save_path, 'w') as fout: + for line in lines: + fout.write(line) diff --git a/nafuma/xrd/snippets.json b/nafuma/xrd/snippets.json index 00266bd..60f169d 100644 --- a/nafuma/xrd/snippets.json +++ b/nafuma/xrd/snippets.json @@ -9,13 +9,36 @@ ], "gauss_fwhm": "gauss_fwhm = Sqrt({} Cos(2 * Th)^4 + {} Cos(2 * Th)^2 + {});", "lp_factor": "LP_Factor({}) 'change the LP correction or lh value if required", - "wavelength": "lam ymin_on_ymax 0.0001 la 1.0 lo {} lh 0.1", - "zero_error": "Zero_Error(zero, 0)", + "synchrotron": "lam ymin_on_ymax 0.0001 la 1.0 lo {} lh 0.1", + "neutron": [ + "lam ymin_on_ymax 0.0001 la 1.0 lo {} lh 0.5", + "neutron_data" + ], + "MoKa":[ + "lam ymin_on_ymax 0.0001", + "la 0.6533 lo 0.7093 lh 0.2695", + "la 0.3467 lo 0.713574 lh 0.2795" + ], + "RECX2": [ + "Rp 280", + "Rs 280" + ], + "zero_error": "Zero_Error(!zero, 0)", "th2_offset": [ - "prm zero \t\t\t0 \t\t\t\tmin = Max(Val - 20 Yobs_dx_at(X1), -100 Yobs_dx_at(X1)); max = Min(Val + 20 Yobs_dx_at(X2), 100 Yobs_dx_at(X2)); del = .01 Yobs_dx_at(X1); val_on_continue 0", - "prm cos_shift \t\t0 \t\t\t\tmin = Val-.8; max = Val+.8; del 0.001", - "prm sin_shift \t\t0 \t\t\t\tmin = Val-.8; max = Val+.8; del 0.001", + "prm !zero\t\t\t= 0 ;: 0 \t\t\t\tmin = Max(Val - 20 Yobs_dx_at(X1), -100 Yobs_dx_at(X1)); max = Min(Val + 20 Yobs_dx_at(X2), 100 Yobs_dx_at(X2)); del = .01 Yobs_dx_at(X1); val_on_continue 0", + "prm !cos_shift\t\t= 0 ;: 0 \t\t\t\tmin = Val-.8; max = Val+.8; del 0.001", + "prm !sin_shift\t\t= 0 ;: 0 \t\t\t\tmin = Val-.8; max = Val+.8; del 0.001", "th2_offset = (zero) + (cos_shift) Cos(Th) + (sin_shift) Sin(Th) ;" ], - "fit_peak_width": "DC1( ad, 0, bd, 0, cd, 0)" + "fit_peak_width": "DC1( ad, 0, bd, 0, cd, 0)", + "TCHZ_Peak_Type": "TCHZ_Peak_Type(pku_1, 0, pkv_1, 0,pkw_1, 0, !pkx_1, 0.0000,pky_1, 0,!pkz_1, 0.0000)", + "Simple_Axial_Model": "Simple_Axial_Model( axial_1, 0)", + "magnetic_moment_str": "mlx = ml_x_{}_{}_XXXX ; \t mly = ml_y_{}_{}_XXXX ; \t mlz = ml_z_{}_{}_XXXX ; \t MM_CrystalAxis_Display( 0, 0, 0)", + "peak": [ + "xo_Is", + "xo @ {}", + "peak_type fp", + "LVol_FWHM_CS_G_L( 1, 0, 0.89, 0,,,@, 2)", + "I @ 35.35632`" + ] } \ No newline at end of file From 4424d3d6c1dffd68eeed1654e82387355ce74871 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:37:14 +0200 Subject: [PATCH 296/355] Remove need for required_options --- nafuma/auxillary.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 4734802..8044914 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -6,10 +6,12 @@ import shutil import time from datetime import datetime -def update_options(options, required_options, default_options): +def update_options(options, default_options, required_options=None): ''' Takes a dictionary of options along with a list of required options and dictionary of default options, and sets all keyval-pairs of options that is not already defined to the default values''' + #FIXME This has been updated so that required_options is not needed. But lots of scripts still passes required_options, so for now it is still accepted, but has a default value and remains unused. Needs to go through all scripts to stop passing of this variable to remove it. - for option in required_options: + + for option in default_options.keys(): if option not in options.keys(): options[option] = default_options[option] @@ -145,4 +147,20 @@ def get_unique(full_list): if not entry in unique_list: unique_list.append(entry) - return unique_list \ No newline at end of file + return unique_list + + +def swap_values(options: dict, key1, key2): + + if not isinstance(key1,list): + key1 = [key1] + if not isinstance(key2,list): + key2 = [key2] + + assert len(key1) == len(key2) + + for k1, k2 in zip(key1, key2): + options[k1], options[k2] = options[k2], options[k1] + + + return options \ No newline at end of file From 3f1d1e4d1f84a4114b784377d1caae4371c1445b Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:37:42 +0200 Subject: [PATCH 297/355] Add inset axes creation and bg drawing --- nafuma/plotting.py | 168 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 136 insertions(+), 32 deletions(-) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 74db433..99b7d7f 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -2,6 +2,11 @@ import nafuma.auxillary as aux import matplotlib.pyplot as plt from matplotlib.ticker import (MultipleLocator) +from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition, BboxPatch, BboxConnector) + +from matplotlib.transforms import TransformedBbox +from matplotlib.patches import Rectangle + import importlib import matplotlib.patches as mpatches from matplotlib.lines import Line2D @@ -33,9 +38,6 @@ def prepare_plot(options={}): format_params = {} - required_format_params = ['single_column_width', 'double_column_width', 'column_type', 'width_ratio', 'aspect_ratio', - 'width', 'height', 'compress_width', 'compress_height', 'upscaling_factor', 'dpi', - 'nrows', 'ncols', 'grid_ratio_height', 'grid_ratio_width'] default_format_params = { 'single_column_width': 8.3, @@ -55,7 +57,7 @@ def prepare_plot(options={}): 'grid_ratio_width': None } - format_params = aux.update_options(format_params, required_format_params, default_format_params) + format_params = aux.update_options(options=format_params, default_options=default_format_params) @@ -96,27 +98,12 @@ def prepare_plot(options={}): def adjust_plot(fig, ax, options): ''' A general function to adjust plot according to contents of the options-dictionary ''' - required_options = [ - 'plot_kind', - 'xlabel', 'ylabel', - 'xunit', 'yunit', - 'hide_x_labels', 'hide_y_labels', - 'hide_x_ticklabels', 'hide_y_ticklabels', - 'hide_x_ticks', 'hide_y_ticks', - 'x_tick_locators', 'y_tick_locators', - 'rotation_x_ticks', 'rotation_y_ticks', - 'xticks', 'yticks', - 'xlim', 'ylim', - 'title', - 'legend', 'legend_position', 'legend_ncol', - 'subplots_adjust', - 'marker_edges', - 'text'] default_options = { 'plot_kind': None, # defaults to None, but should be utilised when requiring special formatting for a particular plot 'xlabel': None, 'ylabel': None, 'xunit': None, 'yunit': None, + 'xlabel_pad': 4.0, 'ylabel_pad': 4.0, 'hide_x_labels': False, 'hide_y_labels': False, # Whether the main labels on the x- and/or y-axes should be hidden 'hide_x_ticklabels': False, 'hide_y_ticklabels': False, # Whether ticklabels on the x- and/or y-axes should be hidden 'hide_x_ticks': False, 'hide_y_ticks': False, # Whether the ticks on the x- and/or y-axes should be hidden @@ -125,30 +112,31 @@ def adjust_plot(fig, ax, options): 'xticks': None, 'yticks': None, # Custom definition of the xticks and yticks. This is not properly implemented now. 'xlim': None, 'ylim': None, # Limits to the x- and y-axes 'title': None, # Title of the plot + 'backgrounds': [], 'legend': False, 'legend_position': ['lower center', (0.5, -0.1)], 'legend_ncol': 1, # Toggles on/off legend. Specifices legend position and the number of columns the legend should appear as. - 'subplots_adjust': [0.1, 0.1, 0.9, 0.9], # Adjustment of the Axes-object within the Figure-object. Fraction of the Figure-object the left, bottom, right and top edges of the Axes-object will start. + 'subplots_adjust': {'left': None, 'right': None, 'top': None, 'bottom': None, 'wspace': None, 'hspace': None}, # Adjustment of the Axes-object within the Figure-object. Fraction of the Figure-object the left, bottom, right and top edges of the Axes-object will start. 'marker_edges': None, 'text': None # Text to show in the plot. Should be a list where the first element is the string, and the second is a tuple with x- and y-coordinates. Could also be a list of lists to show more strings of text. } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) # Set labels on x- and y-axes if not options['hide_y_labels']: if not options['yunit']: - ax.set_ylabel(f'{options["ylabel"]}') + ax.set_ylabel(f'{options["ylabel"]}', labelpad=options['ylabel_pad']) else: - ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]') + ax.set_ylabel(f'{options["ylabel"]} [{options["yunit"]}]', labelpad=options['ylabel_pad']) else: ax.set_ylabel('') if not options['hide_x_labels']: if not options['xunit']: - ax.set_xlabel(f'{options["xlabel"]}') + ax.set_xlabel(f'{options["xlabel"]}', labelpad=options['xlabel_pad']) else: - ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]') + ax.set_xlabel(f'{options["xlabel"]} [{options["xunit"]}]', labelpad=options['xlabel_pad']) else: ax.set_xlabel('') @@ -203,15 +191,17 @@ def adjust_plot(fig, ax, options): - # Create legend + #### DRAW/REMOVE LEGEND #### + # Options: + # 'legend_position': (default ['lower center', (0.5, -0.1)]) - Follows matplotlib's way of specifying legend position + # 'legend_ncol': (default 1) # Number of columns to write the legend in + # Also requires options to contain values in colours, markers and labels. (No defaults) if ax.get_legend(): ax.get_legend().remove() - - if options['legend']: - + if options['legend']: # Make palette and linestyles from original parameters if not options['colours']: colours = generate_colours(palettes=options['palettes']) @@ -250,7 +240,7 @@ def adjust_plot(fig, ax, options): # Adjust where the axes start within the figure. Default value is 10% in from the left and bottom edges. Used to make room for the plot within the figure size (to avoid using bbox_inches='tight' in the savefig-command, as this screws with plot dimensions) - plt.subplots_adjust(left=options['subplots_adjust'][0], bottom=options['subplots_adjust'][1], right=options['subplots_adjust'][2], top=options['subplots_adjust'][3]) + plt.subplots_adjust(**options['subplots_adjust']) # If limits for x- and y-axes is passed, sets these. @@ -261,6 +251,49 @@ def adjust_plot(fig, ax, options): ax.set_ylim(options['ylim']) + #### DRAW BACKGROUNDS #### + # options['backgrounds'] should contain a dictionary or a list of dictionaries. Options to be specified are listed below. + + if options['backgrounds']: + + if not isinstance(options['backgrounds'], list): + options['backgrounds'] = [options['backgrounds']] + + + for background in options['backgrounds']: + default_background_options = { + 'colour': (0,0,0), + 'alpha': 0.2, + 'xlim': list(ax.get_xlim()), + 'ylim': list(ax.get_ylim()), + 'zorder': 0, + 'edgecolour': None, + 'linewidth': None + } + + + background = aux.update_options(options=background, default_options=default_background_options) + + if not background['xlim'][0]: + background['xlim'][0] = ax.get_xlim()[0] + if not background['xlim'][1]: + background['xlim'][1] = ax.get_xlim()[1] + if not background['ylim'][0]: + background['ylim'][0] = ax.get_ylim()[0] + if not background['ylim'][1]: + background['ylim'][1] = ax.get_ylim()[1] + + ax.add_patch(Rectangle( + xy=(background['xlim'][0], background['ylim'][0]), # Anchor point + width=background['xlim'][1]-background['xlim'][0], # Width of background + height=background['ylim'][1]-background['ylim'][0], # Height of background + zorder=background['zorder'], # Placement in stack + facecolor=(background['colour'][0], background['colour'][1], background['colour'][2], background['alpha']), # Colour + edgecolor=background['edgecolour'], # Edgecolour + linewidth=background['linewidth']) # Linewidth + ) + + # Add custom text if options['text']: @@ -360,4 +393,75 @@ def generate_colours(palettes, kind=None): colour_cycle = itertools.cycle(colour_collection) - return colour_cycle \ No newline at end of file + return colour_cycle + + + +def prepare_inset_axes(parent_ax, options): + + default_options = { + 'hide_inset_x_labels': False, # Whether x labels should be hidden + 'hide_inset_x_ticklabels': False, + 'hide_inset_x_ticks': False, + 'rotation_inset_x_ticks': 0, + 'hide_inset_y_labels': False, # whether y labels should be hidden + 'hide_inset_y_ticklabels': False, + 'hide_inset_y_ticks': False, + 'rotation_inset_y_ticks': 0, + 'inset_x_tick_locators': [100, 50], # Major and minor tick locators + 'inset_y_tick_locators': [10, 5], + 'inset_position': [0.1,0.1,0.3,0.3], + 'inset_bounding_box': [0,0,0.1, 0.1], + 'inset_marks': [None, None], + 'legend_position': ['upper center', (0.20, 0.90)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively, + 'connecting_corners': [1,2] + } + + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + + # Create a set of inset Axes: these should fill the bounding box allocated to + # them. + inset_ax = plt.axes(options["inset_bounding_box"]) + # Manually set the position and relative size of the inset axes within ax1 + ip = InsetPosition(parent_ax, options['inset_position']) + inset_ax.set_axes_locator(ip) + + if options['connecting_corners'] and len(options["connecting_corners"]) == 2: + connect_inset(parent_ax, inset_ax, loc1a=options['connecting_corners'][0], loc2a=options['connecting_corners'][1], loc1b=options['connecting_corners'][0], loc2b=options['connecting_corners'][1], fc='none', ec='black') + elif options['connecting_corners'] and len(options['connecting_corners']) == 4: + connect_inset(parent_ax, inset_ax, loc1a=options['connecting_corners'][0], loc2a=options['connecting_corners'][1], loc1b=options['connecting_corners'][2], loc2b=options['connecting_corners'][3], fc='none', ec='black', ls='--') + + inset_ax.xaxis.set_major_locator(MultipleLocator(options['inset_x_tick_locators'][0])) + inset_ax.xaxis.set_minor_locator(MultipleLocator(options['inset_x_tick_locators'][1])) + + + inset_ax.yaxis.set_major_locator(MultipleLocator(options['inset_y_tick_locators'][0])) + inset_ax.yaxis.set_minor_locator(MultipleLocator(options['inset_y_tick_locators'][1])) + + + + + return inset_ax + + + + +def connect_inset(parent_axes, inset_axes, loc1a=1, loc1b=1, loc2a=2, loc2b=2, **kwargs): + rect = TransformedBbox(inset_axes.viewLim, parent_axes.transData) + + pp = BboxPatch(rect, fill=False, **kwargs) + parent_axes.add_patch(pp) + + p1 = BboxConnector(inset_axes.bbox, rect, loc1=loc1a, loc2=loc1b, **kwargs) + inset_axes.add_patch(p1) + p1.set_clip_on(False) + p2 = BboxConnector(inset_axes.bbox, rect, loc1=loc2a, loc2=loc2b, **kwargs) + inset_axes.add_patch(p2) + p2.set_clip_on(False) + + return pp, p1, p2 + + + From f72bd4e77f1238383330e406e58c6ce5a1035fec Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:38:00 +0200 Subject: [PATCH 298/355] Rewrite plot_pdos --- nafuma/dft/electrons.py | 141 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 130 insertions(+), 11 deletions(-) diff --git a/nafuma/dft/electrons.py b/nafuma/dft/electrons.py index d96d2ad..8ea1064 100644 --- a/nafuma/dft/electrons.py +++ b/nafuma/dft/electrons.py @@ -242,7 +242,136 @@ def integrate_coop(coopcar, interval=None, r=None, scale=None, interactions=None -def plot_partial_dos(data: dict, options={}): +def plot_pdos(data: dict, options={}): + + default_options = { + 'xlabel': 'Energy', 'xunit': 'eV', 'xlim': None, 'x_tick_locators': None, + 'ylabel': 'Partial density of states', 'yunit': 'arb.u.', 'ylim': None, 'y_tick_locators': None, + 'mark_fermi_level': True, # Adds a dashed line to mark the Fermi-level + 'flip_axes': False, # Flips x- and y-axes + 'plot_indices': [], # List which indices to plot. If options["sum_atoms"] == True, this needs to be a list of lists, each specifying the index of a given atom + 'plot_atoms': [], # List of which atoms to plot. Only used if options["sum_atoms"] == True. + 'plot_orbitals': [], # List of which orbitals to plot. If options["sum_atoms"] == True, this needs to be a list of lists, each specifying the orbitals of a given atom + 'atom_colours': [], # Colours of each atom. Should be a colour for each atom, only in use if options["sum_atoms"] == True. + 'orbital_colours': [], # Colours of each orbital. The list should always correspond to the shape of options["plot_orbitals"]. + 'fill': False, + 'fig': None, # Matplotlib Figure object + 'ax': None, # Matplotlib Axes object + } + + options = aux.update_options(options=options, default_options=default_options) + + if 'axes_flipped' not in options.keys(): + options['axes_flipped'] = False + + data = dft.io.read_pdos(data=data, options=options) + + + + if not options['fig'] and not options['ax']: + fig, ax = btp.prepare_plot(options=options) + else: + fig, ax = options['fig'], options['ax'] + + + # If options['sum_atoms'] == True + if isinstance(data['pdos'], dict): + + # Populate the plot_atoms and plot_orbitals lists if they are not passed. Defaults to showing everything + if not options['plot_atoms']: + options['plot_atoms'] = data['atoms']['specie'] + + if not options['plot_orbitals']: + for atom in options['plot_atoms']: + options['plot_orbitals'].append([]) + + # This is to fill in each orbital list for each atom. This is in case options['plot_orbitals'] is passes, but one or more of the atoms lack colours + for i, atom in enumerate(options['plot_atoms']): + if not options['plot_orbitals'] or not options['plot_orbitals'][i]: + options['plot_orbitals'][i] = [orbital for orbital in data['pdos'][atom].columns if 'Energy' not in orbital] + + # Populate the atom_colours and orbital_colours. Defaults to same colour for all orbitals of one specie. + if not options['atom_colours']: + options['palettes'] = [('qualitative', 'Dark2_8')] + colour_cycle = generate_colours(options=options) + + for atom in options['plot_atoms']: + options['atom_colours'].append(next(colour_cycle)) + + if not options['orbital_colours']: + for i, atom in enumerate(options['plot_orbitals']): + options['orbital_colours'].append([]) # Make list for specific atom + for orbital in atom: + options['orbital_colours'][i].append(options['atom_colours'][i]) + + + for i, atom in enumerate(options['plot_atoms']): + + if not options['plot_orbitals'] or not options['plot_orbitals'][i]: + options['plot_orbitals'][i] = [orbital for orbital in data['pdos'][atom].columns if 'Energy' not in orbital] + + x = 'Energy' + y = options['plot_orbitals'][i] + + if options['flip_axes']: + for j, orbital in enumerate(options['plot_orbitals'][i]): + + if options['fill']: + ax.fill_betweenx(y=data['pdos'][atom]['Energy'], x1=data['pdos'][atom][orbital], x2=0, color=options['orbital_colours'][i][j], alpha=0.5, ec=(0,0,0,0)) + else: + ax.plot(data['pdos'][atom][orbital], data['pdos'][atom]['Energy'], color=options['orbital_colours'][i][j]) + + + + else: + data['pdos'][atom].plot(x=x, y=y, color=options['orbital_colours'][i], ax=ax) + + #print(options['plot_orbitals'], options['orbital_colours']) + + + if options['flip_axes']: + + if not options['axes_flipped']: + options = aux.swap_values(options=options, + key1=['xlim', 'xunit', 'xlabel', 'x_tick_locators'], + key2=['ylim', 'yunit', 'ylabel', 'y_tick_locators'] + ) + + options['axes_flipped'] = True # + + ax.axvline(x=0, c='black') + + if options['mark_fermi_level']: + ax.axhline(y=0, c='black', ls='--') + + + else: + ax.axhline(y=0, c='black') + ax.axvline(x=0, c='black', ls='--') + + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + + + + #elif isinstance(data['pdos'], list): + # if not options['plot_atoms']: + # options['plot_atoms'] = data['atoms']['specie'] + # + # if not options['plot_indices']: + # for plot_specie in options['plot_atoms']: + # for i, doscar_specie in enumerate(data['atoms']['specie']): + # if plot_specie == doscar_specie: + # options['plot_indices'].append([k for k in range(data['atoms']['number'][i])]) + + + return None + + + + + +def plot_partial_dos_legacy(data: dict, options={}): required_options = ['atoms', 'orbitals', 'up', 'down', 'sum_atoms', 'collapse_spin', 'sum_orbitals', 'palettes', 'colours', 'fig', 'ax'] @@ -1080,16 +1209,6 @@ def scale_figure(options, width, height): -def swap_values(dict, key1, key2): - - key1_val = dict[key1] - dict[key1] = dict[key2] - dict[key2] = key1_val - - return dict - - - def generate_colours(options: dict): if not isinstance(options['palettes'], list): From 0e7a566f0179c74f188aba9f8861b9f4e959911e Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:38:25 +0200 Subject: [PATCH 299/355] Rewrite read_pdos --- nafuma/dft/io.py | 293 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 276 insertions(+), 17 deletions(-) diff --git a/nafuma/dft/io.py b/nafuma/dft/io.py index 6f0a60e..e09de45 100644 --- a/nafuma/dft/io.py +++ b/nafuma/dft/io.py @@ -1,6 +1,7 @@ import pandas as pd import matplotlib.pyplot as plt import numpy as np +from scipy.signal import savgol_filter import warnings @@ -42,15 +43,18 @@ def get_atoms(poscar): atoms = lines[5].split() atom_num = lines[6].split() - atom_num = [int(num) for num in atom_num] - atoms_dict = {} - for ind, atom in enumerate(atoms): - atoms_dict[atom] = atom_num[ind] + atoms_list = make_atoms_list(atoms, atom_num) - return atoms, atom_num, atoms_dict + atoms_info = { + 'specie': atoms, # list of specie in sequential order + 'number': atom_num, # list of number of per specie in sequential order + 'list': atoms_list # list of every individual atom in sequential order + } + + return atoms_info def get_fermi_level(doscar='DOSCAR', headerlines=6): @@ -276,6 +280,269 @@ def read_dos(path, flip_down=True): def read_pdos(data: dict, options={}): + ''' data-dictionary should be structured like this: + data["path"] - dictionary with path to POSCAR/CONTCAR mapped to key 'poscar' and path to DOSCAR in key 'doscar'. ''' + + default_options = { + 'sum_atoms': False, + 'sum_orbitals': False, + 'adjust': False, # Manually adjust the energy scale if the automatic Fermi-level detection didn't work. + 'normalise': False, + } + + options = aux.update_options(options=options, default_options=default_options) + + # Grab some metadata + data['atoms'] = get_atoms(data['path']['poscar']) + data['info'] = get_doscar_information(data['path']['doscar']) + + + # Read the data + with open(data['path']['doscar'], 'r') as f: + doscar_raw = f.readlines() + + # Read basis functions - this will only yield a non-empty list if DOSCAR is generated by LOBSTER. It will only contain columns for the used basis set, so to be able to make proper column headers, this information is needed. + data['basis_functions'] = get_basis_functions(doscar_raw) + + # If DOSCAR is from VASP and not LOBSTER, the basis functions will not be listed in DOSCAR (but all columns will be present) + if not data['basis_functions']: + data['basis_functions'] = [None for _ in data['atoms']['list']] + + + data['pdos'] = [] + for i, (atom, basis_functions) in enumerate(zip(data['atoms']['list'], data['basis_functions'])): + + # Define line to start read-in: (1+i) to skip total DOS in the start, plus every previous PDOS, including the headerline. Then the initial 6 header lines in the start of the file. + start = int( (1 + i) * ( data['info']['NEDOS'] + 1 ) + 6 ) + + pdos_atom = [] + for j in range(0,data['info']['NEDOS']): + pdos_atom.append(doscar_raw[start+j].split()) + + # Convert the data to a DataFrame and convert to float + pdos_atom = pd.DataFrame(pdos_atom) + pdos_atom = pdos_atom.astype(float) + + + # If DOSCAR was written by VASP, set the standard columns to the basis_functions variable + # FIXME This does not allow for f-orbitals. Could do a check vs. shape of pdos_atom to determine + if not basis_functions: + basis_functions = ['s', 'p1', 'p2', 'p3', '2p1', '2p2', '2p3', 'd1', 'd2', 'd3', 'd4', 'd5'] + + + # Split the basis functions into up and down spin channels + if data['info']['spin_polarised']: + columns = ['Energy'] + for function in basis_functions: + columns.append(function+'_up') + columns.append(function+'_down') + + else: + columns = ['Energy'] + basis_functions + + # Set the new column names + pdos_atom.columns = columns + + if options['adjust']: + pdos_atom['Energy'] -= options['adjust'] + + if options['smooth']: + pdos_atom = smoothing(pdos=pdos_atom, options=options) + + if options['normalise']: + orbitals = [orbital for orbital in columns if 'Energy' not in orbital] + for k, specie in enumerate(data['atoms']['specie']): + if specie == atom: + index = k + + for orbital in orbitals: + pdos_atom[orbital] = pdos_atom[orbital] / data['atoms']['number'][index] + + # Make total columns + if data['info']['spin_polarised']: + up_functions = [orbital for orbital in columns if '_up' in orbital] + down_functions = [orbital for orbital in columns if '_down' in orbital] + + pdos_atom['total_up'] = pdos_atom[up_functions].sum(axis=1) + pdos_atom['total_down'] = pdos_atom[down_functions].sum(axis=1) + pdos_atom['total'] = pdos_atom[['total_up', 'total_down']].sum(axis=1) + + # Flip all the values for spin down + down_functions.append('total_down') + for orbital in down_functions: + pdos_atom[orbital] = (-1)*pdos_atom[orbital] + + data['pdos'].append(pdos_atom) + + if options['sum_atoms']: + data['pdos'] = sum_atoms(data, options) + if options['sum_orbitals']: + data['pdos'] = sum_orbitals(data, options) + + return data + + +def sum_atoms(data: dict, options={}): + ''' Needs to have read data through read_pdos() ''' + + default_options = { + + } + + options = aux.update_options(options=options, default_options=default_options) + + + total_atoms = 0 + + # Sort all the DataFrames into a dictionary + sorted_dfs = {} + for i, (specie, number) in enumerate(zip(data['atoms']['specie'], data['atoms']['number'])): + + sorted_dfs[specie] = [] + + for j in range(number): + index = j if i == 0 else j + total_atoms + + sorted_dfs[specie].append(data['pdos'][index]) + + total_atoms += number + + # Sum all the DataFrames for each specie + summed_dfs = {} + for i, (specie, dfs) in enumerate(sorted_dfs.items()): + for j, df in enumerate(dfs): + if j > 0: + dfs[0] += df + + dfs[0]['Energy'] = dfs[0]['Energy'] / data['atoms']['number'][i] + summed_dfs[specie] = dfs[0] + + return summed_dfs + + +def sum_orbitals(data: dict, options={}): + + shells = [ + '1s', '2s', '3s', '4s', '5s', '6s', + '2p', '3p', '4p', '5p', '6p', + '3d', '4d', '5d', '6d', + '4f', '5f', '6f' + ] + + summed_dfs = [] + + # If sum_atoms() is already called + if isinstance(data['pdos'], dict): + summed_dfs = {} + + for specie, specie_pdos in data['pdos'].items(): + orbitals = [orbital for orbital in specie_pdos.columns if 'Energy' not in orbital] + new_pdos = pd.DataFrame(specie_pdos['Energy']) + + for shell in shells: + if data['info']['spin_polarised']: + sub_columns_up = [orbital for orbital in orbitals if orbital.startswith(shell) and orbital.endswith('_up')] + sub_columns_down = [orbital for orbital in orbitals if orbital.startswith(shell) and orbital.endswith('_down')] + + new_pdos[shell+'_up'] = data['pdos'][specie][sub_columns_up].sum(axis=1) + new_pdos[shell+'_down'] = data['pdos'][specie][sub_columns_down].sum(axis=1) + + else: + sub_columns = [orbital for orbital in orbitals if orbital.startswith(shell)] + + new_pdos[shell] = data['pdos'][specie][sub_columns].sum(axis=1) + new_pdos = new_pdos.loc[:, (new_pdos != 0).any(axis=0)] + summed_dfs[specie] = new_pdos + + + else: + for pdos in data['pdos']: + orbitals = [orbital for orbital in pdos.columns if 'Energy' not in orbital] + new_pdos = pd.DataFrame(pdos['Energy']) + for shell in shells: + if data['info']['spin_polarised']: + sub_columns_up = [orbital for orbital in orbitals if orbital.startswith(shell) and orbital.endswith('_up')] + sub_columns_down = [orbital for orbital in orbitals if orbital.startswith(shell) and orbital.endswith('_down')] + new_pdos[shell+'_up'] = pdos[sub_columns_up].sum(axis=1) + new_pdos[shell+'_down'] = pdos[sub_columns_down].sum(axis=1) + else: + sub_columns = [orbital for orbital in orbitals if orbital.startswith(shell)] + new_pdos[shell] = pdos[sub_columns].sum(axis=1) + new_pdos = new_pdos.loc[:, (new_pdos != 0).any(axis=0)] + summed_dfs.append(new_pdos) + + + return summed_dfs + + +def smoothing(pdos: pd.DataFrame, options={}): + ' Smoothes the data using the Savitzky-Golay filter. This is the only algorithm at this moment. ' + + default_options = { + 'smooth_window_length': 3, # Determines the window length of smoothing that the savgol-filter uses for smoothing + 'smooth_polyorder': 2, # Determines the order of the polynomial used in the smoothing algorithm + 'smooth_algorithm': 'savgol', # At the present, only Savitzky-Golay filter is implemented. Add Gaussian and Boxcar later. + } + + options = aux.update_options(options=options, default_options=default_options) + + # Initialise new DataFrame with correct x-values + pdos_smooth = pd.DataFrame(pdos['Energy']) + + orbitals = [orbital for orbital in pdos.columns if 'Energy' not in orbital] + + if options['smooth_algorithm'] == 'savgol': + + for orbital in orbitals: + pdos_smooth.insert(1, orbital, savgol_filter(pdos[orbital], options['smooth_window_length'], options['smooth_polyorder'])) + + return pdos_smooth + + + +def get_basis_functions(doscar): + + basis_functions = [] + for line in doscar: + if 'Z=' in line: + basis_functions.append(line.split(';')[-1].split()) + + + return basis_functions + + + +def get_doscar_information(path): + ''' Reads information from the DOSCAR''' + + kind = 'LOBSTER' if 'LOBSTER' in linecache.getline(path, 5) else 'VASP' + dos_info_raw = linecache.getline(path, 6).split() + spin_polarised = True if len(linecache.getline(path, 7).split()) == 5 else False + + + dos_info = { + 'ENMIN': float(dos_info_raw[0]), + 'ENMAX': float(dos_info_raw[1]), + 'NEDOS': int(dos_info_raw[2]), + 'EFERMI': float(dos_info_raw[3]), + 'spin_polarised': spin_polarised, + 'kind': kind + } + + return dos_info + + +def make_atoms_list(atoms, number_of_atoms): + + atoms_list = [] + for atom, num in zip(atoms, number_of_atoms): + atoms_list = atoms_list + [atom for _ in range(num)] + + return atoms_list + + + +def read_pdos_legacy(data: dict, options={}): required_options = ['flip_down', 'sum_atoms', 'sum_orbitals', 'collapse_spin', 'adjust', 'manual_adjust', 'normalise', 'normalise_unit_cell', 'normalisation_factor'] @@ -319,10 +586,12 @@ def read_pdos(data: dict, options={}): # Define line to start reading start = int((1 + ind) * dos_info["NEDOS"] + 6 + (ind * 1)) - - add_d_orbitals = False add_p_orbitals = False + + + + # Check if d-orbitals are included (if DOSCAR comes from LOBSTER), otherwise they will have to be added to make the DataFrames the same size if dos_info['kind'] == 'LOBSTER': # Extract basis sets from each atom @@ -532,17 +801,7 @@ def read_pdos(data: dict, options={}): return pdos_full, dos_info -def get_doscar_information(path): - ''' Reads information from the DOSCAR''' - kind = 'LOBSTER' if 'LOBSTER' in linecache.getline(path, 5) else 'VASP' - dos_info_raw = linecache.getline(path, 6).split() - spin_polarised = True if len(linecache.getline(path, 7).split()) == 5 else False - - - dos_info = {'ENMIN': float(dos_info_raw[0]), 'ENMAX': float(dos_info_raw[1]), 'NEDOS': int(dos_info_raw[2]), 'EFERMI': float(dos_info_raw[3]), 'spin_polarised': spin_polarised, 'kind': kind} - - return dos_info #def get_bader_charges(poscar='POSCAR', acf='ACF.dat'): From 4ccc7421f743206014e83525db6a8d819a21cad2 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:38:46 +0200 Subject: [PATCH 300/355] Small changes to inset --- nafuma/dft/phonons.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nafuma/dft/phonons.py b/nafuma/dft/phonons.py index c62b560..0f1454f 100644 --- a/nafuma/dft/phonons.py +++ b/nafuma/dft/phonons.py @@ -1571,9 +1571,6 @@ def prettify_thermal_plot_old(fig, ax, options, colour_cycle=('qualitative', 'Da def prepare_inset_axes(parent_ax, options): - required_options = ['hide_inset_x_labels','hide_inset_x_ticklabels', 'hide_inset_x_ticks', 'rotation_inset_x_ticks', 'hide_inset_y_labels', 'hide_inset_y_ticklabels', 'hide_inset_y_ticks', 'rotation_inset_y_ticks', - 'inset_x_tick_locators', 'inset_y_tick_locators', 'inset_position', 'legend_position'] - default_options = { 'hide_inset_x_labels': False, # Whether x labels should be hidden 'hide_inset_x_ticklabels': False, @@ -1586,11 +1583,12 @@ def prepare_inset_axes(parent_ax, options): 'inset_x_tick_locators': [100, 50], # Major and minor tick locators 'inset_y_tick_locators': [10, 5], 'inset_position': [0.1,0.1,0.3,0.3], - 'legend_position': ['upper center', (0.20, 0.90)] # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'legend_position': ['upper center', (0.20, 0.90)], # the position of the legend passed as arguments to loc and bbox_to_anchor respectively + 'connecting_corners': [1,2] } - options = update_options(options=options, required_options=required_options, default_options=default_options) + options = update_options(options=options, required_options=default_options.keys(), default_options=default_options) # Create a set of inset Axes: these should fill the bounding box allocated to @@ -1600,7 +1598,7 @@ def prepare_inset_axes(parent_ax, options): ip = InsetPosition(parent_ax, options['inset_position']) inset_ax.set_axes_locator(ip) - mark_inset(parent_ax, inset_ax, loc1=2, loc2=4, fc='none', ec='black') + mark_inset(parent_ax, inset_ax, loc1a=2, loc2a=4, loc1b=1, loc2b=2, fc='none', ec='black') inset_ax.xaxis.set_major_locator(MultipleLocator(options['inset_x_tick_locators'][0])) inset_ax.xaxis.set_minor_locator(MultipleLocator(options['inset_x_tick_locators'][1])) From 3998005334e05114cf7316c25e10b8a8d6f85e65 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:39:09 +0200 Subject: [PATCH 301/355] Updates to EDS-scripts --- nafuma/eds/io.py | 34 +++++++++++++++++++++++++++++++--- nafuma/eds/plot.py | 19 ++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/nafuma/eds/io.py b/nafuma/eds/io.py index 8f269d6..8d1b34f 100644 --- a/nafuma/eds/io.py +++ b/nafuma/eds/io.py @@ -2,13 +2,19 @@ from PIL import Image import numpy as np import cv2 -def read_image(path, weight=None, colour=None, resize=None): +def read_image(path, weight=None, colour=None, crop=None, resize=None, brightness=None): img = np.array(Image.open(path)) if colour is not None: img = change_colour(img, colour) + if brightness is not None: + img = increase_brightness(img, increase=brightness) + + if crop is not None: + img = crop_image(img, crop) + if resize is not None: img = resize_image(img, resize) @@ -20,7 +26,6 @@ def read_image(path, weight=None, colour=None, resize=None): def scale_image(image, factor): - for i in range(0,image.shape[0]): for j in range(0, image.shape[1]): image[i][j][0] = image[i][j][0]*factor @@ -30,7 +35,7 @@ def scale_image(image, factor): return image -def resize_image(image, factor): +def crop_image(image, factor): y, x = image.shape[0:2] @@ -43,6 +48,29 @@ def resize_image(image, factor): return res +def resize_image(image, factor): + + y, x = image.shape[0:2] + + new_y, new_x = int(y*factor), int(x*factor) + + res = cv2.resize(image, dsize=(new_x, new_y), interpolation=cv2.INTER_CUBIC) + + return res + + +def increase_brightness(image, brightness): + + for i in range(0,image.shape[0]): + for j in range(0, image.shape[1]): + image[i][j][0] = image[i][j][0]+brightness + image[i][j][1] = image[i][j][1]+brightness + image[i][j][2] = image[i][j][2]+brightness + + + return image + + def add_images(image1, image2): assert image1.shape == image2.shape diff --git a/nafuma/eds/plot.py b/nafuma/eds/plot.py index 9e634e4..7726f75 100644 --- a/nafuma/eds/plot.py +++ b/nafuma/eds/plot.py @@ -15,7 +15,12 @@ def show_image(data, options={}): 'hide_x_ticks': True, 'hide_y_ticks': True, 'colours': None, - 'show_image': True + 'brightness': None, + 'show_image': True, + 'resize': None, + 'crop': None, + 'ax': None, + 'fig': None, } options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) @@ -37,7 +42,7 @@ def show_image(data, options={}): options['colours'] = [None for _ in range(len(data['path']))] for i, (path, weight, colour) in enumerate(zip(data['path'], data['weights'], options['colours'])): - data['image'][i] = io.read_image(path=path, weight=weight, colour=colour, resize=options['resize']) + data['image'][i] = io.read_image(path=path, weight=weight, colour=colour, resize=options['resize'], crop=options['crop']) images = [] @@ -45,11 +50,19 @@ def show_image(data, options={}): images.append(image) # final_image = np.mean(images, axis=0) / 255 + if options['brightness']: + final_image = io.increase_brightness(final_image, brightness=options['brightness']) + if len(data['path']) > 1: data['image'].append(final_image) + if options['show_image']: - fig, ax = btp.prepare_plot(options) + if not options['fig'] and not options['ax']: + fig, ax = btp.prepare_plot(options) + else: + fig, ax = options['fig'], options['ax'] + ax.imshow(final_image) btp.adjust_plot(fig=fig, ax=ax, options=options) From 65de5ecf45849e5f5af8963b5f1ca5722893e14b Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:39:28 +0200 Subject: [PATCH 302/355] Expand Battsmall functionality --- nafuma/electrochemistry/io.py | 46 ++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 6cf462f..d46a1d5 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -144,6 +144,9 @@ def process_batsmall_data(df, options=None): set_units(options) options['old_units'] = get_old_units(df, options) + + df = add_columns(df=df, options=options) # adds columns to the DataFrame if active material weight and/or molecular weight has been passed in options + df = unit_conversion(df=df, options=options) @@ -177,6 +180,9 @@ def process_batsmall_data(df, options=None): if chg_df.empty and dchg_df.empty: continue + chg_df['reaction_coordinate'] = chg_df['time'] * np.abs(chg_df['current'].mean()) + dchg_df['reaction_coordinate'] = dchg_df['time'] * np.abs(dchg_df['current'].mean()) + if options['reverse_discharge']: max_capacity = dchg_df['capacity'].max() dchg_df['capacity'] = np.abs(dchg_df['capacity'] - max_capacity) @@ -480,6 +486,7 @@ def process_biologic_data(df, options=None): def add_columns(df, options): + from . import unit_tables if options['kind'] == 'neware': @@ -513,6 +520,28 @@ def add_columns(df, options): df["IonsExtracted"] = (df["SpecificCapacity({}/mg)".format(options['old_units']['capacity'])]*options['molecular_weight'])*1000/f + + if options['kind'] == 'batsmall': + if options['active_material_weight']: + + + active_material_weight = options['active_material_weight'] * unit_tables.mass()['mg'].loc[options['units']['mass']] + capacity = options['old_units']['capacity'] + + df[f'Capacity [{options["old_units"]["capacity"]}]'] = df[f'C [{options["old_units"]["capacity"]}/{options["old_units"]["mass"]}]'] * active_material_weight + + if options['molecular_weight']: + faradays_constant = 96485.3365 # [F] = C mol^-1 = As mol^-1 + seconds_per_hour = 3600 # s h^-1 + f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 + + molecular_weight = options['molecular_weight'] * unit_tables.mass()['g'].loc[options['units']['mass']] + df["IonsExtracted"] = (df[f'C [{options["old_units"]["capacity"]}/{options["old_units"]["mass"]}]'] * molecular_weight)/f + + + #df['reaction_coordinate'] = (df[f'TT [{options["old_units"]["time"]}]'] * unit_tables.time()[options['old_units']["time"]].loc["h"]) / np.abs(df[f'I [{options["old_units"]["current"]}]'] * unit_tables.current()[options['old_units']["current"]].loc['A']) + + return df @@ -545,7 +574,22 @@ def unit_conversion(df, options): df["I [{}]".format(options['old_units']["current"])] = df["I [{}]".format(options['old_units']["current"])] * unit_tables.current()[options['old_units']["current"]].loc[options['units']['current']] df["C [{}/{}]".format(options['old_units']["capacity"], options['old_units']["mass"])] = df["C [{}/{}]".format(options['old_units']["capacity"], options['old_units']["mass"])] * (unit_tables.capacity()[options['old_units']["capacity"]].loc[options['units']["capacity"]] / unit_tables.mass()[options['old_units']["mass"]].loc[options['units']["mass"]]) - df.columns = ['time', 'voltage', 'current', 'count', 'specific_capacity', 'comment'] + columns = ['time', 'voltage', 'current', 'count', 'specific_capacity', 'comment'] + + # Add column labels for specific capacity and ions if they exist + if f'Capacity [{options["old_units"]["capacity"]}]' in df.columns: + df[f'Capacity [{options["old_units"]["capacity"]}]'] = df[f'Capacity [{options["old_units"]["capacity"]}]'] * unit_tables.capacity()[options['old_units']['capacity']].loc[options['units']['capacity']] + columns.append('capacity') + + if 'IonsExtracted' in df.columns: + columns.append('ions') + + #columns.append('reaction_coordinate') + + df.columns = columns + + + if options['kind'] == 'neware': From 11592301fed7c623b197612708e297dd88471e53 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:39:37 +0200 Subject: [PATCH 303/355] Allow passing of own Axes-object --- nafuma/electrochemistry/plot.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 8fbee49..f8ef088 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -68,7 +68,12 @@ def plot_gc(data, options=None): if options['show_plot']: # Prepare plot - fig, ax = btp.prepare_plot(options=options) + + if not options['fig'] and not options['ax']: + fig, ax = btp.prepare_plot(options=options) + else: + fig, ax = options['fig'], options['ax'] + for i, cycle in enumerate(data['cycles']): if i in options['which_cycles']: if options['charge']: From 9e46504ac9435474258bf860ebb73b64d46974f8 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:39:55 +0200 Subject: [PATCH 304/355] Initial add of PPMS-analysis module --- nafuma/ppms/__init__.py | 1 + nafuma/ppms/io.py | 120 ++++++++++++++++++++++++++++++++++++++++ nafuma/ppms/plot.py | 0 3 files changed, 121 insertions(+) create mode 100644 nafuma/ppms/__init__.py create mode 100644 nafuma/ppms/io.py create mode 100644 nafuma/ppms/plot.py diff --git a/nafuma/ppms/__init__.py b/nafuma/ppms/__init__.py new file mode 100644 index 0000000..e0e052c --- /dev/null +++ b/nafuma/ppms/__init__.py @@ -0,0 +1 @@ +from . import io, plot \ No newline at end of file diff --git a/nafuma/ppms/io.py b/nafuma/ppms/io.py new file mode 100644 index 0000000..aab94eb --- /dev/null +++ b/nafuma/ppms/io.py @@ -0,0 +1,120 @@ +import pandas as pd +import numpy as np + +import nafuma.auxillary as aux + +def read_data(path, options={}): + + index = find_start(path) + + df = pd.read_csv(path, skiprows=index+1) + mask = df.loc[df['Comment'].notna()] + + df = df[['Comment', 'Time Stamp (sec)', 'Temperature (K)', 'Magnetic Field (Oe)', + 'DC Moment (emu)', 'DC Std. Err. (emu)', 'DC Quad. Moment (emu)', + 'AC=1 DC=2 Locate=3', 'Max. Field (Oe)', 'Pressure (Torr)', 'Temp. Status (code)', + ]] + + new_columns = ['Comment', 'Time', 'Temperature', 'Magnetic_Field', + 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', + 'Status', 'Max_Field', 'Pressure', 'Temperature_Status'] + + df.columns = new_columns + + df[['Temperature', 'Magnetic_Field', 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Max_Field', 'Pressure']] = df[['Temperature', 'Magnetic_Field', 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Max_Field', 'Pressure']].astype(float) + + df = df.loc[df['DC_Std_Err'] < 0.001] + + if all([option in options.keys() for option in ['molar_mass', 'sample_mass']]): + df = calculate_emu_per_mol_oe(df, options) + df = calculate_bohr_magnetons(df, options) + df = calculate_chi_inverse(df, options) + + + dfs = [] + for i in range(1,len(mask.index)): + dfs.append(df.iloc[mask.index[i-1]:mask.index[i]]) + + return dfs + + +def read_hysteresis(path): + + index = find_start(path) + + df = pd.read_csv(path, skiprows=index+1) + + df = df[['Comment', 'Time Stamp (sec)', 'Temperature (K)', 'Magnetic Field (Oe)', + 'DC Moment (emu)', 'DC Std. Err. (emu)', 'DC Quad. Moment (emu)', + 'AC=1 DC=2 Locate=3', 'Max. Field (Oe)', 'Pressure (Torr)', 'Temp. Status (code)', + ]] + + new_columns = ['Comment', 'Time', 'Temperature', 'Magnetic_Field', + 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', + 'Status', 'Max_Field', 'Pressure', 'Temperature_Status'] + + df.columns = new_columns + + df[['Temperature', 'Magnetic_Field', 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Max_Field', 'Pressure']] = df[['Temperature', 'Magnetic_Field', 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Max_Field', 'Pressure']].astype(float) + + df = df.loc[df['DC_Std_Err'] < 0.001] + + return df + + +def find_start(path): + + with open(path, 'r') as f: + + i = 0 + line = f.readline() + + while '[Data]' not in line: + line = f.readline() + i += 1 + + + if i > 1000: + break + + + return i + + + +def calculate_emu_per_mol_oe(df, options={}): + + m = options['sample_mass'] / 1000 # convert from mg to g + n = m / options['molar_mass'] + + + df['DC_Moment_emu_per_mol'] = df['DC_Moment'] / n + df['DC_Moment_emu_per_mol_oe'] = df['DC_Moment'] / (n * df['Magnetic_Field']) + + + return df + + + + +def calculate_bohr_magnetons(df, options={}): + + + default_options = { + 'units': 'cgs', + } + + options = aux.update_options(options=options, default_options=default_options) + + if options['units'] == 'cgs': + df['bohr_magnetons'] = df['DC_Moment_emu_per_mol'] * 1.07828E20 / 6.023E23 ## mu_B per emu divided by Avogadro's number + + return df + + + +def calculate_chi_inverse(df, options={}): + + df['chi_inverse'] = 1/ df['DC_Moment_emu_per_mol'] + + return df \ No newline at end of file diff --git a/nafuma/ppms/plot.py b/nafuma/ppms/plot.py new file mode 100644 index 0000000..e69de29 From 576ce0301b949739a8942517ed3822a25bf3f7a7 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:40:11 +0200 Subject: [PATCH 305/355] Add pre-edge feature fitting function --- nafuma/xanes/calib.py | 280 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 6162d99..0e0ff33 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -1,4 +1,5 @@ from logging import raiseExceptions +from signal import default_int_handler from jinja2 import TemplateRuntimeError import pandas as pd import numpy as np @@ -14,6 +15,7 @@ from datetime import datetime import ipywidgets as widgets from IPython.display import display +import warnings ##Better to make a new function that loops through the files, and performing the split_xanes_scan on @@ -445,6 +447,7 @@ def smoothing(data: dict, options={}): # FIXME Add other types of filters for i, filename in enumerate(data['path']): + if options['smooth_algorithm'] == 'savgol': if options['log']: aux.write_log(message=f'Smoothing {filename} with algorithm: {options["smooth_algorithm"]} ({i+1}/{len(data["path"])})', options=options) @@ -1012,5 +1015,282 @@ def flatten(data:dict, options={}): return flattened_df, fit_function_diff +def extract_partial_range(data: dict, options={}): + + default_options = { + 'extract_range': None, + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + if not options['extract_range']: + warnings.warn('You did not specify a range - do so with the keyword "extract_range" in the options dictionary. Returning data without modification') + return data + + + partial_data = data['xanes_data'].loc[(data['xanes_data']['ZapEnergy'] > options['extract_range'][0]) & (data['xanes_data']['ZapEnergy'] < options['extract_range'][1])] + + return partial_data + + + + +def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: + + from scipy.interpolate import UnivariateSpline + from scipy.optimize import curve_fit + from scipy.stats import norm + + default_options = { + 'remove_background': True, + 'background_model': 'exponential', + 'fit_model': 'gaussian', + 'extract_range': None, + 'extract_range_increments': [0, 0], + 'background_limits': [None, None], + 'background_limits_increments': [0, 0], + 'background_polyorder': 2, + 'log': False, + 'logfile': f'{datetime.now().strftime("%Y-%m-%d-%H-%M-%S")}_peak_fit.log', + 'show_plots': False, + 'save_plots': False, + 'save_folder': './', + 'ylim': None, + 'xlim': None, + 'interactive': False, + 'interactive_session_active': False + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + + if not options['extract_range']: + warnings.warn('You did not specify a range - do so with the keyword "extract_range" in the options dictionary. No modification is done.') + return None, None, None + + if options['log']: + aux.write_log(message='Starting fit of pre edge feature', options=options) + + + centroids = [] + errors = [] + for i, filename in enumerate(data['path']): + + options['extract_range'][0] += options['extract_range_increments'][0] + options['extract_range'][1] += options['extract_range_increments'][1] + + partial_data = extract_partial_range(data, options) + + removed_background_df = pd.DataFrame(partial_data["ZapEnergy"]) + background_df = pd.DataFrame(partial_data["ZapEnergy"]) + + if options['remove_background']: + if not options['background_limits'][0]: + options['background_limits'][0] = partial_data[1].max() - 0.003 + if not options['background_limits'][1]: + options['background_limits'][1] = partial_data[1].max() + 0.003 + + if i > 0: + options['background_limits'][0][0] += options['background_limits_increments'][0] + options['background_limits'][0][1] += options['background_limits_increments'][0] + options['background_limits'][1][0] += options['background_limits_increments'][1] + options['background_limits'][1][1] += options['background_limits_increments'][1] + + peak_background = partial_data.copy() + + #peak_background.loc[(peak_background['ZapEnergy'] > options['background_limits'][0]) & (peak_background['ZapEnergy'] < options['background_limits'][1])] = np.nan + peak_background.loc[(peak_background['ZapEnergy'] < options['background_limits'][0][0]) | + ((peak_background['ZapEnergy'] > options['background_limits'][0][1]) & + (peak_background['ZapEnergy'] < options['background_limits'][1][0])) | + (peak_background['ZapEnergy'] > options['background_limits'][1][1])] = np.nan + peak_background = peak_background.dropna() + + + # FIXME Originally tried with spline and polynomials, but they worked very poorly. This is as best as it gets at this moment, but alternatives should be considered. + if options['background_model'] == 'exponential': + def linear(x, a, b): + return a*x + b + + # Fit linear curve to the logarithm of the background + popt, pcov = curve_fit(linear, peak_background['ZapEnergy'], np.log(peak_background[filename])) + #fit_function = np.poly1d(popt) + + # Restore exponential nature of background + background = np.exp(linear(background_df['ZapEnergy'], *popt)) + background_df.insert(1,filename,background) + + removed_background_df.insert(1, filename, partial_data[filename]-background_df[filename]) + removed_background_df = removed_background_df.loc[(removed_background_df['ZapEnergy'] > options['background_limits'][0][1]) & + (removed_background_df['ZapEnergy'] < options['background_limits'][1][0]) + ] + + + + + # Fit Gaussian + # FIXME Should have options for Lorentzian and Pseudo-Voigt here as well. + if options['fit_model'] == 'gaussian': + # FIXME Understand what this deprecation warning means and what changes should be made to make it future proof + warnings.filterwarnings(action='ignore', category=np.VisibleDeprecationWarning) + + + sigma_init = removed_background_df['ZapEnergy'].loc[removed_background_df[filename] == removed_background_df[filename].max()] + popt, pcov = curve_fit( gauss, + removed_background_df['ZapEnergy'], + removed_background_df[filename], + p0=[0.0005, sigma_init, 0.001], + bounds=[ + (0, sigma_init-0.002, -np.inf), + (0.5, sigma_init+0.002, np.inf) + ] + ) + + centroids.append(popt) + errors.append(pcov) + + + if options['show_plots'] or options['save_plots']: + + fig, axes = plt.subplots(figsize=(10,5), ncols=2) + + + # Background removal + partial_data.plot(x='ZapEnergy', y=filename, ax=axes[0], color='black', label='Original data', kind='scatter') + background_df.plot(x='ZapEnergy', y=filename, ax=axes[0], color='black', ls='--', label='Fitted background') + removed_background_df.plot(x='ZapEnergy', y=filename, ax=axes[0], color='red', label='Background subtracted', kind='scatter') + axes[0].axvline(x=options['background_limits'][0][0], color='black', ls='--') + axes[0].axvline(x=options['background_limits'][0][1], color='black', ls='--') + axes[0].axvline(x=options['background_limits'][1][0], color='black', ls='--') + axes[0].axvline(x=options['background_limits'][1][1], color='black', ls='--') + axes[0].set_title(f'{os.path.basename(filename)} - Background removal', size=20) + axes[0].set_ylabel('Normalised x$\mu$(E)', size=20) + axes[0].set_xlabel('Energy (keV)', size=20) + peak_background.plot(x='ZapEnergy', y=filename, ax=axes[0], color='green', kind='scatter') + if options['xlim']: + axes[0].set_xlim(options['xlim']) + if options['ylim']: + axes[0].set_ylim(options['ylim']) + + # Fitted curve + + y_fit = gauss(removed_background_df['ZapEnergy'], *popt) + + removed_background_df.plot(x='ZapEnergy', y=filename, ax=axes[1], color='black', label='Background subtracted', kind='scatter') + axes[1].plot(removed_background_df['ZapEnergy'], y_fit, color='red', label=f'Fit data ({options["fit_model"]})') + axes[1].set_title(f'{os.path.basename(filename)} - Pre-edge feature fit', size=20) + axes[1].set_ylabel('Normalised x$\mu$(E)', size=20) + axes[1].set_xlabel('Energy (keV)', size=20) + + if options['xlim']: + axes[1].set_xlim(options['xlim']) + + + # Save plots if toggled + if options['save_plots']: + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + dst = os.path.join(options['save_folder'], os.path.basename(filename)) + '_pre_edge_feature_fit.png' + + plt.savefig(dst, transparent=False) + + + # Close plots if show_plots not toggled + if not options['show_plots']: + plt.close() + + return centroids, errors, removed_background_df + + + + +def gauss(x, A, mu, sigma): + return (A/(sigma*np.sqrt(np.pi)))*np.exp(-(x-mu)**2/(2*sigma**2)) + + + +def save_data(data, options={}): + + default_options = { + 'save_data': 'xanes_data', + 'save_folder': '.' + } + + options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) + + + filenames = [filename for filename in data[options["save_data"]].columns if not 'ZapEnergy' in filename] + + for filename in filenames: + + options['save_filename'] = os.path.basename(filename).split('.')[0] + '_exported.dat' + + save_path = os.path.join(options['save_folder'], options['save_filename']) + + if not os.path.isdir(options['save_folder']): + os.makedirs(options['save_folder']) + + to_export = data[options['save_data']][['ZapEnergy', filename]] + to_export.columns = ['E', 'I'] + to_export.to_csv(save_path) + + + + + + +def save_centroids(data: dict, options={}): + + default_options = { + 'save_path': 'centroids.dat', + 'overwrite': False, + 'append': False, + } + + options = aux.update_options(options=options, default_options=default_options) + + if options['overwrite']: + mode = 'w' + elif options['append']: + mode = 'a' + else: + mode = False + + if os.path.exists(options['save_path']) and not options['overwrite']: + with open(options['save_path'], 'r') as f: + reference = float(f.readline().split()[1]) + + else: + reference = data['centroid_fit'][0][1]*1000 + + + if not os.path.exists(os.path.dirname(options['save_path'])): + os.makedirs(os.path.dirname(options['save_path'])) + + + + if mode: + with open(options['save_path'], mode) as f: + for path, fit, error in zip(data['path'], data['centroid_fit'], data['centroid_fit_errors']): + + A = fit[0] + mu = fit[1] + sigma = fit[2] + mu_adj = (fit[1]*1000)-reference + + stddevs = np.sqrt(np.diag(error)) + + #f.write(f'{path} \t {fit[1]*1000} \t {(fit[1]-reference)*1000} \t {fit[0]} \t {fit[2]} \n') + f.write('{: <40} \t {: <25} \t {: <25} \t {: <25} \t {: <25} \t {: <25} \t {: <25} \t {: <25}\n'.format(path, mu*1000, mu_adj, A, sigma, stddevs[1]*1000, stddevs[0], stddevs[2])) + + + +def read_centroids(path): + + df = pd.read_csv(path, delim_whitespace=True, header=None) + + df.columns = ['scan', 'mu', 'mu_adj', 'A', 'sigma', 'mu_err', 'A_err', 'sigma_err'] + + return df From 76cf0f22755274ea66ba4937f0d90fdf60c1600e Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:40:54 +0200 Subject: [PATCH 306/355] Draft of write_data function --- nafuma/xanes/io.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 373a3bc..ff7f911 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -442,4 +442,22 @@ def determine_active_roi(scan_data): active_roi = None return active_roi - \ No newline at end of file + + + +def write_data(data: dict, options={}): + + + default_options = { + 'save_filenames': None, + 'save_dir': '.', + } + + options = aux.update_options(options=options, default_options=default_options, required_options=default_options.keys()) + + + if not options['save_filenames']: + options['save_filenames'] = [os.path.basename(col).split('.')[0]+'_exported.dat' for col in data['xanes_data'].columns if 'ZapEnergy' not in col] + + + print(options['save_filenames']) \ No newline at end of file From 8b0a5bcff77ce2481f2784fa914832842fe51788 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:41:30 +0200 Subject: [PATCH 307/355] Add multiplication and drawdown for diffs --- nafuma/xrd/io.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 78375d2..3ba5d82 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -720,7 +720,7 @@ def read_data(data, options={}, index=0): diffractogram['I_org'] = diffractogram['I'] diffractogram['2th_org'] = diffractogram['2th'] - diffractogram = apply_offset(diffractogram, wavelength, index, options) + diffractogram = adjust_intensities(diffractogram, wavelength, index, options) @@ -729,7 +729,7 @@ def read_data(data, options={}, index=0): return diffractogram, wavelength -def apply_offset(diffractogram, wavelength, index, options): +def adjust_intensities(diffractogram, wavelength, index, options): if 'current_offset_y' not in options.keys(): options['current_offset_y'] = options['offset_y'] @@ -748,6 +748,10 @@ def apply_offset(diffractogram, wavelength, index, options): if options['normalise']: diffractogram['I'] = diffractogram['I'] / diffractogram['I'].max() + diffractogram['I'] = diffractogram['I'] * options['multiply'] + + if options['drawdown']: + diffractogram['I'] = diffractogram['I'] - diffractogram['I'].mean() diffractogram['I'] = diffractogram['I'] + index*options['offset_y'] From 2e469909d7bf1ea6751fb4d8d45248cf98f9b2e6 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:42:05 +0200 Subject: [PATCH 308/355] Add multiply and drawdown to plotting --- nafuma/xrd/plot.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index e75dc9b..357cb83 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -22,16 +22,14 @@ def plot_diffractogram(data, options={}): Input: data (dict): Must include path = string to diffractogram data, and plot_kind = (recx, beamline, image)''' - # Update options - required_options = ['x_vals', 'y_vals', 'ylabel', 'xlabel', 'xunit', 'yunit', 'line', 'scatter', 'xlim', 'ylim', 'normalise', 'offset', 'offset_x', 'offset_y', 'offset_change', - 'reflections_plot', 'reflections_indices', 'reflections_data', 'heatmap', 'cmap', 'plot_kind', 'palettes', 'highlight', 'highlight_colours', 'interactive', 'rc_params', 'format_params', 'interactive_session_active', 'plot_diff'] - default_options = { 'x_vals': '2th', 'y_vals': 'I', 'xlabel': '2$\\theta$', 'ylabel': None, 'xunit': '$^{\circ}$', 'yunit': None, 'xlim': None, 'ylim': None, 'normalise': True, + 'multiply': 1, # Factor to multiply the normalised data - only used if normalising. + 'drawdown': False, 'offset': True, 'offset_x': 0, 'offset_y': 1, @@ -42,6 +40,7 @@ def plot_diffractogram(data, options={}): 'reflections_indices': False, # whether to plot the reflection indices 'reflections_data': None, # Should be passed as a list of dictionaries on the form {path: rel_path, reflection_indices: number of indices, colour: [r,g,b], min_alpha: 0-1] 'heatmap': False, + 'heatmap_reverse': False, 'cmap': 'viridis', 'plot_kind': None, 'palettes': [('qualitative', 'Dark2_8')], @@ -58,7 +57,7 @@ def plot_diffractogram(data, options={}): if len(data['path']) > 10: default_options['offset_y'] = 0.05 - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) #options['current_offset_y'] = options['offset_y'] # Convert data['path'] to list to allow iteration over this to accommodate both single and multiple diffractograms @@ -322,7 +321,7 @@ def plot_diffractogram(data, options={}): if options['plot_diff'] and len(data['path']) == 2: diff = data['diffractogram'][0] diff['I'] = diff['I'] - data['diffractogram'][1]['I'] - diff['I'] = diff['I'] - 0.5 + diff['I'] = diff['I'] - 0.75 diff.plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=next(colours)) From b98fd25a5c6ca45cf5628994cff0c003b4a47c89 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 9 Oct 2022 18:42:16 +0200 Subject: [PATCH 309/355] Add errors to writeouts for TOPAS --- nafuma/xrd/refinement.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 1b33bc3..71b2da9 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -451,9 +451,9 @@ def write_output(fout, data, options, index=0): fout.write('\t\t{: <40} {: <40} {: <40}'.format( - f'Out(vol_{label}_XXXX, "%11.5f")', - f'Out(mass_{label}_XXXX, "%11.5f")', - f'Out(wp_{label}_XXXX, "%11.5f")', + f'Out(vol_{label}_XXXX, "%11.5f", "%11.5f")', + f'Out(mass_{label}_XXXX, "%11.5f", "%11.5f")', + f'Out(wp_{label}_XXXX, "%11.5f", "%11.5f")', ) ) @@ -463,12 +463,12 @@ def write_output(fout, data, options, index=0): fout.write('\t\t{: <40} {: <40} {: <40} {: <40} {: <40} {: <40}'.format( - f'Out(lpa_{label}, "%11.5f")', - f'Out(lpb_{label}, "%11.5f")', - f'Out(lpc_{label}, "%11.5f")', - f'Out(lpal_{label}, "%11.5f")', - f'Out(lpbe_{label}, "%11.5f")', - f'Out(lpga_{label}, "%11.5f")', + f'Out(lpa_{label}, "%11.5f", "%11.5f")', + f'Out(lpb_{label}, "%11.5f", "%11.5f")', + f'Out(lpc_{label}, "%11.5f", "%11.5f")', + f'Out(lpal_{label}, "%11.5f", "%11.5f")', + f'Out(lpbe_{label}, "%11.5f", "%11.5f")', + f'Out(lpga_{label}, "%11.5f", "%11.5f")', ) ) @@ -477,11 +477,11 @@ def write_output(fout, data, options, index=0): for atom in atoms['atoms']: fout.write('\t\t{: <40} {: <40} {: <40} {: <40} {: <40}'.format( - f'Out(x_{atom}_{label}, "%11.5f")', - f'Out(y_{atom}_{label}, "%11.5f")', - f'Out(z_{atom}_{label}, "%11.5f")', - f'Out(occ_{atom}_{label}, "%11.5f")', - f'Out(beq_{atom}_{label}, "%11.5f")', + f'Out(x_{atom}_{label}, "%11.5f", "%11.5f")', + f'Out(y_{atom}_{label}, "%11.5f", "%11.5f")', + f'Out(z_{atom}_{label}, "%11.5f", "%11.5f")', + f'Out(occ_{atom}_{label}, "%11.5f", "%11.5f")', + f'Out(beq_{atom}_{label}, "%11.5f", "%11.5f")', ) ) @@ -930,12 +930,12 @@ def read_results(path): results = pd.read_csv(path, delim_whitespace=True, index_col=0, header=None) - atoms = int((results.shape[1] - 15) / 5) + atoms = int((results.shape[1] - 24) / 10) headers = [ 'r_wp', 'r_exp', 'r_p', 'r_p_dash', 'r_exp_dash', 'gof', - 'vol', 'mass', 'wp', - 'a', 'b', 'c', 'alpha', 'beta', 'gamma', + 'vol', 'vol_err', 'mass', 'mass_err', 'wp', 'wp_err', + 'a', 'a_err', 'b', 'b_err', 'c', 'c_err', 'alpha', 'alpha_err', 'beta', 'beta_err', 'gamma', 'gamma_err', ] @@ -943,6 +943,7 @@ def read_results(path): for i in range(atoms): for label in labels: headers.append(f'atom_{i+1}_{label}') + headers.append(f'atom_{i+1}_{label}_err') results.columns = headers From fd662e3cbdc48a3373db9f3d26af1928f3be9b67 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 10 Oct 2022 17:42:29 +0200 Subject: [PATCH 310/355] Change filename behaviour for integration --- nafuma/xrd/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 3ba5d82..696699e 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -42,7 +42,7 @@ def integrate_scans(data: dict, options={}): 'extension': '.dat', 'save': True, 'integration_save_folder': './integrated/', - 'integration_save_filename': None, + 'filename_base': 'integrated', } options = aux.update_options(options=options, required_options=default_options.keys(), default_options=default_options) @@ -53,10 +53,10 @@ def integrate_scans(data: dict, options={}): diffractograms, wavelengths = [], [] - for img in imgs: + for i, img in enumerate(imgs): data['image'] = get_image_array(img) - options['integration_save_filename'] = os.path.basename(img).split('.')[0] + '_int.xy' + options['integration_save_filename'] = options['filename_base'] + '_' + f'{i}'.zfill(4) + '.xy' diff, wl = integrate_1d(data=data, options=options) From f30426e95d92ec8caa24549b754d68a2250ab230 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 10 Oct 2022 17:42:39 +0200 Subject: [PATCH 311/355] Add arctan fit function to pre-edge background --- nafuma/xanes/calib.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 0e0ff33..cd1ce5d 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -1125,6 +1125,19 @@ def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: ] + elif options['background_model'] == 'arctan': + + popt, pcov = curve_fit(arctan, peak_background['ZapEnergy']-data['e0_diff'][filename], peak_background[filename]) + + background = arctan(background_df['ZapEnergy']-data['e0_diff'][filename], *popt) + background_df.insert(1, filename, background) + + + removed_background_df.insert(1, filename, partial_data[filename]-background_df[filename]) + removed_background_df = removed_background_df.loc[(removed_background_df['ZapEnergy'] > options['background_limits'][0][1]) & + (removed_background_df['ZapEnergy'] < options['background_limits'][1][0]) + ] + # Fit Gaussian @@ -1206,6 +1219,10 @@ def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: def gauss(x, A, mu, sigma): return (A/(sigma*np.sqrt(np.pi)))*np.exp(-(x-mu)**2/(2*sigma**2)) + + +def arctan(x,a,b,c,d): + return a*np.arctan(x*b+c) + d From 4496c34fd264ffdf71c395ddfb2cb1bac9ff4653 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 10 Oct 2022 20:32:37 +0200 Subject: [PATCH 312/355] Implement Lorentzian, PV and double peak fit --- nafuma/xanes/calib.py | 155 +++++++++++++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 23 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index cd1ce5d..2df8528 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -1044,7 +1044,7 @@ def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: default_options = { 'remove_background': True, 'background_model': 'exponential', - 'fit_model': 'gaussian', + 'peak_model': 'gaussian', 'extract_range': None, 'extract_range_increments': [0, 0], 'background_limits': [None, None], @@ -1136,30 +1136,81 @@ def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: removed_background_df.insert(1, filename, partial_data[filename]-background_df[filename]) removed_background_df = removed_background_df.loc[(removed_background_df['ZapEnergy'] > options['background_limits'][0][1]) & (removed_background_df['ZapEnergy'] < options['background_limits'][1][0]) - ] + ] # Fit Gaussian # FIXME Should have options for Lorentzian and Pseudo-Voigt here as well. - if options['fit_model'] == 'gaussian': - # FIXME Understand what this deprecation warning means and what changes should be made to make it future proof - warnings.filterwarnings(action='ignore', category=np.VisibleDeprecationWarning) + + # FIXME Understand what this deprecation warning means and what changes should be made to make it future proof + warnings.filterwarnings(action='ignore', category=np.VisibleDeprecationWarning) - sigma_init = removed_background_df['ZapEnergy'].loc[removed_background_df[filename] == removed_background_df[filename].max()] - popt, pcov = curve_fit( gauss, - removed_background_df['ZapEnergy'], - removed_background_df[filename], - p0=[0.0005, sigma_init, 0.001], - bounds=[ - (0, sigma_init-0.002, -np.inf), - (0.5, sigma_init+0.002, np.inf) - ] - ) + mu_init = float(removed_background_df['ZapEnergy'].loc[removed_background_df[filename] == removed_background_df[filename].max()]) + + if options['peak_model'] == 'gaussian': + popt, pcov = curve_fit(gauss, + removed_background_df['ZapEnergy'], + removed_background_df[filename], + p0=[0.0005, mu_init, 0.001], + bounds=[ + (0, mu_init-0.002, -np.inf), + (0.5, mu_init+0.002, np.inf) + ] + ) - centroids.append(popt) - errors.append(pcov) + + elif options['peak_model'] == '2gaussian': + popt, pcov = curve_fit(_2gauss, + removed_background_df['ZapEnergy'], + removed_background_df[filename], + p0=[0.0005, mu_init, 0.001, 0.0005, mu_init+0.003, 0.0005], + #bounds=[ + # (0, mu_init-0.002, -np.inf), + # (0.5, mu_init+0.002, np.inf) + # ] + #) + ) + + elif options['peak_model'] == 'lorentzian': + + popt, pcov = curve_fit(lorentz, + removed_background_df['ZapEnergy'], + removed_background_df[filename], + p0=[1, mu_init, 0.001], + # bounds=[ + # (mu_init-0.001, 0.00001), + # (mu_init+0.001, 0.01) + # ] + ) + + + elif options['peak_model'] == '2lorentzian': + + popt, pcov = curve_fit(_2lorentz, + removed_background_df['ZapEnergy'], + removed_background_df[filename], + p0=[1, mu_init, 0.001, 1, mu_init+0.003, 0.001], + # bounds=[ + # (mu_init-0.001, 0.00001), + # (mu_init+0.001, 0.01) + # ] + ) + + + + elif options['peak_model'] == 'pseudo-voigt': + + popt, pcov = curve_fit(pseudo_voigt, + removed_background_df['ZapEnergy'], + removed_background_df[filename], + p0=[1, mu_init, 0.001, 0.5] + ) + + + centroids.append(popt) + errors.append(pcov) if options['show_plots'] or options['save_plots']: @@ -1175,24 +1226,63 @@ def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: axes[0].axvline(x=options['background_limits'][0][1], color='black', ls='--') axes[0].axvline(x=options['background_limits'][1][0], color='black', ls='--') axes[0].axvline(x=options['background_limits'][1][1], color='black', ls='--') - axes[0].set_title(f'{os.path.basename(filename)} - Background removal', size=20) - axes[0].set_ylabel('Normalised x$\mu$(E)', size=20) - axes[0].set_xlabel('Energy (keV)', size=20) + axes[0].set_title(f'{os.path.basename(filename)} - Background removal', size=10) + peak_background.plot(x='ZapEnergy', y=filename, ax=axes[0], color='green', kind='scatter') if options['xlim']: axes[0].set_xlim(options['xlim']) if options['ylim']: axes[0].set_ylim(options['ylim']) + + axes[0].set_ylabel('Normalised x$\mu$(E)', size=20) + axes[0].set_xlabel('Energy (keV)', size=20) + axes[0].axhline(y=0, ls='--', color='black') # Fitted curve - y_fit = gauss(removed_background_df['ZapEnergy'], *popt) + if options['peak_model'] == 'gaussian': + y_fit = gauss(removed_background_df['ZapEnergy'], *popt) + components = [y_fit] + + elif options['peak_model'] == 'lorentzian': + y_fit = lorentz(removed_background_df['ZapEnergy'], *popt) + components = [y_fit] + + elif options['peak_model'] == 'pseudo-voigt': + y_fit = pseudo_voigt(removed_background_df['ZapEnergy'], *popt) + components = [y_fit] + + elif options['peak_model'] == '2gaussian': + y_fit = _2gauss(removed_background_df['ZapEnergy'], *popt) + y_fit1 = gauss(removed_background_df['ZapEnergy'], *popt[0:3]) + y_fit2 = gauss(removed_background_df['ZapEnergy'], *popt[3:6]) + + components = [y_fit1, y_fit2] + + elif options['peak_model'] == '2lorentzian': + y_fit = _2lorentz(removed_background_df['ZapEnergy'], *popt) + y_fit1 = lorentz(removed_background_df['ZapEnergy'], *popt[0:3]) + y_fit2 = lorentz(removed_background_df['ZapEnergy'], *popt[3:6]) + + components = [y_fit1, y_fit2] removed_background_df.plot(x='ZapEnergy', y=filename, ax=axes[1], color='black', label='Background subtracted', kind='scatter') - axes[1].plot(removed_background_df['ZapEnergy'], y_fit, color='red', label=f'Fit data ({options["fit_model"]})') - axes[1].set_title(f'{os.path.basename(filename)} - Pre-edge feature fit', size=20) + + axes[1].plot(removed_background_df['ZapEnergy'], y_fit, color='red', label=f'Fit data ({options["peak_model"]})') + for comp in components: + axes[1].fill_between(x=removed_background_df['ZapEnergy'], y1=comp, y2=0, alpha=0.2) + + + residuals = (removed_background_df[filename] - y_fit) - 0.1*removed_background_df[filename].max() + axes[1].scatter(x=removed_background_df['ZapEnergy'], y=residuals) + axes[1].axhline(y=-0.1*removed_background_df[filename].max(), ls='--', color='black') + + axes[1].set_title(f'{os.path.basename(filename)} - Pre-edge feature fit', size=10) axes[1].set_ylabel('Normalised x$\mu$(E)', size=20) axes[1].set_xlabel('Energy (keV)', size=20) + axes[1].legend() + axes[1].axhline(y=0, ls='--', color='black') + if options['xlim']: axes[1].set_xlim(options['xlim']) @@ -1220,9 +1310,28 @@ def fit_pre_edge_feautre(data: dict, options={}) -> pd.DataFrame: def gauss(x, A, mu, sigma): return (A/(sigma*np.sqrt(np.pi)))*np.exp(-(x-mu)**2/(2*sigma**2)) +def _2gauss(x, A1, mu1, sigma1, A2, mu2, sigma2): + return (A1/(sigma1*np.sqrt(np.pi)))*np.exp(-(x-mu1)**2/(2*sigma1**2))+(A2/(sigma2*np.sqrt(np.pi)))*np.exp(-(x-mu2)**2/(2*sigma2**2)) + +def lorentz(x, A, mu, sigma): + return (A/np.pi * ((sigma)/(((x-mu)**2) + (sigma)**2))) + +def _2lorentz(x, A1, mu1, sigma1, A2, mu2, sigma2): + return (A1/np.pi * ((sigma1)/(((x-mu1)**2) + (sigma1)**2))) + (A2/np.pi * ((sigma2)/(((x-mu2)**2) + (sigma2)**2))) + + +def pseudo_voigt(x, A, mu, sigma, eta): + + G = gauss(x, A, mu, sigma) + L = lorentz(x, A, mu, sigma) + + return eta*G + (1-eta)*L + def arctan(x,a,b,c,d): return a*np.arctan(x*b+c) + d + + From 2a2f092f8b8dba6964ea3a8528a438675ff21ac0 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Tue, 11 Oct 2022 19:56:32 +0200 Subject: [PATCH 313/355] Add make_animation() --- nafuma/plotting.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 99b7d7f..3a0b71b 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -14,6 +14,8 @@ import matplotlib.lines as mlines import matplotlib.markers as mmarkers import itertools +from PIL import Image +import os import numpy as np @@ -465,3 +467,22 @@ def connect_inset(parent_axes, inset_axes, loc1a=1, loc1b=1, loc2a=2, loc2b=2, * +def make_animation(paths, options={}): + + default_options = { + 'save_folder': '.', + 'save_filename': 'animation.gif', + 'fps': 5 + } + + options = aux.update_options(options=options, default_options=default_options) + + + frames = [] + for path in paths: + frame = Image.open(path) + frames.append(frame) + + frames[0].save(os.path.join(options['save_folder'], options['save_filename']), format='GIF', append_images=frames[1:], save_all=True, duration=(1/options['fps'])*1000, loop=0) + + \ No newline at end of file From 2deb640b3d489569618c114df19fdce3a6372cc4 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Tue, 11 Oct 2022 19:57:05 +0200 Subject: [PATCH 314/355] Add option to exclude (make region = 0) in diffs --- nafuma/xrd/io.py | 9 +++++++++ nafuma/xrd/plot.py | 1 + 2 files changed, 10 insertions(+) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 696699e..f456230 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -714,6 +714,15 @@ def read_data(data, options={}, index=0): elif file_extension in['xy', 'xye']: diffractogram, wavelength = read_xy(data=data, options=options, index=index) + + if options['exclude']: + + if not isinstance(options['exclude'], list): + options['exclude'] = [options['exclude']] + + for excl in options['exclude']: + diffractogram['I'].loc[(diffractogram['2th'] > excl[0]) & (diffractogram['2th'] < excl[1])] = 0 + if options['offset'] or options['normalise']: # Make copy of the original intensities before any changes are made through normalisation or offset, to easily revert back if need to update. diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 357cb83..8f6d07c 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -28,6 +28,7 @@ def plot_diffractogram(data, options={}): 'xunit': '$^{\circ}$', 'yunit': None, 'xlim': None, 'ylim': None, 'normalise': True, + 'exclude': None, 'multiply': 1, # Factor to multiply the normalised data - only used if normalising. 'drawdown': False, 'offset': True, From c2477bfd9956ce0c913708862646185318d1a343 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Wed, 12 Oct 2022 21:07:22 +0200 Subject: [PATCH 315/355] Update COOP-plotting function --- nafuma/dft/electrons.py | 18 +++++++++--------- nafuma/dft/io.py | 6 +++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/nafuma/dft/electrons.py b/nafuma/dft/electrons.py index 8ea1064..1fd3092 100644 --- a/nafuma/dft/electrons.py +++ b/nafuma/dft/electrons.py @@ -786,12 +786,10 @@ def prettify_dos_plot(fig, ax, options): -def plot_coop(plot_data, options): +def plot_coop(data, options): ''' interactions = list with number of interaction (index + 1 of interactions list from read_coop)''' - required_options = ['plot_kind', 'mode', 'up', 'down', 'collapse', 'interactions', 'flip_xy', 'fill', 'colours', 'palettes'] - default_options = { 'plot_kind': 'COOP', 'mode': 'individual', @@ -806,17 +804,17 @@ def plot_coop(plot_data, options): } - options = update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) - fig, ax = prepare_plot(options=options) + fig, ax = btp.prepare_plot(options=options) - coopcar, coop_interactions = dft.io.read_coop(plot_data=plot_data, options=options) + coopcar, coop_interactions = dft.io.read_coop(data=data, options=options) if not options['colours']: - colour_cycle = generate_colours(palettes=options['palettes']) + colour_cycle = btp.generate_colours(palettes=options['palettes']) colours = [] for interaction in range(len(coop_interactions)): @@ -894,6 +892,8 @@ def plot_coop(plot_data, options): to_plot = ['mean_down'] + + # Plot all columns as decided above for j, column in enumerate(to_plot): if options['fill']: @@ -912,9 +912,9 @@ def plot_coop(plot_data, options): else: coopcar.plot(x='Energy', y=column, ax=ax, color=colour) - prettify_dos_plot(fig=fig, ax=ax, options=options) + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) - return coopcar + return coopcar, fig, ax diff --git a/nafuma/dft/io.py b/nafuma/dft/io.py index e09de45..23d7d41 100644 --- a/nafuma/dft/io.py +++ b/nafuma/dft/io.py @@ -94,7 +94,8 @@ def read_coop(data={}, options={}): required_options = ['collapse'] default_options = { - 'collapse': False + 'collapse': False, + 'adjust': None, } @@ -128,6 +129,9 @@ def read_coop(data={}, options={}): coopcar.columns = columns + if options['adjust']: + coopcar['Energy'] = coopcar['Energy'] - options['adjust'] + if options['collapse']: From 33012c60354c48cc34b5c03c54e866dfd1f20584 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Wed, 12 Oct 2022 21:07:44 +0200 Subject: [PATCH 316/355] Allow user to skip read-in of error columns --- nafuma/xrd/refinement.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 71b2da9..75ecee2 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -926,24 +926,44 @@ def refine(data: dict, options={}): -def read_results(path): +def read_results(path, options={}): + + default_options = { + 'errors': True + } + + options = aux.update_options(options=options, default_options=default_options) + results = pd.read_csv(path, delim_whitespace=True, index_col=0, header=None) - atoms = int((results.shape[1] - 24) / 10) + if options['errors']: + atoms = int((results.shape[1] - 24) / 10) - headers = [ + headers = [ 'r_wp', 'r_exp', 'r_p', 'r_p_dash', 'r_exp_dash', 'gof', 'vol', 'vol_err', 'mass', 'mass_err', 'wp', 'wp_err', 'a', 'a_err', 'b', 'b_err', 'c', 'c_err', 'alpha', 'alpha_err', 'beta', 'beta_err', 'gamma', 'gamma_err', - ] + ] + + else: + atoms = int((results.shape[1] - 15) / 5) + + headers = [ + 'r_wp', 'r_exp', 'r_p', 'r_p_dash', 'r_exp_dash', 'gof', + 'vol', 'mass', 'wp', + 'a', 'b', 'c', 'alpha', 'beta', 'gamma', + ] labels = ['x', 'y', 'z', 'occ', 'beq'] for i in range(atoms): for label in labels: headers.append(f'atom_{i+1}_{label}') - headers.append(f'atom_{i+1}_{label}_err') + + if options['errors']: + headers.append(f'atom_{i+1}_{label}_err') + results.columns = headers From aa6fa7875d601c1242a3f4e228410543294d31cb Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 12 Oct 2022 21:13:17 +0200 Subject: [PATCH 317/355] Added "" so also files with "dash" can be refined --- nafuma/xrd/refinement.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 71b2da9..6d7939c 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -422,17 +422,17 @@ def write_output(fout, data, options, index=0): fout.write('#ifdef output\n') - fout.write(f'\t\tOut_Riet({options["save_dir"]}/{filename}_{label}_riet.xy)\n') - fout.write(f'\t\tOut_CIF_STR({options["save_dir"]}/{filename}_{label}.cif)\n') - fout.write(f'\t\tOut_CIF_ADPs({options["save_dir"]}/{filename}_{label}.cif)\n') - fout.write(f'\t\tOut_CIF_Bonds_Angles({options["save_dir"]}/{filename}_{label}.cif)\n') - fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file({options["save_dir"]}/{filename}_{label}_hkl.dat)\n') + fout.write(f'\t\tOut_Riet("{options["save_dir"]}/{filename}_{label}_riet.xy")\n') + fout.write(f'\t\tOut_CIF_STR("{options["save_dir"]}/{filename}_{label}.cif")\n') + fout.write(f'\t\tOut_CIF_ADPs("{options["save_dir"]}/{filename}_{label}.cif")\n') + fout.write(f'\t\tOut_CIF_Bonds_Angles("{options["save_dir"]}/{filename}_{label}.cif")\n') + fout.write(f'\t\tCreate_hklm_d_Th2_Ip_file("{options["save_dir"]}/{filename}_{label}_hkl.dat")\n') if options['magnetic_space_group']: - fout.write(f'\t\tOut_CIF_mag({options["save_dir"]}/{filename}_{label}_magnetic.cif)\n') + fout.write(f'\t\tOut_CIF_mag("{options["save_dir"]}/{filename}_{label}_magnetic.cif")\n') fout.write('\n') - fout.write(f'out {options["save_dir"]}/{filename}_{label}.dat append\n') + fout.write(f'out "{options["save_dir"]}/{filename}_{label}.dat" append\n') fout.write(f'\t\tOut_String("XXXX")\n') From 536160666e59ccf9bbd3faebd134e10b03365288 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Thu, 13 Oct 2022 20:51:54 +0200 Subject: [PATCH 318/355] Fix bug that didn't plot text in correct Axes-obj --- nafuma/plotting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 3a0b71b..6d9bf09 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -305,7 +305,7 @@ def adjust_plot(fig, ax, options): # Plot all passed texts for text in options['text']: - plt.text(x=text[1][0], y=text[1][1], s=text[0]) + ax.text(x=text[1][0], y=text[1][1], s=text[0]) return fig, ax From c6842f61968e0f7ae49b3d80b5e99866fcb8c1d0 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Thu, 13 Oct 2022 20:52:37 +0200 Subject: [PATCH 319/355] Allow fetching of equi data if one dosn't converge --- nafuma/dft/structure.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nafuma/dft/structure.py b/nafuma/dft/structure.py index 20f2994..6558315 100644 --- a/nafuma/dft/structure.py +++ b/nafuma/dft/structure.py @@ -894,9 +894,10 @@ def get_equilibrium_data(path, atoms_per_formula_unit, eos=None): atoms, atom_num, atoms_dict = get_atoms(os.path.join(dir, 'POSCAR')) scaling_factor = sum(atom_num) / atoms_per_formula_unit - label = dir.split('/')[-1] + label = os.path.basename(dir) - dft_df = pd.read_csv(os.path.join(dir, 'energ.dat'), header=None, delim_whitespace=True) + dft_df = pd.read_csv(os.path.join(dir, 'energ.dat'), header=None, delim_whitespace=True, index_col=0) + dft_df.reset_index(drop=True, inplace=True) dft_df.columns = ['Volume', 'Energy'] volume = dft_df["Volume"].to_numpy() / scaling_factor @@ -904,10 +905,15 @@ def get_equilibrium_data(path, atoms_per_formula_unit, eos=None): p0 = get_initial_guesses(volume, energy) - equilibrium_constants = fit_eos_curve(volume, energy, p0, eos) - e0, v0, b0, bp = equilibrium_constants[0], equilibrium_constants[1], equilibrium_constants[2], equilibrium_constants[3] + try: + equilibrium_constants = fit_eos_curve(volume, energy, p0, eos) - data.append([label, e0, v0, b0/kJ*1e24, bp]) + e0, v0, b0, bp = equilibrium_constants[0], equilibrium_constants[1], equilibrium_constants[2], equilibrium_constants[3] + + data.append([label, e0, v0, b0/kJ*1e24, bp]) + + except: + data.append([label, None, None, None, None]) df = pd.DataFrame(data) From 36c648292880fff59ada45ccad1f61443e1d8c2e Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Thu, 13 Oct 2022 20:52:58 +0200 Subject: [PATCH 320/355] Change way PPMS-data is read --- nafuma/ppms/io.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/nafuma/ppms/io.py b/nafuma/ppms/io.py index aab94eb..8691163 100644 --- a/nafuma/ppms/io.py +++ b/nafuma/ppms/io.py @@ -5,10 +5,15 @@ import nafuma.auxillary as aux def read_data(path, options={}): + default_options = { + 'split': False, + } + + options = aux.update_options(options=options, default_options=default_options) + index = find_start(path) df = pd.read_csv(path, skiprows=index+1) - mask = df.loc[df['Comment'].notna()] df = df[['Comment', 'Time Stamp (sec)', 'Temperature (K)', 'Magnetic Field (Oe)', 'DC Moment (emu)', 'DC Std. Err. (emu)', 'DC Quad. Moment (emu)', @@ -19,7 +24,8 @@ def read_data(path, options={}): 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Status', 'Max_Field', 'Pressure', 'Temperature_Status'] - df.columns = new_columns + df.columns = new_columns + df[['Temperature', 'Magnetic_Field', 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Max_Field', 'Pressure']] = df[['Temperature', 'Magnetic_Field', 'DC_Moment', 'DC_Std_Err', 'DC_Quad_Moment', 'Max_Field', 'Pressure']].astype(float) @@ -30,12 +36,15 @@ def read_data(path, options={}): df = calculate_bohr_magnetons(df, options) df = calculate_chi_inverse(df, options) + if options['split']: + mask = df.loc[df['Comment'].notna()] + dfs = [] + for i in range(1,len(mask.index)): + dfs.append(df.iloc[mask.index[i-1]:mask.index[i]]) - dfs = [] - for i in range(1,len(mask.index)): - dfs.append(df.iloc[mask.index[i-1]:mask.index[i]]) + return dfs - return dfs + return df def read_hysteresis(path): From 4cc6f4557fbf7036010d892efc77c2cc61c0d862 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Thu, 13 Oct 2022 20:53:42 +0200 Subject: [PATCH 321/355] Improve refinement plotting function --- nafuma/xrd/plot.py | 108 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 89 insertions(+), 19 deletions(-) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 8f6d07c..6a11fd4 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -441,7 +441,8 @@ def generate_heatmap(data, options={}): xticks[xval] = xticks_xval - options['x_tick_locators'] = None + # FIXME COMMENTED OUT THIS LINE TO FIX SOMETHING - NOT SURE WHAT UNINTENDED CONSEQUENCES THAT MAY HAVE.... + #options['x_tick_locators'] = None heatmap = heatmap.reset_index().pivot_table(index='scan', columns='2th', values='I') @@ -826,7 +827,8 @@ def plot_reflection_table(data, reflections_params, ax=None, options={}): 'wavelength': 1.54059, # CuKalpha, [Å] 'format_params': {}, 'rc_params': {}, - 'label': None + 'label': None, + 'heatmap': False } @@ -975,6 +977,16 @@ def plot_refinement(data, options={}): 'r_wp': True, 'r_exp': False, 'wp': False, + 'wavelength': None, + 'xlabel': '2$\\theta$', 'xunit': '$^{\circ}$', + 'ylabel': 'Intensity', 'yunit': 'arb. u.', + 'text': [], + 'text_pos': [0.7, 0.9], + 'text_pos_increments': [0, -0.1], + 'reflections_plot': False, # whether to plot reflections as a plot + 'reflections_indices': False, # whether to plot the reflection indices + 'reflections_data': None, # Should be passed as a list of dictionaries on the form {path: rel_path, reflection_indices: number of indices, colour: [r,g,b], min_alpha: 0-1] + } options = aux.update_options(options=options, default_options=default_options, required_options=required_options) @@ -1008,31 +1020,89 @@ def plot_refinement(data, options={}): for attr in results.keys(): results[attr].append(result[attr].iloc[options['index']]) - fig, ax = plt.subplots(figsize=(20,10)) + # CREATE AND ASSIGN AXES - df.plot(x='2th', y='Yobs', kind='scatter', ax=ax, c='black', marker='$\u25EF$') + # Makes a list out of reflections_data if it only passed as a dict, as it will be looped through later + if options['reflections_data']: + if not isinstance(options['reflections_data'], list): + options['reflections_data'] = [options['reflections_data']] + + + # Determine the grid layout based on how many sets of reflections data has been passed + if options['reflections_data'] and len(options['reflections_data']) >= 1: + options = determine_grid_layout(options=options) + + # Create the Figure and Axes objects + fig, ax = btp.prepare_plot(options=options) + + # Assign the correct axes to the indicies, reflections and figure itself + if options['reflections_plot'] or options['reflections_indices']: + + if options['reflections_indices']: + indices_ax = ax[0] + + if options['reflections_plot']: + ref_axes = [axx for axx in ax[range(1,len(options['reflections_data'])+1)]] + + else: + ref_axes = [axx for axx in ax[range(0,len(options['reflections_data']))]] + + ax = ax[-1] + + df.plot.scatter(x='2th', y='Yobs', ax=ax, c='black', marker='$\u25EF$', s=plt.rcParams['lines.markersize']*10) df.plot(x='2th', y='Ycalc', ax=ax, c='red') df.plot(x='2th', y='diff', ax=ax) - if options['r_wp']: - ax.text(x=0.7*df['2th'].max(), y=0.7*df['Yobs'].max(), s='R$_{wp}$ = '+f'{r_wp}', fontsize=20) - - if options['r_exp']: - ax.text(x=0.70*df['2th'].max(), y=0.60*df['Yobs'].max(), s='R$_{exp}$ = '+f'{r_exp}', fontsize=20) + + if options['sample']: + options['text'].append([options['sample'], [options['text_pos'][0]*df['2th'].max(), options['text_pos'][1]*df['Yobs'].max()]]) + options['text_pos'][0] += options['text_pos_increments'][0] + options['text_pos'][1] += options['text_pos_increments'][1] + + if options['wavelength']: + options['text'].append([f'$\lambda$ = {options["wavelength"]} Å', [options['text_pos'][0]*df['2th'].max(), options['text_pos'][1]*df['Yobs'].max()]]) + options['text_pos'][0] += options['text_pos_increments'][0] + options['text_pos'][1] += options['text_pos_increments'][1] + if options['wp']: for i, (result, label) in enumerate(zip(data['results'], options['labels'])): - ax.text(x=0.7*df['2th'].max(), y=(0.9-0.1*i)*df['Yobs'].max(), s=f'{label}: {np.round(float(results["wp"][i]), 2)}%', fontsize=20) + options['text'].append([f'{label}: {np.round(float(results["wp"][i]), 1)}%', [options['text_pos'][0]*df['2th'].max(), options['text_pos'][1]*df['Yobs'].max()]]) + + + #ax.text(x=0.7*df['2th'].max(), y=ypos*df['Yobs'].max(), s=f'{label}: {np.round(float(results["wp"][i]), 2)}%', fontsize=20) + options['text_pos'][0] += options['text_pos_increments'][0] + options['text_pos'][1] += options['text_pos_increments'][1] - if options['title']: - ax.set_title(options['title'], size=30) + if options['r_wp']: + options['text'].append(['R$_{wp}$ = '+f'{np.round(r_wp, 2)}', [options['text_pos'][0]*df['2th'].max(), options['text_pos'][1]*df['Yobs'].max()]]) + options['text_pos'][0] += options['text_pos_increments'][0] + options['text_pos'][1] += options['text_pos_increments'][1] + #ax.text(x=0.7*df['2th'].max(), y=0.7*df['Yobs'].max(), s='R$_{wp}$ = '+f'{r_wp}') + + if options['r_exp']: + options['text'].append(['R$_{exp}$ = '+f'{np.round(r_exp, 2)}', [options['text_pos'][0]*df['2th'].max(), options['text_pos'][1]*df['Yobs'].max()]]) + options['text_pos'][0] += options['text_pos_increments'][0] + options['text_pos'][1] += options['text_pos_increments'][1] + #ax.text(x=0.70*df['2th'].max(), y=0.60*df['Yobs'].max(), s='R$_{exp}$ = '+f'{r_exp}') - if options['xlim']: - ax.set_xlim(options['xlim']) - else: - ax.set_xlim([df['2th'].min(), df['2th'].max()]) - ax.tick_params(which='both', labelleft=False, left=False, labelsize=20, direction='in') - ax.set_ylabel('Intensity [arb. u.]', size=20) - ax.set_xlabel('2$\\theta$ [$^{\circ}$]', size=20) \ No newline at end of file + if 'xlim' not in options.keys() or options['xlim'] == None: + options['xlim'] = [df['2th'].min(), df['2th'].max()] + + + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + # PLOT REFLECTION TABLES + if options['reflections_plot'] and options['reflections_data']: + options['xlim'] = ax.get_xlim() + options['to_wavelength'] = options['wavelength'] # By default, the wavelength of the first diffractogram will be used for these. + + # Plot each reflection table in the relevant axis + for reflections_params, axis in zip(options['reflections_data'], ref_axes): + plot_reflection_table(data=data, reflections_params=reflections_params, ax=axis, options=options) + + + + From b483d8e5fb7c968420e974e8b7a76e9c59ad6d65 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 14 Oct 2022 10:21:26 +0200 Subject: [PATCH 322/355] Generalized read_xy, so that WL can be float --- nafuma/xrd/io.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index f456230..30e9ea7 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -576,10 +576,12 @@ def read_xy(data, options={}, index=0): #if 'wavelength' not in data.keys(): # Get wavelength from scan - + if 'wavelength' in data.keys() and not type(data['wavelength']) == list: + data['wavelength'] = [data['wavelength']] if not 'wavelength' in data.keys() or not data['wavelength'][index]: wavelength = read_metadata_from_xy(path=data['path'][index])['wavelength'] + else: wavelength = data['wavelength'][index] From 7501ac528e052b63b888b8ef75706656b0647667 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 14 Oct 2022 13:24:39 +0200 Subject: [PATCH 323/355] Inclusion of manual background in write_xdd --- nafuma/xrd/refinement.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 47e0c12..f97833a 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -13,10 +13,9 @@ import pandas as pd import nafuma.auxillary as aux - def make_initial_inp(data: dict, options={}): - required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'instrument', 'topas_options', 'background', 'capillary', 'th2_offset', 'fit_peak_width', 'simple_axial_model', 'TCHZ_Peak_Type', 'start', 'finish', 'exclude', 'magnetic_atoms', 'radiation', 'magnetic_space_group', 'interval'] + # required_options = ['filename', 'overwrite', 'include', 'save_results', 'save_dir', 'instrument', 'topas_options', 'background', 'capillary', 'th2_offset', 'fit_peak_width', 'simple_axial_model', 'TCHZ_Peak_Type', 'start', 'finish', 'exclude', 'magnetic_atoms', 'radiation', 'magnetic_space_group', 'interval'] default_options = { 'filename': 'start.inp', @@ -38,10 +37,11 @@ def make_initial_inp(data: dict, options={}): 'interval': [None, None], # Start and finish values that TOPAS should refine on. Overrides 'start' and 'finish' 'start': None, # Start value only. Overridden by 'interval' if this is set. 'finish': None, # Finish value only. Overridden by 'interval' if this is set. - 'exclude': [] # Excluded regions. List of lists. + 'exclude': [], # Excluded regions. List of lists. + 'manual_background': False #Upload a background-file } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) options['topas_options'] = update_topas_options(options=options) @@ -134,9 +134,18 @@ def write_xdd(fout, data, options): fout.write('\t'+snippets['gauss_fwhm'].format(peak_width_params[0], peak_width_params[1], peak_width_params[2])+'\n') # Write background - fout.write('\tbkg @ ') + fout.write('\tbkg @ ') for i in range(options['background']): fout.write('0 ') + #EXTRA for manual background: + if options['manual_background'] != False: + print('YEAH1') + fout.write('\n\t\'manual background file:') + print('YEAH2') + fout.write('\n\tuser_y my_shape {_xy #include "'+options['manual_background']+'"} \n') + print('YEAH3') + fout.write('\tprm !my_scale = 1;: 1.00000') + # Write wavelength and LP-factor fout.write('\n\n') From 366768ecbf4a69ebcee8dfb26ed5ab66cdeb3efd Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 14 Oct 2022 13:47:13 +0200 Subject: [PATCH 324/355] removing random print-functions --- nafuma/xrd/refinement.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index f97833a..84a76f2 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -137,13 +137,10 @@ def write_xdd(fout, data, options): fout.write('\tbkg @ ') for i in range(options['background']): fout.write('0 ') - #EXTRA for manual background: + #EXTRA for implementation of manual background: if options['manual_background'] != False: - print('YEAH1') fout.write('\n\t\'manual background file:') - print('YEAH2') fout.write('\n\tuser_y my_shape {_xy #include "'+options['manual_background']+'"} \n') - print('YEAH3') fout.write('\tprm !my_scale = 1;: 1.00000') From 82b0981adabd4dcd36cf322475b3fd6bffd448d4 Mon Sep 17 00:00:00 2001 From: halvorhv Date: Fri, 14 Oct 2022 17:09:42 +0200 Subject: [PATCH 325/355] Initial func to make manual background data in inp --- nafuma/xrd/refinement.py | 50 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 84a76f2..921c1cf 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -104,8 +104,53 @@ def make_initial_inp(data: dict, options={}): fout.write(line) +def make_manual_background(data, options): + #FIXME generalize this so it works properly + #FIXME fix so that plotting is optional + import numpy as np + import nafuma.xrd as xrd + import matplotlib.pyplot as plt - + default_options = { + 'plot_background': True, + 'interval_length': 0.05, + 'save_dir': 'background' + } + if "noheaders" in data['path'][0]: + filename = os.path.basename(data['path'][0]).split('_noheaders.')[0] + else: + filename = os.path.basename(data['path'][0]).split('.')[0] + + + options = aux.update_options(options=options, default_options=default_options) + + #data['background_in_q'] is a .txt-file with one column of x-values that are good starting points for background determination, as they should not include any peaks for that specific material + + #importing the pre-made background points in Q into a dataframe + df_backgroundpoints_q=pd.read_csv(data['background_in_q'],names=["background_q"]) + + diffractogram, wavelength = xrd.io.read_xy(data=data) + + #transferring q-range background to the respective wavelength + x_background_points=[] + for q in df_backgroundpoints_q["background_q"]: + twotheta=2*180/np.pi*np.arcsin(q*wavelength/(4*np.pi)) + x_background_points.append(twotheta) + + fig,ax=plt.subplots(figsize=(20,20)) + diffractogram.plot(x="2th",y="I", kind="scatter",ax=ax) + intervallength=options['interval_length'] + background=pd.DataFrame() + + for i, x in enumerate(x_background_points): + test=diffractogram.loc[(diffractogram["2th"]>x-intervallength) & (diffractogram["2th"] Date: Sat, 15 Oct 2022 17:14:31 +0200 Subject: [PATCH 326/355] Add function to find index closest to a val in df --- nafuma/auxillary.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 8044914..c01940c 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -163,4 +163,25 @@ def swap_values(options: dict, key1, key2): options[k1], options[k2] = options[k2], options[k1] - return options \ No newline at end of file + return options + + + +def find_neighbours(value, df, colname, start=0, end=-1): + ''' Finds closest match to a given value in colname of df. If there is an exact match, returns index of this value. Else, it returns the nearest neighbors (upper and lower)''' + + df = df.iloc[start:end] + + exactmatch = df[df[colname] == value] + if not exactmatch.empty: + return exactmatch.index + else: + lower_df = df[df[colname] < value][colname] + upper_df = df[df[colname] > value][colname] + + lowerneighbour_ind = lower_df.idxmax() + upperneighbour_ind = upper_df.idxmin() + + print(lowerneighbour_ind, upperneighbour_ind) + + return [lowerneighbour_ind, upperneighbour_ind] \ No newline at end of file From 317805f4c31e1e0be1153292e7403efa2e801cfb Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sat, 15 Oct 2022 17:16:04 +0200 Subject: [PATCH 327/355] Add functions to read and plot EDS spectrum --- nafuma/eds/io.py | 25 +++++++++++++++++++ nafuma/eds/plot.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/nafuma/eds/io.py b/nafuma/eds/io.py index 8d1b34f..91c8c51 100644 --- a/nafuma/eds/io.py +++ b/nafuma/eds/io.py @@ -1,6 +1,7 @@ from PIL import Image import numpy as np import cv2 +import pandas as pd def read_image(path, weight=None, colour=None, crop=None, resize=None, brightness=None): @@ -125,3 +126,27 @@ def change_colour(image, new_colour): return image + + +def read_spectrum(path): + + headers = find_start(path) + + spectrum = pd.read_csv(path, skiprows=headers, delim_whitespace=True) + + + return spectrum + + + +def find_start(path): + + with open(path, 'r') as f: + line = f.readline() + i = 0 + while not line.startswith('Energy'): + line = f.readline() + i += 1 + + return i + diff --git a/nafuma/eds/plot.py b/nafuma/eds/plot.py index 7726f75..be8ee39 100644 --- a/nafuma/eds/plot.py +++ b/nafuma/eds/plot.py @@ -71,3 +71,65 @@ def show_image(data, options={}): else: return data['image'], None, None + + +def plot_spectrum(data: dict, options={}): + + default_options = { + 'deconvolutions': None, + 'lines': None, + 'colours': None, + 'xlabel': 'Energy', 'xunit': 'keV', 'xlim': None, + 'ylabel': 'Counts', 'yunit': 'arb. u.', 'ylim': None, 'hide_y_ticklabels': True, 'hide_y_ticks': True, + } + + options = aux.update_options(options=options, default_options=default_options) + + fig, ax = btp.prepare_plot(options=options) + + + spectrum = io.read_spectrum(data['path']) + + if options['deconvolutions']: + + deconvolutions = [] + if not isinstance(options['deconvolutions'], list): + options['deconvolutions'] = [options['deconvolutions']] + + if options['colours'] and (len(options['colours']) != len(options['deconvolutions'])): + options['colours'] = None + + for deconv in options['deconvolutions']: + df = io.read_spectrum(deconv) + deconvolutions.append(df) + + + + spectrum.plot(x='Energy', y='Counts', ax=ax, color='black') + + if options['deconvolutions']: + if options['colours']: + for deconv, colour in zip(deconvolutions, options['colours']): + ax.fill_between(x=deconv['Energy'], y1=deconv['Counts'], y2=0, color=colour, alpha=0.4) + else: + for deconv in deconvolutions: + ax.fill_between(x=deconv['Energy'], y1=deconv['Counts'], y2=0, alpha=0.4) + + + if not options['xlim']: + options['xlim'] = [spectrum['Energy'].min(), spectrum['Energy'].max()] + + if not options['ylim']: + options['ylim'] = [0, 1.1*spectrum['Counts'].max()] + + if options['lines']: + for i, (line, energy) in enumerate(options['lines'].items()): + ax.axvline(x=energy, ls='--', lw=0.5, c='black') + ax.text(s=line, x=energy, y=(0.9-0.1*i)*options['ylim'][1], fontsize=8) + + + + fig, ax = btp.adjust_plot(fig=fig, ax=ax, options=options) + + + return spectrum, fig, ax \ No newline at end of file From 2265b2a69e8746f65cf50a5983e814a4fa5b9b51 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sat, 15 Oct 2022 17:16:34 +0200 Subject: [PATCH 328/355] Update to use new version of update_options --- nafuma/electrochemistry/plot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index f8ef088..ea97681 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -23,7 +23,6 @@ def plot_gc(data, options=None): # Update options - required_options = ['force_reload', 'x_vals', 'y_vals', 'which_cycles', 'limit', 'exclude_cycles', 'show_plot', 'summary', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] default_options = { 'force_reload': False, 'x_vals': 'capacity', 'y_vals': 'voltage', @@ -42,10 +41,11 @@ def plot_gc(data, options=None): 'format_params': {}, 'save_gif': False, 'save_path': 'animation.gif', - 'fps': 1 + 'fps': 1, + 'fig': None, 'ax': None } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) # Read data if not already loaded From f8f5e2ebb2f0bb37890258fe01b1c2821b07c82c Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sat, 15 Oct 2022 17:16:54 +0200 Subject: [PATCH 329/355] Reset index of cycles to start at 0 --- nafuma/electrochemistry/io.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index d46a1d5..9b57f38 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -357,6 +357,12 @@ def process_neware_data(df, options={}): max_capacity = dchg_df['ions'].max() dchg_df['ions'] = np.abs(dchg_df['ions'] - max_capacity) + + if not chg_df.empty: + chg_df.reset_index(inplace=True) + if not dchg_df.empty: + dchg_df.reset_index(inplace=True) + cycles.append((chg_df, dchg_df)) From ed2bb8158e2904d04a1ef182b71ea8781fb23e74 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 16 Oct 2022 19:19:37 +0200 Subject: [PATCH 330/355] Allow specification of colours for several cycles --- nafuma/electrochemistry/plot.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index ea97681..c32c8a1 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -74,13 +74,12 @@ def plot_gc(data, options=None): else: fig, ax = options['fig'], options['ax'] - for i, cycle in enumerate(data['cycles']): - if i in options['which_cycles']: + for i, cycle in enumerate(options['which_cycles']): if options['charge']: - cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + data['cycles'][cycle][0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) if options['discharge']: - cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + data['cycles'][cycle][1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) if options['interactive_session_active']: @@ -159,11 +158,11 @@ def plot_gc(data, options=None): else: if options['charge']: yval = 'charge_' + options['x_vals'] - data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']*10) if options['discharge']: yval = 'discharge_' + options['x_vals'] - data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']*10) if options['limit']: @@ -611,17 +610,25 @@ def generate_colours(cycles, options): + # Generate lists of colours colours = [] + + if len(charge_colour) != len(options['which_cycles']): + + for cycle_number in range(0, len(options['which_cycles'])): + if options['gradient']: + weight_start = ((len(options['which_cycles'])) - cycle_number)/(len(options['which_cycles'])) + weight_end = cycle_number/len(options['which_cycles']) - for cycle_number in range(0, len(cycles)): - if options['gradient']: - weight_start = (len(cycles) - cycle_number)/len(cycles) - weight_end = cycle_number/len(cycles) + charge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(charge_colour_start, charge_colour_end)] + discharge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(discharge_colour_start, discharge_colour_end)] + + colours.append([charge_colour, discharge_colour]) - charge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(charge_colour_start, charge_colour_end)] - discharge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(discharge_colour_start, discharge_colour_end)] - colours.append([charge_colour, discharge_colour]) + else: + for charge, discharge in zip(charge_colour, discharge_colour): + colours.append([charge, discharge]) return colours From 97a82353a643f44ec3f50ffc8dee6a477b5ccf63 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 17 Oct 2022 19:34:23 +0200 Subject: [PATCH 331/355] Remove print-statement --- nafuma/auxillary.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index c01940c..9d30948 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -182,6 +182,4 @@ def find_neighbours(value, df, colname, start=0, end=-1): lowerneighbour_ind = lower_df.idxmax() upperneighbour_ind = upper_df.idxmin() - print(lowerneighbour_ind, upperneighbour_ind) - return [lowerneighbour_ind, upperneighbour_ind] \ No newline at end of file From 2e9b5e5bc066093d8440e9895278d49774a55f20 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 17 Oct 2022 19:34:39 +0200 Subject: [PATCH 332/355] Add colour mixer --- nafuma/plotting.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index 6d9bf09..c0105b5 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -485,4 +485,30 @@ def make_animation(paths, options={}): frames[0].save(os.path.join(options['save_folder'], options['save_filename']), format='GIF', append_images=frames[1:], save_all=True, duration=(1/options['fps'])*1000, loop=0) - \ No newline at end of file + + + +def mix_colours(colour1, colour2, options): + + default_options = { + 'number_of_colours': 10, + 'weights': None + } + + options = aux.update_options(options=options, default_options=default_options) + + if not options['weights']: + options['weights'] = [x/options['number_of_colours'] for x in range(options['number_of_colours'])] + + colours = [] + for weight in options['weights']: + colour = [] + + for c1, c2 in zip(colour1, colour2): + colour.append(np.round(((1-weight)*c1 + weight*c2), 5)) + + colours.append(colour) + + + return colours + \ No newline at end of file From c02b575b54f375de354b070e6bc994685381f3f9 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 17 Oct 2022 19:35:48 +0200 Subject: [PATCH 333/355] Allow looping through cycles if max is 0 --- nafuma/electrochemistry/io.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 9b57f38..3bc93d0 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -41,11 +41,10 @@ def read_neware(path, options={}): # Convert from .xlsx to .csv to make readtime faster if path.endswith('xlsx'): csv_details = ''.join(path[:-5]) + '_details.csv' - csv_summary = ''.join(path[:-5]) + '_summary.csv' + csv_summary = os.path.abspath(''.join(path[:-5]) + '_summary.csv') - if not os.path.isfile(csv_summary): - Xlsx2csv(path, outputencoding="utf-8").convert(csv_summary, sheetid=3) + Xlsx2csv(path, outputencoding="utf-8").convert(os.path.abspath(csv_summary), sheetid=3) if not os.path.isfile(csv_details): Xlsx2csv(path, outputencoding="utf-8").convert(csv_details, sheetid=4) @@ -432,15 +431,17 @@ def process_biologic_data(df, options=None): df = df[headers].copy() + # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. set_units(options) options['old_units'] = get_old_units(df=df, options=options) df = add_columns(df=df, options=options) - df = unit_conversion(df=df, options=options) + df = unit_conversion(df=df, options=options) + # Creates masks for charge and discharge curves if options['mode'] == 'GC': chg_mask = (df['status'] == 1) & (df['status_change'] != 1) @@ -453,8 +454,13 @@ def process_biologic_data(df, options=None): # Initiate cycles list cycles = [] + if df['cycle'].max() == 0: + no_cycles = 1 + else: + no_cycles = int(df['cycle'].max()) + # Loop through all the cycling steps, change the current and capacities in the - for i in range(int(df["cycle"].max())): + for i in range(no_cycles): sub_df = df.loc[df['cycle'] == i].copy() From 940e794a1c07335fabf3c3c4301409ec32fba261 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 17 Oct 2022 19:36:16 +0200 Subject: [PATCH 334/355] Improve handling of colour and other minor fixes --- nafuma/electrochemistry/plot.py | 118 +++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 34 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index c32c8a1..322b782 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -33,6 +33,7 @@ def plot_gc(data, options=None): 'summary': False, 'charge': True, 'discharge': True, 'colours': None, + 'markers': None, 'differentiate_charge_discharge': True, 'gradient': False, 'interactive': False, @@ -42,7 +43,9 @@ def plot_gc(data, options=None): 'save_gif': False, 'save_path': 'animation.gif', 'fps': 1, - 'fig': None, 'ax': None + 'fig': None, 'ax': None, + 'edgecolor': plt.rcParams['lines.markeredgecolor'], + 'plot_every': 1, } options = aux.update_options(options=options, default_options=default_options) @@ -52,7 +55,6 @@ def plot_gc(data, options=None): if not 'cycles' in data.keys() or options['force_reload']: data['cycles'] = ec.io.read_data(data=data, options=options) - # Update list of cycles to correct indices update_cycles_list(data=data, options=options) @@ -63,7 +65,9 @@ def plot_gc(data, options=None): - colours = generate_colours(cycles=data['cycles'], options=options) + colours = generate_colours(options=options) + markers = generate_markers(options=options) + if not options['summary']: if options['show_plot']: @@ -74,6 +78,7 @@ def plot_gc(data, options=None): else: fig, ax = options['fig'], options['ax'] + for i, cycle in enumerate(options['which_cycles']): if options['charge']: data['cycles'][cycle][0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) @@ -133,7 +138,10 @@ def plot_gc(data, options=None): elif options['summary'] and options['show_plot']: # Prepare plot - fig, ax = btp.prepare_plot(options=options) + if not options['fig'] and not options['ax']: + fig, ax = btp.prepare_plot(options=options) + else: + fig, ax = options['fig'], options['ax'] mask = [] for i in range(data['cycles'].shape[0]): @@ -151,18 +159,18 @@ def plot_gc(data, options=None): # FIXME To begin, the default is that y-values correspond to x-values. This should probably be implemented in more logical and consistent manner in the future. if options['x_vals'] in ['coulombic_efficiency', 'energy_efficiency']: - data['cycles'].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']) + data['cycles'].loc[mask].plot(x='cycle', y=options['x_vals'], ax=ax, color=colours[0][1], kind='scatter', s=plt.rcParams['lines.markersize']*10, marker=markers[0], edgecolor=plt.rcParams['lines.markeredgecolor']) if options['limit']: ax.axhline(y=options['limit'], ls='--', c='black') else: if options['charge']: yval = 'charge_' + options['x_vals'] - data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']*10) + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][0], kind='scatter', s=plt.rcParams['lines.markersize']*10, marker=markers[0], edgecolor=plt.rcParams['lines.markeredgecolor']) if options['discharge']: yval = 'discharge_' + options['x_vals'] - data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', marker="$\u25EF$", s=plt.rcParams['lines.markersize']*10) + data['cycles'].loc[mask].plot(x='cycle', y=yval, ax=ax, color=colours[0][1], kind='scatter', s=plt.rcParams['lines.markersize']*10, marker=markers[1], edgecolor=plt.rcParams['lines.markeredgecolor']) if options['limit']: @@ -233,18 +241,26 @@ def plot_cv(data, options): # Update list of cycles to correct indices update_cycles_list(data=data, options=options) - colours = generate_colours(cycles=data['cycles'], options=options) + colours = generate_colours(options=options) if options['show_plot']: # Prepare plot fig, ax = btp.prepare_plot(options=options) - for i, cycle in enumerate(data['cycles']): - if i in options['which_cycles']: - if options['charge']: - cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) - if options['discharge']: - cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + for i, cycle in enumerate(options['which_cycles']): + if options['charge']: + data['cycles'][cycle][0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + + if options['discharge']: + data['cycles'][cycle][1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) + + # for i, cycle in enumerate(data['cycles']): + # if i in options['which_cycles']: + # if options['charge']: + # cycle[0].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][0]) + + # if options['discharge']: + # cycle[1].plot(x=options['x_vals'], y=options['y_vals'], ax=ax, c=colours[i][1]) update_labels(options) @@ -260,6 +276,7 @@ def plot_cv(data, options): options['format_params']['dpi'] = 200 + for i, cycle in enumerate(data['cycles']): if i in options['which_cycles']: @@ -359,13 +376,15 @@ def update_cycles_list(data, options: dict) -> None: options['which_cycles'] = [i-1 for i in range(which_cycles[0], which_cycles[1]+1)] - for i, cycle in enumerate(options['which_cycles']): if cycle in options['exclude_cycles']: del options['which_cycles'][i] + options['which_cycles'] = options['which_cycles'][::options['plot_every']] + + def prettify_gc_plot(fig, ax, options=None): @@ -580,16 +599,27 @@ def prettify_labels(label): -def generate_colours(cycles, options): +def generate_colours(options): + + default_options = { + 'gradient_colours': None, + } + + aux.update_options(options=options, default_options=default_options) # Assign colours from the options dictionary if it is defined, otherwise use standard colours. if options['colours']: charge_colour = options['colours'][0] discharge_colour = options['colours'][1] + + if isinstance(charge_colour, tuple): + charge_colour = [charge_colour] + if isinstance(discharge_colour, tuple): + discharge_colour = [discharge_colour] else: - charge_colour = (40/255, 70/255, 75/255) # Dark Slate Gray #28464B, coolors.co - discharge_colour = (239/255, 160/255, 11/255) # Marigold #EFA00B, coolors.co + charge_colour = [(40/255, 70/255, 75/255)] # Dark Slate Gray #28464B, coolors.co + discharge_colour = [(239/255, 160/255, 11/255)] # Marigold #EFA00B, coolors.co if not options['differentiate_charge_discharge']: discharge_colour = charge_colour @@ -599,14 +629,19 @@ def generate_colours(cycles, options): # If gradient is enabled, find start and end points for each colour if options['gradient']: - add_charge = min([(1-x)*0.75 for x in charge_colour]) - add_discharge = min([(1-x)*0.75 for x in discharge_colour]) + if not options['gradient_colours']: - charge_colour_start = charge_colour - charge_colour_end = [x+add_charge for x in charge_colour] + options['gradient_colours'] = [[None, None], [None, None]] + + add_charge = min([(1-x)*0.75 for x in charge_colour]) + add_discharge = min([(1-x)*0.75 for x in discharge_colour]) + + options['gradient_colours'][0][0] = charge_colour + options['gradient_colours'][0][1] = [x+add_charge for x in charge_colour] + + options['gradient_colours'][1][0] = discharge_colour + options['gradient_colours'][1][1] = [x+add_discharge for x in discharge_colour] - discharge_colour_start = discharge_colour - discharge_colour_end = [x+add_discharge for x in discharge_colour] @@ -615,20 +650,35 @@ def generate_colours(cycles, options): colours = [] if len(charge_colour) != len(options['which_cycles']): + if options['gradient']: + options['number_of_colours'] = len(options['which_cycles']) - for cycle_number in range(0, len(options['which_cycles'])): - if options['gradient']: - weight_start = ((len(options['which_cycles'])) - cycle_number)/(len(options['which_cycles'])) - weight_end = cycle_number/len(options['which_cycles']) + charge_colours = btp.mix_colours(colour1=options['gradient_colours'][0][0], colour2=options['gradient_colours'][0][1], options=options) + discharge_colours = btp.mix_colours(colour1=options['gradient_colours'][1][0], colour2=options['gradient_colours'][1][1], options=options) - charge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(charge_colour_start, charge_colour_end)] - discharge_colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(discharge_colour_start, discharge_colour_end)] - - colours.append([charge_colour, discharge_colour]) + for chg, dchg in zip(charge_colours, discharge_colours): + colours.append([chg, dchg]) + + else: + for i in options['which_cycles']: + colours.append([charge_colour, discharge_colour]) else: - for charge, discharge in zip(charge_colour, discharge_colour): - colours.append([charge, discharge]) + for chg, dchg in zip(charge_colour, discharge_colour): + colours.append([chg, dchg]) + return colours + + + +def generate_markers(options): + + if not options['markers']: + markers = ['o', 'v'] + + else: + markers = [options['markers'][0], options['markers'][1]] + + return markers \ No newline at end of file From f23d14045327d1bcfaf6c19e1c868c2ce236b9ad Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Tue, 18 Oct 2022 19:23:24 +0200 Subject: [PATCH 335/355] Allow passing fig and ax to plot_cv() --- nafuma/electrochemistry/plot.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 322b782..9ab421f 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -209,7 +209,6 @@ def plot_gc_interactive(data, options): def plot_cv(data, options): # Update options - required_options = ['force_reload', 'x_vals', 'y_vals', 'which_cycles', 'limit', 'exclude_cycles', 'show_plot', 'charge', 'discharge', 'colours', 'differentiate_charge_discharge', 'gradient', 'interactive', 'interactive_session_active', 'rc_params', 'format_params', 'save_gif', 'save_path', 'fps'] default_options = { 'force_reload': False, 'x_vals': 'voltage', 'y_vals': 'current', @@ -227,10 +226,13 @@ def plot_cv(data, options): 'format_params': {}, 'save_gif': False, 'save_path': 'animation.gif', - 'fps': 1 + 'fps': 1, + 'plot_every': 1, + 'fig': None, + 'ax': None } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) # Read data if not already loaded @@ -245,7 +247,10 @@ def plot_cv(data, options): if options['show_plot']: # Prepare plot - fig, ax = btp.prepare_plot(options=options) + if not options['fig'] and not options['ax']: + fig, ax = btp.prepare_plot(options=options) + else: + fig, ax = options['fig'], options['ax'] for i, cycle in enumerate(options['which_cycles']): if options['charge']: From ecfe7106f1d37beb460c1904e020ec9449c4a4bc Mon Sep 17 00:00:00 2001 From: halvorhv Date: Wed, 19 Oct 2022 17:06:15 +0200 Subject: [PATCH 336/355] Add func for making general background in q --- nafuma/xrd/refinement.py | 54 ++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/nafuma/xrd/refinement.py b/nafuma/xrd/refinement.py index 921c1cf..8b0eb5b 100644 --- a/nafuma/xrd/refinement.py +++ b/nafuma/xrd/refinement.py @@ -103,6 +103,31 @@ def make_initial_inp(data: dict, options={}): for line in lines: fout.write(line) +def make_general_background_in_q(data, options): + #Function where you get a txt.-file of q-values, based on the 2th values you have picked out manually from a data set. Idea is that the same background points can be used for several samples, measured at several wavelengths. + #The input is a list of 2th-values that the user has found by manually inspecting a data set + import numpy as np + import nafuma.xrd as xrd + import matplotlib.pyplot as plt + + default_options = { + 'save_dir': 'background', + 'filename': 'test' + } + + options = aux.update_options(options=options, default_options=default_options) + + list_of_2th_values = data['2th_values'] + + #x_background_points=list_of_2th_values + x_background_points_q=[] + for x in list_of_2th_values: + q=np.abs((4*np.pi/data['wavelength'])*np.sin(x/2 * np.pi/180)) + x_background_points_q.append(q) + + background_points=pd.DataFrame(x_background_points_q) + background_filename="C:/Users/halvorhv/OneDriveUiO/1_OrderPaper/analysis/exsitu/probing_ordering/refinements/"+options['filename']+"_background_points_in_q.txt" + background_points.to_csv(background_filename,index=None, header=None)#,index=None, header=None,sep=' ') def make_manual_background(data, options): #FIXME generalize this so it works properly @@ -113,7 +138,7 @@ def make_manual_background(data, options): default_options = { 'plot_background': True, - 'interval_length': 0.05, + 'interval_length': 0.05, #picking out the region to be fitted for each background point, bigger interval-length means a bigger region for each point. 'save_dir': 'background' } if "noheaders" in data['path'][0]: @@ -136,21 +161,34 @@ def make_manual_background(data, options): for q in df_backgroundpoints_q["background_q"]: twotheta=2*180/np.pi*np.arcsin(q*wavelength/(4*np.pi)) x_background_points.append(twotheta) - - fig,ax=plt.subplots(figsize=(20,20)) - diffractogram.plot(x="2th",y="I", kind="scatter",ax=ax) + intervallength=options['interval_length'] + background=pd.DataFrame() for i, x in enumerate(x_background_points): test=diffractogram.loc[(diffractogram["2th"]>x-intervallength) & (diffractogram["2th"] Date: Sun, 23 Oct 2022 13:47:26 +0200 Subject: [PATCH 337/355] Start correlation of ions to timestamps --- nafuma/xanes/io.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index ff7f911..4657006 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -400,6 +400,43 @@ def read_metadata(data: dict, options={}) -> dict: timestamps = new_times + # Match timestamps against electrochemistry-data + # TODO This could be generalised to match up against any other dataset with timestamps. + if 'cycles' in data.keys(): + ions = [] + i = 0 + for timestamp in timestamps: + if timestamp < 0: + ions.append(0) + + else: + closest_chg = aux.find_neighbours(value=timestamp, df=data['cycles'][i][0], colname='time') + closest_dchg = aux.find_neighbours(value=timestamp, df=data['cycles'][i][1], colname='time') + + if not isinstance(closest_chg, list): + closest_chg = [closest_chg, closest_chg] + if not isinstance(closest_dchg, list): + closest_dchg = [closest_dchg, closest_dchg] + + + if all([x==x for x in closest_chg]): + print(f'Charge, cycle {i}') + continue + + elif all([x==x for x in closest_dchg]): + print(f'Discharge, cycle {i}') + continue + + elif (closest_chg[1]!=closest_chg[1]) and (closest_dchg[0]!=closest_dchg[0]): + print('Rest step!') + continue + else: + print('Rest step, new cycle!') + i += 1 + + if i > len(data['cycles'])-1: + break + metadata = {'time': timestamps, 'temperature': temperatures} From d617e50cdafd2ab3d5b88b5bb6223cd4c4e4c9de Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 23 Oct 2022 13:48:02 +0200 Subject: [PATCH 338/355] Allow passing df that has no idxmax or idxmin --- nafuma/auxillary.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 9d30948..1cc6f24 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -179,7 +179,15 @@ def find_neighbours(value, df, colname, start=0, end=-1): lower_df = df[df[colname] < value][colname] upper_df = df[df[colname] > value][colname] - lowerneighbour_ind = lower_df.idxmax() - upperneighbour_ind = upper_df.idxmin() + + if not lower_df.empty: + lowerneighbour_ind = lower_df.idxmax() + else: + lowerneighbour_ind = np.nan + + if not upper_df.empty: + upperneighbour_ind = upper_df.idxmin() + else: + upperneighbour_ind = np.nan return [lowerneighbour_ind, upperneighbour_ind] \ No newline at end of file From 97ac3505de03d6042cbf38f48550ffc6ff10bd57 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 23 Oct 2022 13:48:26 +0200 Subject: [PATCH 339/355] Generalise colours of fill_between --- nafuma/dft/electrons.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nafuma/dft/electrons.py b/nafuma/dft/electrons.py index 1fd3092..d99bf71 100644 --- a/nafuma/dft/electrons.py +++ b/nafuma/dft/electrons.py @@ -317,7 +317,7 @@ def plot_pdos(data: dict, options={}): for j, orbital in enumerate(options['plot_orbitals'][i]): if options['fill']: - ax.fill_betweenx(y=data['pdos'][atom]['Energy'], x1=data['pdos'][atom][orbital], x2=0, color=options['orbital_colours'][i][j], alpha=0.5, ec=(0,0,0,0)) + ax.fill_betweenx(y=data['pdos'][atom]['Energy'], x1=data['pdos'][atom][orbital], x2=0, color=options['orbital_colours'][i][j], ec=(0,0,0,1)) else: ax.plot(data['pdos'][atom][orbital], data['pdos'][atom]['Energy'], color=options['orbital_colours'][i][j]) @@ -654,11 +654,11 @@ def prettify_dos_plot(fig, ax, options): if options['flip_xy']: # Switch all the x- and y-specific values - options = swap_values(dict=options, key1='xlim', key2='ylim') - options = swap_values(dict=options, key1='xunit', key2='yunit') - options = swap_values(dict=options, key1='xlabel', key2='ylabel') - options = swap_values(dict=options, key1='x_tick_locators', key2='y_tick_locators') - options = swap_values(dict=options, key1='hide_x_labels', key2='hide_y_labels') + options = aux.swap_values(dict=options, key1='xlim', key2='ylim') + options = aux.swap_values(dict=options, key1='xunit', key2='yunit') + options = aux.swap_values(dict=options, key1='xlabel', key2='ylabel') + options = aux.swap_values(dict=options, key1='x_tick_locators', key2='y_tick_locators') + options = aux.swap_values(dict=options, key1='hide_x_labels', key2='hide_y_labels') # Set labels on x- and y-axes if not options['hide_y_labels']: From 27f9d2d2e6f0d01ad4cc61ec773fd261db63a257 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 23 Oct 2022 13:48:45 +0200 Subject: [PATCH 340/355] Fix bug that calculated wrong molecular weight --- nafuma/electrochemistry/io.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index 3bc93d0..b1d3786 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -547,7 +547,10 @@ def add_columns(df, options): seconds_per_hour = 3600 # s h^-1 f = faradays_constant / seconds_per_hour * 1000.0 # [f] = mAh mol^-1 - molecular_weight = options['molecular_weight'] * unit_tables.mass()['g'].loc[options['units']['mass']] + molecular_weight = options['molecular_weight'] * unit_tables.mass()['g'].loc[options['old_units']['mass']] + + print(options['old_units']['capacity'], options['old_units']['mass']) + df["IonsExtracted"] = (df[f'C [{options["old_units"]["capacity"]}/{options["old_units"]["mass"]}]'] * molecular_weight)/f From f7ba19b103e7b7dc5d839325fc60f607c0186145 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Sun, 23 Oct 2022 18:04:56 +0200 Subject: [PATCH 341/355] Fix small bug in colour generation --- nafuma/electrochemistry/plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 9ab421f..8c7f860 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -618,9 +618,9 @@ def generate_colours(options): discharge_colour = options['colours'][1] if isinstance(charge_colour, tuple): - charge_colour = [charge_colour] + charge_colour = list(charge_colour) if isinstance(discharge_colour, tuple): - discharge_colour = [discharge_colour] + discharge_colour = list(discharge_colour) else: charge_colour = [(40/255, 70/255, 75/255)] # Dark Slate Gray #28464B, coolors.co From cefd7a2eddb83f4ff6767fca28aab6680cecb5f7 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 23 Oct 2022 20:23:59 +0200 Subject: [PATCH 342/355] Add correlation between ions extracted and time --- nafuma/xanes/io.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 4657006..4b447a7 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -400,10 +400,12 @@ def read_metadata(data: dict, options={}) -> dict: timestamps = new_times + metadata = {'time': timestamps, 'temperature': temperatures} + # Match timestamps against electrochemistry-data # TODO This could be generalised to match up against any other dataset with timestamps. if 'cycles' in data.keys(): - ions = [] + ions, specific_capacity = [], [] i = 0 for timestamp in timestamps: if timestamp < 0: @@ -420,26 +422,46 @@ def read_metadata(data: dict, options={}) -> dict: if all([x==x for x in closest_chg]): - print(f'Charge, cycle {i}') + ions.append(np.mean([data['cycles'][i][0]['ions'].loc[data['cycles'][i][0].index == closest_chg[0]], data['cycles'][i][0]['ions'].loc[data['cycles'][i][0].index == closest_chg[1]]])) + specific_capacity.append(np.mean([data['cycles'][i][0]['specific_capacity'].loc[data['cycles'][i][0].index == closest_chg[0]], data['cycles'][i][0]['specific_capacity'].loc[data['cycles'][i][0].index == closest_chg[1]]])) continue elif all([x==x for x in closest_dchg]): - print(f'Discharge, cycle {i}') + ions.append(np.mean([data['cycles'][i][1]['ions'].loc[data['cycles'][i][1].index == closest_dchg[0]], data['cycles'][i][1]['ions'].loc[data['cycles'][i][1].index == closest_dchg[1]]])) + specific_capacity.append(np.mean([data['cycles'][i][1]['specific_capacity'].loc[data['cycles'][i][1].index == closest_dchg[0]], data['cycles'][i][1]['specific_capacity'].loc[data['cycles'][i][1].index == closest_dchg[1]]])) continue - elif (closest_chg[1]!=closest_chg[1]) and (closest_dchg[0]!=closest_dchg[0]): - print('Rest step!') + elif aux.isnan(closest_chg[1]) and aux.isnan(closest_dchg[0]): + ions.append(np.nan) + specific_capacity.append(np.nan) continue else: - print('Rest step, new cycle!') + ions.append(np.nan) + specific_capacity.append(np.nan) i += 1 if i > len(data['cycles'])-1: break + for i, (ion, cap) in enumerate(zip(ions, specific_capacity)): + if aux.isnan(ion): # if a resting step, assign a meaningful value + if i < len(ions)-1: # if resting step in the middle of the run, take the mean between the last of previous and first of next run + ions[i] = np.mean([ions[i-1], ions[i+1]]) + + else: # If last element, set to last values plus the delta between the last two previous measurements + ions[i] = ions[i-1] + (ions[i-1]-ions[i-2]) + + if aux.isnan(cap) and i < len(specific_capacity)-1: # do same thing for specific capacity + if i < len(specific_capacity)-1: + specific_capacity[i] = np.mean([specific_capacity[i-1], specific_capacity[i+1]]) + + else: + specific_capacity[i] = specific_capacity[i-1] + (specific_capacity[i-1]-specific_capacity[i-2]) - metadata = {'time': timestamps, 'temperature': temperatures} + metadata['ions'] = ions + metadata['specific_capacity'] = specific_capacity + return metadata From 686ef6ce2863ea455c599ee8484ebc822f0c2288 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 23 Oct 2022 20:24:31 +0200 Subject: [PATCH 343/355] Fix index return type of exact match and add isnan --- nafuma/auxillary.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 1cc6f24..8e428b5 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -174,7 +174,7 @@ def find_neighbours(value, df, colname, start=0, end=-1): exactmatch = df[df[colname] == value] if not exactmatch.empty: - return exactmatch.index + return exactmatch.index.values[0] else: lower_df = df[df[colname] < value][colname] upper_df = df[df[colname] > value][colname] @@ -190,4 +190,9 @@ def find_neighbours(value, df, colname, start=0, end=-1): else: upperneighbour_ind = np.nan - return [lowerneighbour_ind, upperneighbour_ind] \ No newline at end of file + return [lowerneighbour_ind, upperneighbour_ind] + + +def isnan(value): + + return value!=value \ No newline at end of file From adfed84526fa3b954cf70513227f7c1b44eea501 Mon Sep 17 00:00:00 2001 From: rasmusvt Date: Sun, 23 Oct 2022 20:24:59 +0200 Subject: [PATCH 344/355] Add possibilty to append ion count --- nafuma/electrochemistry/io.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/nafuma/electrochemistry/io.py b/nafuma/electrochemistry/io.py index b1d3786..884cc39 100644 --- a/nafuma/electrochemistry/io.py +++ b/nafuma/electrochemistry/io.py @@ -126,17 +126,18 @@ def process_batsmall_data(df, options=None): Output: cycles: A list with ''' - - required_options = ['splice_cycles', 'molecular_weight', 'reverse_discharge', 'units'] default_options = { - 'splice_cycles': False, + 'splice_cycles': False, + 'append': False, # Add max of ions and specific_capacity of previous run #TODO Generalise + 'append_gap': 0, # Add a gap between cyclces - only used if append == True. 'molecular_weight': None, 'reverse_discharge': False, - 'units': None} + 'units': None, + } - aux.update_options(options=options, required_options=required_options, default_options=default_options) + aux.update_options(options=options, default_options=default_options) options['kind'] = 'batsmall' # Complete set of new units and get the units used in the dataset, and convert values in the DataFrame from old to new. @@ -171,6 +172,9 @@ def process_batsmall_data(df, options=None): sub_df.loc[dchg_mask, 'current'] *= -1 sub_df.loc[dchg_mask, 'specific_capacity'] *= -1 + sub_df.loc[dchg_mask, 'ions'] *= -1 + + chg_df = sub_df.loc[chg_mask] dchg_df = sub_df.loc[dchg_mask] @@ -179,8 +183,11 @@ def process_batsmall_data(df, options=None): if chg_df.empty and dchg_df.empty: continue - chg_df['reaction_coordinate'] = chg_df['time'] * np.abs(chg_df['current'].mean()) - dchg_df['reaction_coordinate'] = dchg_df['time'] * np.abs(dchg_df['current'].mean()) + if options['append']: + if cycles: + chg_df.loc[chg_mask, 'ions'] += cycles[-1][1]['ions'].max() + options['append_gap'] + + dchg_df.loc[dchg_mask, 'ions'] += chg_df['ions'].max() + options['append_gap'] if options['reverse_discharge']: max_capacity = dchg_df['capacity'].max() @@ -549,8 +556,6 @@ def add_columns(df, options): molecular_weight = options['molecular_weight'] * unit_tables.mass()['g'].loc[options['old_units']['mass']] - print(options['old_units']['capacity'], options['old_units']['mass']) - df["IonsExtracted"] = (df[f'C [{options["old_units"]["capacity"]}/{options["old_units"]["mass"]}]'] * molecular_weight)/f From e6243d4d38bccf05d1b434ed1a1f78e91938b102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Sun, 23 Oct 2022 18:57:42 +0000 Subject: [PATCH 345/355] Remove old version of swap_values --- nafuma/auxillary.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/nafuma/auxillary.py b/nafuma/auxillary.py index 8e428b5..4931a79 100644 --- a/nafuma/auxillary.py +++ b/nafuma/auxillary.py @@ -50,16 +50,6 @@ def load_options(path): -def swap_values(dict, key1, key2): - - key1_val = dict[key1] - dict[key1] = dict[key2] - dict[key2] = key1_val - - return dict - - - def ceil(a, roundto=1): fac = 1/roundto @@ -195,4 +185,4 @@ def find_neighbours(value, df, colname, start=0, end=-1): def isnan(value): - return value!=value \ No newline at end of file + return value!=value From fd73dc577f93ed67302badbfd56ce452eb577a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Vester=20Th=C3=B8gersen?= <34004462+rasmusthog@users.noreply.github.com> Date: Sun, 23 Oct 2022 18:59:00 +0000 Subject: [PATCH 346/355] Fix test issues --- nafuma/test/test_auxillary.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/nafuma/test/test_auxillary.py b/nafuma/test/test_auxillary.py index 4cb9cd1..3425814 100644 --- a/nafuma/test/test_auxillary.py +++ b/nafuma/test/test_auxillary.py @@ -4,16 +4,16 @@ import os def test_swap_values(): - dict = {'test1': 1, 'test2': 2} + test_dict = {'test1': 1, 'test2': 2} key1 = 'test1' key2 = 'test2' - oldval1 = dict[key1] - oldval2 = dict[key2] + oldval1 = test_dict[key1] + oldval2 = test_dict[key2] - new_dict = aux.swap_values(dict=dict, key1=key1, key2=key2) + new_dict = aux.swap_values(options=test_dict, key1=key1, key2=key2) - assert (dict[key1] == oldval2) and (dict[key2] == oldval1) + assert (test_dict[key1] == oldval2) and (test_dict[key2] == oldval1) def test_ceil() -> None: @@ -35,7 +35,7 @@ def test_options() -> None: options = {} - required_options = ['test1', 'test2', 'test3', 'test4'] + default_options = { 'test1': 1, 'test2': 2, @@ -45,11 +45,9 @@ def test_options() -> None: } - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) assert options['test1'] == default_options['test1'] - assert len(options.items()) == len(required_options) - assert 'test5' not in options.keys() def test_save_options() -> None: @@ -75,4 +73,4 @@ def test_load_options() -> None: assert (loaded_options['test1'] == 1) and (loaded_options['test2'] == 2) - os.remove(path) \ No newline at end of file + os.remove(path) From 8fafdefc715adb0f8fd84ac4860e35f778059885 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 24 Oct 2022 20:58:22 +0200 Subject: [PATCH 347/355] Change which chamber is used for transmission --- nafuma/xanes/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nafuma/xanes/io.py b/nafuma/xanes/io.py index 4b447a7..30b6759 100644 --- a/nafuma/xanes/io.py +++ b/nafuma/xanes/io.py @@ -301,7 +301,8 @@ def read_data(data: dict, options={}) -> pd.DataFrame: required_options = ['adjust', 'mode'] default_options = { 'adjust': 0, - 'mode': 'fluoresence' + 'mode': 'fluoresence', + 'active_roi': None } options = aux.update_options(options=options, required_options=required_options, default_options=default_options) @@ -328,7 +329,7 @@ def read_data(data: dict, options={}) -> pd.DataFrame: scan_data = scan_data[options['active_roi']] elif options['mode'] == 'transmission': - scan_data = scan_data['MonEx'] / scan_data['Ion2'] + scan_data = scan_data['MonEx'] / scan_data['Ion1'] xanes_data = pd.concat([xanes_data, scan_data], axis=1) From 8d1bee56be9631c0b809068d7b8f08562c24bd26 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 24 Oct 2022 20:58:39 +0200 Subject: [PATCH 348/355] Add functions to assign tickmarks --- nafuma/electrochemistry/plot.py | 70 ++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/nafuma/electrochemistry/plot.py b/nafuma/electrochemistry/plot.py index 8c7f860..cb4c045 100644 --- a/nafuma/electrochemistry/plot.py +++ b/nafuma/electrochemistry/plot.py @@ -599,11 +599,6 @@ def prettify_labels(label): - - - - - def generate_colours(options): default_options = { @@ -686,4 +681,67 @@ def generate_markers(options): else: markers = [options['markers'][0], options['markers'][1]] - return markers \ No newline at end of file + return markers + + + +def get_tickmarks(df: pd.DataFrame, ticks: list, value: str, exclude=None): + + + min_val = df[value].min() + max_val = df[value].max() + + + # Get major ticks + major_ticks = [np.round((min_val + ticks[0]*i),2) for i in range(int(np.floor((max_val-min_val)/ticks[0]))+1)] + major_ticks.append(np.round(max_val, 2)) + + major_ticks = aux.get_unique(major_ticks) + + major_ticklabels = [i*ticks[0] for i in range(len(major_ticks)-1)] + major_ticklabels.append(np.round((max_val-min_val),1)) + + if exclude: + for i, tick in enumerate(major_ticklabels): + if tick in exclude: + del major_ticks[i] + del major_ticklabels[i] + + + # Get minor ticks + minor_ticks = [np.round((min_val + ticks[1]*i),2) for i in range(int(np.floor((max_val-min_val)/ticks[1]))+1) if np.round((min_val + ticks[1]*i),2) not in major_ticks] + minor_ticklabels = [np.round(tick - min_val, 2) for tick in minor_ticks] + + return major_ticks, major_ticklabels, minor_ticks, minor_ticklabels + + + +def assign_tickmarks(dfs: list, options, fig, ax, exclude=None): + + major_ticks, major_ticklabels, minor_ticks = [], [], [] + + if not exclude: + exclude = [[None, None] for i in range(len(options['which_cycles']))] + + for i, cycle in enumerate(options['which_cycles']): + #Get ticks from charge cycle + major_tick, major_ticklabel, minor_tick, minor_ticklabel = ec.plot.get_tickmarks(dfs[cycle][0], ticks=options['x_tick_locators'], value=options['x_vals'], exclude=exclude[i][0]) + major_ticks += major_tick + major_ticklabels += major_ticklabel + minor_ticks += minor_tick + + # Get ticks from discharge cycle + major_tick, major_ticklabel, minor_tick, minor_ticklabel = ec.plot.get_tickmarks(dfs[cycle][1], ticks=[1, 0.25], value='ions', exclude=exclude[i][1]) + major_ticks += major_tick + major_ticklabels += major_ticklabel + minor_ticks += minor_tick + + + ax.set_xticks(major_ticks, minor=False) + ax.set_xticklabels(major_ticklabels) + ax.set_xticks(minor_ticks, minor=True) + + + + + return fig, ax \ No newline at end of file From 2b57b2ce8608ea1b9e763b0d493abf7b7c23f21c Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Tue, 25 Oct 2022 15:15:05 +0200 Subject: [PATCH 349/355] Make small adjustment to Fe-interval --- nafuma/xanes/calib.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nafuma/xanes/calib.py b/nafuma/xanes/calib.py index 2df8528..3e3eea7 100644 --- a/nafuma/xanes/calib.py +++ b/nafuma/xanes/calib.py @@ -26,7 +26,7 @@ def find_element(data: dict, index=0) -> str: element_energy_intervals = { 'Mn': [5.9, 6.5], - 'Fe': [7.0, 7.2], + 'Fe': [6.9, 7.2], 'Co': [7.6, 7.8], 'Ni': [8.0, 8.6] } @@ -86,8 +86,6 @@ def pre_edge_fit(data: dict, options={}) -> pd.DataFrame: edge_position = estimate_edge_position(data, options, index=0) options['pre_edge_limits'][1] = edge_position - pre_edge_limit_offset - print(edge_position) - if options['pre_edge_limits'][0] >= options['pre_edge_limits'][1]: options['pre_edge_limits'][1] = options['pre_edge_limits'][0] + 0.03 From eb5116fb1dbf777d621565816388a4b9dde1c7e6 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Tue, 25 Oct 2022 15:15:35 +0200 Subject: [PATCH 350/355] Add force reset of ylim when plotting backgrounds --- nafuma/plotting.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nafuma/plotting.py b/nafuma/plotting.py index c0105b5..0231ad9 100644 --- a/nafuma/plotting.py +++ b/nafuma/plotting.py @@ -113,6 +113,7 @@ def adjust_plot(fig, ax, options): 'rotation_x_ticks': 0, 'rotation_y_ticks': 0, # Degrees the x- and/or y-ticklabels should be rotated 'xticks': None, 'yticks': None, # Custom definition of the xticks and yticks. This is not properly implemented now. 'xlim': None, 'ylim': None, # Limits to the x- and y-axes + 'xlim_reset': False, 'ylim_reset': False, # For use in setting limits of backgrounds - forcing reset of xlim and ylim, useful when more axes 'title': None, # Title of the plot 'backgrounds': [], 'legend': False, 'legend_position': ['lower center', (0.5, -0.1)], 'legend_ncol': 1, # Toggles on/off legend. Specifices legend position and the number of columns the legend should appear as. @@ -276,6 +277,11 @@ def adjust_plot(fig, ax, options): background = aux.update_options(options=background, default_options=default_background_options) + if options['xlim_reset']: + background['xlim'] = list(ax.get_xlim()) + if options['ylim_reset']: + background['ylim'] = list(ax.get_ylim()) + if not background['xlim'][0]: background['xlim'][0] = ax.get_xlim()[0] if not background['xlim'][1]: From 9c87ed0346973bab95fe7caf8745c88cd2191f8c Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Tue, 25 Oct 2022 20:04:04 +0200 Subject: [PATCH 351/355] Refactor slightly --- nafuma/xrd/plot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 6a11fd4..7ca1ea9 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -111,11 +111,12 @@ def plot_diffractogram(data, options={}): # Sets the xlim if this has not been specified if not options['xlim']: options['xlim'] = [data['diffractogram'][0][options['x_vals']].min(), data['diffractogram'][0][options['x_vals']].max()] - + # GENERATE HEATMAP DATA data['heatmap'], data['heatmap_xticks'], data['heatmap_xticklabels'], data['heatmap_yticks'], data['heatmap_yticklabels'] = generate_heatmap(data=data, options=options) options['heatmap_loaded'] = True + if options['heatmap']: xlim_start_frac, xlim_end_frac = options['xlim'][0] / data['diffractogram'][0][options['x_vals']].max(), options['xlim'][1] / data['diffractogram'][0][options['x_vals']].max() options['xlim'] = [options['heatmap_xlim'][0]*xlim_start_frac, options['heatmap_xlim'][1]*xlim_end_frac] @@ -150,7 +151,7 @@ def plot_diffractogram(data, options={}): if options['offset']: if (options['offset_x'] != options['current_offset_x']) or (options['offset_y'] != options['current_offset_y']): for i, (diff, wl) in enumerate(zip(data['diffractogram'], data['wavelength'])): - xrd.io.apply_offset(diff, wl, i, options) + xrd.io.adjust_intensities(diff, wl, i, options) @@ -292,7 +293,7 @@ def plot_diffractogram(data, options={}): if options['highlight']: for i, highlight in enumerate(options['highlight']): if i < len(options['highlight']) or len(options['highlight']) == 1: - ax.axhline(y=highlight[1], c=options['highlight_colours'][i], ls='--', lw=0.5) + ax.axhline(y=highlight, c=options['highlight_colours'][i], ls='--', lw=0.5) # PLOT DIFFRACTOGRAM From 996b53195cbddeb8c94c2e663cf821905d260fbd Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Thu, 27 Oct 2022 19:23:11 +0200 Subject: [PATCH 352/355] Repair xticks and xlim for heatmaps --- nafuma/xrd/plot.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nafuma/xrd/plot.py b/nafuma/xrd/plot.py index 7ca1ea9..a984bc1 100644 --- a/nafuma/xrd/plot.py +++ b/nafuma/xrd/plot.py @@ -119,7 +119,7 @@ def plot_diffractogram(data, options={}): if options['heatmap']: xlim_start_frac, xlim_end_frac = options['xlim'][0] / data['diffractogram'][0][options['x_vals']].max(), options['xlim'][1] / data['diffractogram'][0][options['x_vals']].max() - options['xlim'] = [options['heatmap_xlim'][0]*xlim_start_frac, options['heatmap_xlim'][1]*xlim_end_frac] + options['xlim'] = [options['heatmap_xlim'][1]*xlim_start_frac, options['heatmap_xlim'][1]*xlim_end_frac] if options['heatmap_reverse']: data['heatmap'] = data['heatmap'].iloc[::-1] @@ -213,6 +213,10 @@ def plot_diffractogram(data, options={}): if not isinstance(options['highlight_colours'], list): options['highlight_colours'] = [options['highlight_colours']] + if options['heatmap_reverse']: + print(len(data['diffractogram'])) + options['highlight'] = [len(data['diffractogram'])-highlight for highlight in options['highlight']] + colours = [] # Loop through each scan - assign the correct colour to each of the scan intervals in options['highlight'] @@ -254,6 +258,8 @@ def plot_diffractogram(data, options={}): # PLOT HEATMAP if options['heatmap']: + options['x_tick_locators'] = None + # Add locators for y-axis - otherwise it will tend to break (too many ticks) when switching between diffractograms and heatmap in interactive mode. These values will be updated later anyway, and is only # to allow the initial call to Seaborn to have values that are sensible. # FIXME A more elegant solution to this? @@ -262,7 +268,6 @@ def plot_diffractogram(data, options={}): # Call Seaborn to plot the data sns.heatmap(data['heatmap'], cmap=options['cmap'], cbar=False, ax=ax) - # Set the ticks and ticklabels to match the data point number with 2th values ax.set_xticks(data['heatmap_xticks'][options['x_vals']]) @@ -397,7 +402,7 @@ def generate_heatmap(data, options={}): if d['I'].min() < 0: d['I'] = d['I'] - d['I'].min() - d['I'] = d['I']**(1/options['contrast_factor']) + d['I'] =d['I']**(1/options['contrast_factor']) twotheta = np.append(twotheta, d['2th'].to_numpy()) intensities = np.append(intensities, d['I'].to_numpy()) From d662d3567d2cdc75d4c18b614ee2eda57c1385ec Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Fri, 28 Oct 2022 16:03:06 +0200 Subject: [PATCH 353/355] Add function to strip regions from .xy-files --- nafuma/xrd/io.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 30e9ea7..2be187c 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -899,3 +899,37 @@ def translate_wavelengths(data: pd.DataFrame, wavelength: float, to_wavelength=N +def trim_xy_region(path, region): + + df = pd.read_csv(path, header=None, delim_whitespace=True) + df.columns = ['2th', 'I'] + + df = df.loc[(df['2th'] > region[0]) & (df['2th'] < region[1])] + + folder = os.path.dirname(path) + save_folder = os.path.join(folder, 'trimmed') + + if not os.path.exists(save_folder): + os.makedirs(save_folder) + + df.to_csv(os.path.join(save_folder, os.path.basename(path)), sep='\t', header=None, index=None) + + +def raise_intensities_xy(path, region=None): + + df = pd.read_csv(path, header=None, delim_whitespace=True) + df.columns = ['2th', 'I'] + + if region: + df = df.loc[(df['2th'] > region[0]) & (df['2th'] < region[1])] + + df['I'] = df['I'] - df['I'].min() + + + folder = os.path.dirname(path) + save_folder = os.path.join(folder, 'raised') + + if not os.path.exists(save_folder): + os.makedirs(save_folder) + + df.to_csv(os.path.join(save_folder, os.path.basename(path)), sep='\t', header=None, index=None) \ No newline at end of file From 1fc003c755559f3c6b2e73d038608de985603434 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Mon, 31 Oct 2022 20:56:05 +0100 Subject: [PATCH 354/355] Add highlight and better colour mixing --- nafuma/xanes/plot.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/nafuma/xanes/plot.py b/nafuma/xanes/plot.py index 78230c6..c98932f 100644 --- a/nafuma/xanes/plot.py +++ b/nafuma/xanes/plot.py @@ -18,9 +18,9 @@ def plot_xanes(data, options={}): # Update options - required_options = ['which_scans', 'xlabel', 'ylabel', 'xunit', 'yunit', 'exclude_scans', 'colours', 'gradient', 'rc_params', 'format_params'] default_options = { - 'which_scans': 'all', + 'which_scans': 'all', # Use real numbers, not indices - update_scans_list() will adjust. + 'highlight': [], 'xlabel': 'Energy', 'ylabel': 'Intensity', 'xunit': 'keV', 'yunit': 'arb. u.', 'exclude_scans': [], @@ -29,7 +29,7 @@ def plot_xanes(data, options={}): 'rc_params': {}, 'format_params': {}} - options = aux.update_options(options=options, required_options=required_options, default_options=default_options) + options = aux.update_options(options=options, default_options=default_options) if not 'xanes_data' in data.keys(): @@ -49,7 +49,9 @@ def plot_xanes(data, options={}): counter = 0 for i, path in enumerate(data['path']): if i in options['which_scans']: - data['xanes_data'].plot(x='ZapEnergy', y=path, ax=ax, c=colours[counter]) + lw = plt.rcParams['lines.linewidth']*5 if i in options['highlight'] else plt.rcParams['lines.linewidth'] + + data['xanes_data'].plot(x='ZapEnergy', y=path, ax=ax, c=colours[counter], lw=lw) counter += 1 @@ -151,22 +153,30 @@ def generate_colours(scans, options): # If gradient is enabled, find start and end points for each colour if options['gradient']: - add = min([(1-x)*0.75 for x in colour]) + if isinstance(colour, list) and len(colour) == 2: + options['number_of_colours'] = len(scans) + colours = btp.mix_colours(colour1=colour[0], colour2=colour[1], options=options) - colour_start = colour - colour_end = [x+add for x in colour] + + else: + add = min([(1-x)*0.75 for x in colour]) + + colour_start = colour + colour_end = [x+add for x in colour] + # Generate lists of colours - colours = [] - - for scan_number in range(0, len(scans)): - if options['gradient']: - weight_start = (len(scans) - scan_number)/len(scans) - weight_end = scan_number/len(scans) + if not isinstance(colour, list): + colours = [] + for scan_number in range(0, len(scans)): - colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(colour_start, colour_end)] + if options['gradient']: + weight_start = (len(scans) - scan_number)/len(scans) + weight_end = scan_number/len(scans) + + colour = [weight_start*start_colour + weight_end*end_colour for start_colour, end_colour in zip(colour_start, colour_end)] + + colours.append(colour) - colours.append(colour) - return colours \ No newline at end of file From 7dfc0444f0f9bbb6ef41cea9253332402572a953 Mon Sep 17 00:00:00 2001 From: rasmusthog Date: Thu, 3 Nov 2022 20:35:51 +0100 Subject: [PATCH 355/355] Allow different multiplication factor for each scan --- nafuma/xrd/io.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nafuma/xrd/io.py b/nafuma/xrd/io.py index 2be187c..0d3f16f 100644 --- a/nafuma/xrd/io.py +++ b/nafuma/xrd/io.py @@ -759,7 +759,11 @@ def adjust_intensities(diffractogram, wavelength, index, options): if options['normalise']: diffractogram['I'] = diffractogram['I'] / diffractogram['I'].max() - diffractogram['I'] = diffractogram['I'] * options['multiply'] + + if not isinstance(options['multiply'], list): + options['multiply'] = [options['multiply']] + + diffractogram['I'] = diffractogram['I'] * options['multiply'][index] if options['drawdown']: diffractogram['I'] = diffractogram['I'] - diffractogram['I'].mean()