Specify values as range and add error scaling (#187)

* Add support for ranges in IDS Operator * Implement scaling to error bounds on range operations * Update documentation * Remove IDSSampler * Add tests for IDS Operation * Delete file and fix test * Skip tests if imas is not available
duqtools · Jul 21, 2022 · 8007b1f · 8007b1f
1 parent bf324ac
commit 8007b1f
Show file tree

Hide file tree

Showing 16 changed files with 322 additions and 195 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -18,7 +18,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       fail-fast: false
-      dimensions:
+      matrix:
         os: ['ubuntu-latest']
         python-version: ['3.7']
 
@@ -42,7 +42,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install -e .[develop]
+          python -m pip install -e .[develop,dash]
 
       - name: Test with pytest
         run: python -m pytest
diff --git a/docs/gendocs.py b/docs/gendocs.py
@@ -49,12 +49,14 @@ def model2config(key: str, model) -> str:
     extra_schemas['plot_schema'] = PlotModel.schema()
 
 if 'create' in models:
-    from duqtools.schema import IDSOperationDim, IDSSamplerDim
+    from duqtools.schema import ARange, IDSOperationDim, LinSpace
 
     extra_schemas['ops_schema'] = IDSOperationDim.schema()
-    extra_schemas['sampler_schema'] = IDSSamplerDim.schema()
     extra_schemas['data_loc_schema'] = cfg.create.data.schema()
 
+    extra_schemas['linspace_schema'] = LinSpace.schema()
+    extra_schemas['arange_schema'] = ARange.schema()
+
     extra_yamls['data_loc_yaml'] = model2config('data', cfg.create.data)
 
 if 'introduction' in models:

diff --git a/docs/templates/template_create.md b/docs/templates/template_create.md
@@ -18,7 +18,7 @@ Check out [the command-line interface](/command-line-interface/#create) for more
 : {{ prop['description'] }}
 {% endfor %}
 
-### Example
+For example:
 
 ```yaml title="duqtools.yaml"
 {{ yaml_example }}
@@ -39,7 +39,7 @@ For example:
 {{ data_loc_yaml }}
 ```
 
-## IDS operations
+## Dimensions
 
 These instructions operate on the template model. Note that these are compound operations, so they are expanded to fill the matrix with possible entries for data modifications (depending on the sampling method).
 
@@ -78,24 +78,75 @@ With the default `sampler: latin-hypercube`, this means 9 new data files will be
 
     The python equivalent is essentially `np.<operator>(ids, value, out=ids)` for each of the given values.
 
-### Error bound sampling
+### Specifying value ranges
+
+Although it is possible to specify value ranges explicitly in an operator, sometimes it may be easier to specify a range.
+
+There are two ways to specify ranges in *duqtools*.
+
+#### By number of samples
+
+{{ linspace_schema['description'] }}
+
+{% for name, prop in linspace_schema['properties'].items() %}
+`{{ name }}`
+: {{ prop['description'] }}
+{% endfor %}
+
+This example generates a range from 0.7 to 1.3 with 10 steps:
+
+```yaml title="duqtools.yaml"
+ids: profiles_1d/0/t_i_average
+operator: multiply
+values:
+  start: 0.7
+  stop: 1.3
+  num: 10
+```
+
+#### By stepsize
 
-{{ sampler_schema['description'] }}
+{{ arange_schema['description'] }}
 
-{% for name, prop in sampler_schema['properties'].items() %}
+{% for name, prop in linspace_schema['properties'].items() %}
 `{{ name }}`
 : {{ prop['description'] }}
 {% endfor %}
 
-Example:
+This example generates a range from 0.7 to 1.3 with steps of 0.1:
+
+```yaml title="duqtools.yaml"
+ids: profiles_1d/0/t_i_average
+operator: multiply
+values:
+  start: 0.7
+  stop: 1.3
+  step: 0.1
+```
+
+### Sampling between error bounds
+
+The following example takes `electrons/temperature`, and generates a range from $-2\sigma$ to $+2\sigma$ with defined steps:
 
 ```yaml title="duqtools.yaml"
-ids: profiles_1d/0/q
-sampling: normal
-bounds: symmetric
-n_samples: 5
+ids: profiles_1d/0/electrons/temperature
+operator: add
+values: [-2, -1, 0, 1, 2]
+scale_to_error: True
+```
+
+The following example takes `t_i_average`, and generates a range from $-3\sigma$ to $+3\sigma$ with 10 equivalent steps:
+
+```yaml title="duqtools.yaml"
+ids: profiles_1d/0/t_i_average
+operator: add
+values:
+  start: -3
+  stop: 3
+  num: 10
+scale_to_error: True
 ```
 
 !!! note
 
-     Note that the example above adds 5 (`n_samples`) entries to the matrix. This is independent from the hypercube sampling above.
+    When specifying a sigma range, make sure you use `add` as the operator. While the other operators are also supported, they do not make much sense in this context.
diff --git a/duqtools/create.py b/duqtools/create.py
@@ -3,8 +3,7 @@
 
 from duqtools.config import cfg
 
-from .ids import IDSMapping, ImasHandle
-from .ids.dimensions import apply_model
+from .ids import IDSMapping, ImasHandle, apply_model
 from .matrix_samplers import get_matrix_sampler
 from .models import WorkDirectory
 from .schema.runs import Runs

diff --git a/duqtools/ids/__init__.py b/duqtools/ids/__init__.py
@@ -1,5 +1,6 @@
 import logging
 
+from ._dimensions import apply_model
 from ._get_ids_tree import get_ids_tree
 from ._handle import ImasHandle
 from ._mapping import IDSMapping
@@ -8,6 +9,7 @@
 
 __all__ = [
     'get_ids_tree',
+    'apply_model',
     'IDSMapping',
     'ImasHandle',
 ]
diff --git a/duqtools/ids/_dimensions.py b/duqtools/ids/_dimensions.py
@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import logging
+from functools import singledispatch
+
+import numpy as np
+
+from ..schema import BaseModel, IDSOperation
+from ._mapping import IDSMapping
+
+logger = logging.getLogger(__name__)
+
+
+@singledispatch
+def apply_model(model: BaseModel, ids_mapping: IDSMapping) -> None:
+    """Apply operation in model to IDS. Data are modified in-place.
+
+    Parameters
+    ----------
+    model
+        The model describes the operation to apply to the data.
+    ids_mapping : IDSMapping
+        Core profiles IDSMapping, data to apply operation to.
+        Must contain the IDS path.
+
+    Raises
+    ------
+    NotImplementedError
+        When the model is unknown
+    """
+
+    raise NotImplementedError(f'Unknown model: {model}')
+
+
+@apply_model.register
+def _(model: IDSOperation, ids_mapping: IDSMapping) -> None:
+
+    npfunc = getattr(np, model.operator)
+
+    profile = ids_mapping.flat_fields[model.ids]
+
+    if model.scale_to_error:
+        sigma_key = model.ids + model._upper_suffix
+
+        if model.value < 0:
+            lower_key = model.ids + model._lower_suffix
+            if lower_key in ids_mapping.flat_fields:
+                sigma_key = lower_key
+
+        sigma_bound = ids_mapping.flat_fields[sigma_key]
+        sigma = abs(sigma_bound - profile)
+
+        value = sigma * model.value
+    else:
+        value = model.value
+
+    logger.info('Apply %s', model)
+
+    logger.debug('data range before: %s - %s', profile.min(), profile.max())
+    npfunc(profile, value, out=profile)
+    logger.debug('data range after: %s - %s', profile.min(), profile.max())
diff --git a/duqtools/ids/dimensions.py b/duqtools/ids/dimensions.py
diff --git a/duqtools/schema/__init__.py b/duqtools/schema/__init__.py
@@ -1,15 +1,14 @@
 from ._basemodel import BaseModel
-from ._dimensions import (IDSOperation, IDSOperationDim, IDSSampler,
-                          IDSSamplerDim)
+from ._dimensions import ARange, IDSOperation, IDSOperationDim, LinSpace
 from ._imas import ImasBaseModel
 from ._plot import PlotModel
 
 __all__ = [
     'BaseModel',
     'IDSOperation',
     'IDSOperationDim',
-    'IDSSampler',
-    'IDSSamplerDim',
     'ImasBaseModel',
     'PlotModel',
+    'LinSpace',
+    'ARange',
 ]