duqtools · v1kko · Jul 25, 2022 · Jul 21, 2022 · Jul 21, 2022 · Jul 21, 2022
diff --git a/docs/operations.md b/docs/operations.md
@@ -0,0 +1,11 @@
+# Operations
+
+*Duqtools* uses delayed operations for filesystem-changing operations.
+They are implemented mostly with decorators, but a function could be added directly
+to the `op_queue`
+
+
+
+::: duqtools.operations
+    options:
+      show_source: false
diff --git a/duqtools/cleanup.py b/duqtools/cleanup.py
@@ -6,10 +6,12 @@
 from .config import cfg
 from .ids import ImasHandle
 from .models import WorkDirectory
+from .operations import confirm_operations, op_queue
 
 logger = logging.getLogger(__name__)
 
 
+@confirm_operations
 def cleanup(out, force, **kwargs):
     """Read runs.yaml and clean the current directory.
 
@@ -29,15 +31,15 @@ def cleanup(out, force, **kwargs):
         data_in = ImasHandle.parse_obj(run.data_in)
         data_out = ImasHandle.parse_obj(run.data_out)
 
-        logger.info('Removing %s', data_in)
         data_in.delete()
 
         if out:
-            logger.info('Removing %s', data_out)
             data_out.delete()
 
-        logger.info('Removing run dir %s', run.dirname.resolve())
-        shutil.rmtree(run.dirname)
+        op_queue.add(action=shutil.rmtree,
+                     args=(run.dirname, ),
+                     description=f'Removing run dir {run.dirname}')
 
-    logger.info('Moving %s', workspace.runs_yaml)
-    shutil.move(workspace.runs_yaml, workspace.runs_yaml_old)
+    op_queue.add(action=shutil.move,
+                 args=(workspace.runs_yaml, workspace.runs_yaml_old),
+                 description=f'Moving {workspace.runs_yaml}')
diff --git a/duqtools/cli.py b/duqtools/cli.py
@@ -3,7 +3,8 @@
 import click
 import coverage
 
-from duqtools.config import cfg
+from .config import cfg
+from .operations import op_queue
 
 logger = logging.getLogger(__name__)
 coverage.process_startup()
@@ -44,7 +45,9 @@ def dry_run_option(f):
     def callback(ctx, param, dry_run):
         if dry_run:
             logger.info('--dry-run enabled')
-            cfg.dry_run = True
+            op_queue.dry_run = True
+        else:
+            op_queue.dry_run = False
 
         return dry_run
 
@@ -54,8 +57,25 @@ def callback(ctx, param, dry_run):
                         callback=callback)(f)
 
 
+def yes_option(f):
+
+    def callback(ctx, param, yes):
+        if yes:
+            logger.info('--yes enabled')
+            op_queue.yes = True
+        else:
+            op_queue.yes = False
+        return yes
+
+    return click.option('--yes',
+                        '-y',
+                        is_flag=True,
+                        help='Answer yes to questions automatically.',
+                        callback=callback)(f)
+
+
 def common_options(func):
-    for wrapper in (debug_option, config_option, dry_run_option):
+    for wrapper in (debug_option, config_option, dry_run_option, yes_option):
         # config_option MUST BE BEFORE dry_run_option
         func = wrapper(func)
     return func

diff --git a/duqtools/create.py b/duqtools/create.py
@@ -6,6 +6,7 @@
 from .ids import IDSMapping, ImasHandle, apply_model
 from .matrix_samplers import get_matrix_sampler
 from .models import WorkDirectory
+from .operations import add_to_op_queue, confirm_operations, op_queue
 from .schema.runs import Runs
 from .system import get_system
 
@@ -27,7 +28,29 @@ def fail_if_locations_exist(locations: Iterable[ImasHandle]):
             'remove or `--force` to override.')
 
 
-def create(*, force, dry_run, **kwargs):
+@add_to_op_queue('Applying combination to {target_in}')
+def apply_combination(target_in: ImasHandle, combination) -> None:
+    core_profiles = target_in.get('core_profiles')
+    ids_mapping = IDSMapping(core_profiles)
+
+    for model in combination:
+        apply_model(model, ids_mapping)
+
+        logger.info('Writing data entry: %s', target_in)
+        with target_in.open() as data_entry_target:
+            core_profiles.put(db_entry=data_entry_target)
+
+
+@add_to_op_queue('Writing out {workspace.runs_yaml}')
+def write_runs_file(runs: list, workspace) -> None:
+
+    runs = Runs.parse_obj(runs)
+    with open(workspace.runs_yaml, 'w') as f:
+        runs.yaml(stream=f)
+
+
+@confirm_operations
+def create(*, force, **kwargs):
     """Create input for jetto and IDS data structures.
 
     Parameters
@@ -78,8 +101,13 @@ def create(*, force, dry_run, **kwargs):
     for i, combination in enumerate(combinations):
         run_name = f'{RUN_PREFIX}{i:04d}'
         run_drc = workspace.cwd / run_name
-        if not dry_run:
-            run_drc.mkdir(parents=True, exist_ok=force)
+
+        op_queue.add(action=run_drc.mkdir,
+                     kwargs={
+                         'parents': True,
+                         'exist_ok': force
+                     },
+                     description=f'Create folder {run_drc}')
 
         target_in = ImasHandle(db=options.data.db,
                                shot=source.shot,
@@ -90,16 +118,7 @@ def create(*, force, dry_run, **kwargs):
 
         source.copy_ids_entry_to(target_in)
 
-        core_profiles = target_in.get('core_profiles')
-        ids_mapping = IDSMapping(core_profiles)
-
-        if not dry_run:
-            for model in combination:
-                apply_model(model, ids_mapping)
-
-            logger.info('Writing data entry: %s', target_in)
-            with target_in.open() as data_entry_target:
-                core_profiles.put(db_entry=data_entry_target)
+        apply_combination(target_in, combination)
 
         system.copy_from_template(template_drc, run_drc)
         system.write_batchfile(workspace, run_name)
@@ -115,8 +134,4 @@ def create(*, force, dry_run, **kwargs):
             'operations': combination
         })
 
-    if not dry_run:
-        runs = Runs.parse_obj(runs)
-
-        with open(workspace.runs_yaml, 'w') as f:
-            runs.yaml(stream=f)
+    write_runs_file(runs, workspace)
diff --git a/duqtools/ids/_copy.py b/duqtools/ids/_copy.py
@@ -9,6 +9,7 @@
 from packaging import version
 
 from .._logging_utils import LoggingContext
+from ..operations import add_to_op_queue
 from ._imas import imas
 
 if TYPE_CHECKING:
@@ -50,6 +51,7 @@ def get_imas_ual_version():
     return imas_version, ual_version
 
 
+@add_to_op_queue('Create {target} from template')
 def copy_ids_entry(source: ImasHandle, target: ImasHandle):
     """Copies the ids entry to a new location.
 
@@ -78,29 +80,26 @@ def copy_ids_entry(source: ImasHandle, target: ImasHandle):
 
     idss_out = imas.ids(target.shot, target.run)
 
-    from ..config import cfg
-    if not cfg.dry_run:
-        idss_out.create_env(target.user, target.db, str(imas_version.major))
-        idx = idss_out.expIdx
+    idss_out.create_env(target.user, target.db, str(imas_version.major))
+    idx = idss_out.expIdx
 
-        parser = Parser.load_idsdef()
+    parser = Parser.load_idsdef()
 
-        # Temporarily hide warnings, because this loop is very spammy
-        with LoggingContext(level=logging.CRITICAL):
+    # Temporarily hide warnings, because this loop is very spammy
+    with LoggingContext(level=logging.CRITICAL):
 
-            for ids_info in parser.idss:
-                name = ids_info['name']
-                maxoccur = int(ids_info['maxoccur'])
+        for ids_info in parser.idss:
+            name = ids_info['name']
+            maxoccur = int(ids_info['maxoccur'])
 
-                if name in ('ec_launchers', 'numerics', 'sdn'):
-                    continue
+            if name in ('ec_launchers', 'numerics', 'sdn'):
+                continue
 
-                for i in range(maxoccur + 1):
-                    ids = idss_in.__dict__[name]
-                    ids.get(i)
-                    ids.setExpIdx(
-                        idx)  # this line sets the index to the output
-                    ids.put(i)
+            for i in range(maxoccur + 1):
+                ids = idss_in.__dict__[name]
+                ids.get(i)
+                ids.setExpIdx(idx)  # this line sets the index to the output
+                ids.put(i)
 
-        idss_in.close()
+    idss_in.close()
     idss_out.close()
diff --git a/duqtools/ids/_handle.py b/duqtools/ids/_handle.py
@@ -5,8 +5,8 @@
 from getpass import getuser
 from pathlib import Path
 
+from ..operations import add_to_op_queue
 from ..schema import ImasBaseModel
-from ..utils import dry_run_toggle
 from ._copy import copy_ids_entry
 from ._imas import imas, imasdef
 
@@ -66,18 +66,17 @@ def copy_ids_entry_to(self, destination: ImasHandle):
         """
         copy_ids_entry(self, destination)
 
+    @add_to_op_queue('Removing {self}')
     def delete(self):
         """Remove data from entry."""
-        from ..config import cfg
 
         # ERASE_PULSE operation is yet supported by IMAS as of June 2022
         path = self.path()
         for suffix in SUFFIXES:
             to_delete = path.with_suffix(suffix)
             logger.debug('Removing %s', to_delete)
             try:
-                if not cfg.dry_run:
-                    to_delete.unlink()
+                to_delete.unlink()
             except FileNotFoundError:
                 logger.warning('%s does not exist', to_delete)
 
@@ -102,7 +101,6 @@ def copy_ids_entry_to_run(self, *, run: int) -> ImasHandle:
         self.copy_ids_entry_to(destination)
         return destination
 
-    @dry_run_toggle
     def get(self, key: str = 'core_profiles', **kwargs):
         """Get data from IDS entry.
 

diff --git a/duqtools/init.py b/duqtools/init.py
@@ -1,21 +1,19 @@
 import logging
 from pathlib import Path
 
-from duqtools.config import Config
-
+from .config import Config
+from .operations import confirm_operations, op_queue
 from .schema import BaseModel
 
 logger = logging.getLogger(__name__)
 
 
-def init(*, dry_run: bool, config: str, full: bool, force: bool,
-         comments: bool, **kwargs):
+@confirm_operations
+def init(*, config: str, full: bool, force: bool, comments: bool, **kwargs):
     """Initialize a brand new config file with all the default values.
 
     Parameters
     ----------
-    dry_run : bool
-        Do not make any changes to the file system.
     config : str
         Filename of the config.
     full : bool
@@ -65,6 +63,5 @@ def init(*, dry_run: bool, config: str, full: bool, force: bool,
                                 'plot': {'plots'}
                             })
 
-    if not dry_run:
-        with open(config_filepath, 'w') as f:
-            f.write(cfg_yaml)
+    op_queue.add(action=lambda: open(config_filepath, 'w').write(cfg_yaml),
+                 description=f'Writing out {config_filepath} config file')
diff --git a/duqtools/jetto/_system.py b/duqtools/jetto/_system.py
@@ -7,7 +7,7 @@
 from typing_extensions import Literal
 
 from ..models import AbstractSystem
-from ..utils import dry_run_toggle
+from ..operations import add_to_op_queue
 from ._imas_functions import imas_from_jset_input
 from ._jset import JettoSettings
 
@@ -29,13 +29,13 @@ class JettoSystem(AbstractSystem):
     name: Literal['jetto'] = Field('jetto', description='Name of the system.')
 
     @staticmethod
-    @dry_run_toggle
+    @add_to_op_queue('Writing new batchfile for {run_name}')
     def write_batchfile(workspace: WorkDirectory, run_name: str):
         from duqtools.jetto import write_batchfile as jetto_write_batchfile
         return jetto_write_batchfile(workspace, run_name)
 
     @staticmethod
-    @dry_run_toggle
+    @add_to_op_queue('Copying template to {target_drc}')
     def copy_from_template(source_drc: Path, target_drc: Path):
         from duqtools.jetto import copy_files
         return copy_files(source_drc, target_drc)
@@ -49,7 +49,7 @@ def imas_from_path(template_drc: Path):
         return source
 
     @staticmethod
-    @dry_run_toggle
+    @add_to_op_queue('Updating imas locations of {run}')
     def update_imas_locations(run: Path, inp: ImasHandle, out: ImasHandle):
         jset = JettoSettings.from_directory(run)
         jset_copy = jset.set_imas_locations(inp=inp, out=out)