From a674c9677a6163da7f6c07f20582d1ffc8127b2c Mon Sep 17 00:00:00 2001 From: jmills-ncar Date: Mon, 23 Jul 2018 14:06:40 -0600 Subject: [PATCH] updating json namelists and starting refactor of conftest --- tests/conftest.py | 168 +++++++++++------------------- tests/test_1_fundamental.py | 15 ++- trunk/NDHMS/compile_options.json | 8 +- trunk/NDHMS/hrldas_namelists.json | 34 ++---- trunk/NDHMS/hydro_namelists.json | 64 +++--------- 5 files changed, 99 insertions(+), 190 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7e0599112..1e32e84b5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,4 @@ -import json -import pathlib import pytest -import sys -import warnings from wrfhydropy import * @@ -53,145 +49,107 @@ def pytest_addoption(parser): default='gfort', required=False, action='store', - help='compiler key from config' + help='compiler, options are intel or gfort' ) parser.addoption( - '--job_default', - default=None, + '--ncores', + default='2', required=False, action='store', - help='candidate job' + help='Number of cores to use for testing' ) parser.addoption( - '--job_ncores', + '--scheduler', default=None, required=False, action='store', - help='ncores job' - ) + help='Scheduler to use for testing, options are PBSCheyenne or do not specify for no ' + 'scheduler') parser.addoption( - '--scheduler', - default=None, + '--account', + default='NRAL0017', required=False, action='store', - help='scheduler for all jobs') - - -def pytest_generate_tests(metafunc): - if 'configuration' in metafunc.fixturenames: - metafunc.parametrize( - "configuration", - metafunc.config.option.config, - scope="session" - ) - - -@pytest.fixture(scope="session") -def candidate_setup(request, configuration): - - domain_dir = request.config.getoption("--domain_dir") - candidate_dir = request.config.getoption("--candidate_dir") - + help='Account number to use if using a scheduler.') + +def _make_sim(domain_dir, + source_dir, + configuration, + ncores, + scheduler, + account): # Candidate model - candidate_model = WrfHydroModel( - source_dir=candidate_dir, + model = Model( + source_dir=source_dir, model_config=configuration ) # Candidate domain - domain = WrfHydroDomain( + domain = Domain( domain_top_dir=domain_dir, - domain_config=configuration, - model_version=candidate_model.version - ) - - # Candidate setup - candidate_setup = WrfHydroSetup(candidate_model, domain) - - return candidate_setup - + domain_config=configuration) -@pytest.fixture(scope="session") -def reference_setup(request, configuration): - - domain_dir = request.config.getoption("--domain_dir") - reference_dir = request.config.getoption("--reference_dir") - - # Reference model - reference_model = WrfHydroModel( - source_dir=reference_dir, - model_config=configuration - ) - - # Reference domain - domain = WrfHydroDomain( - domain_top_dir=domain_dir, - domain_config=configuration, - model_version=reference_model.version - ) + # Job + exe_command = ('mpirun -np {0} ./wrf_hydro.exe').format(str(ncores)) + job = Job(job_id='test_job',exe_cmd=exe_command) - # Reference setup - reference_setup = WrfHydroSetup(reference_model, domain) + # Candidate simulation + sim = Simulation() + sim.add(model) + sim.add(domain) + sim.add(job) - return reference_setup + if scheduler is not None and scheduler == 'pbscheyenne': + sim.add(schedulers.PBSCheyenne(account=account,nproc=int(ncores))) + return sim @pytest.fixture(scope="session") -def compiler(request): - return request.config.getoption("--compiler") +def candidate_setup(request): + domain_dir = request.config.getoption("--domain_dir") + candidate_dir = request.config.getoption("--candidate_dir") + configuration = request.config.getoption("--config") + ncores = request.config.getoption("--ncores") + scheduler = str(request.config.getoption("--scheduler")).lower() + account = request.config.getoption("--account") -@pytest.fixture(scope="session") -def job_default(request, configuration): - job_in = request.config.getoption("--job_default") - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - job_dum = Job(nproc=-1) - if job_in is not None: - job_in = json.loads(job_in) - job_dum.__dict__.update(job_in) - return job_dum + candidate_sim = _make_sim(domain_dir = domain_dir, + source_dir= candidate_dir, + configuration=configuration, + ncores = ncores, + scheduler = scheduler, + account = account) + return candidate_sim @pytest.fixture(scope="session") -def job_ncores(request, configuration): - job_in = request.config.getoption("--job_ncores") - print("job_in: ", job_in) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - job_dum = Job(nproc=-1) - if job_in is not None: - job_in = json.loads(job_in) - job_dum.__dict__.update(job_in) - return job_dum +def reference_setup(request): + domain_dir = request.config.getoption("--domain_dir") + reference_dir = request.config.getoption("--reference_dir") + configuration = request.config.getoption("--config") + ncores = request.config.getoption("--ncores") + scheduler = str(request.config.getoption("--scheduler")).lower() + account = request.config.getoption("--account") -@pytest.fixture(scope="session") -def scheduler(request, configuration): - sched_in = request.config.getoption("--scheduler") - if sched_in is None or sched_in == 'null': - return None - else: - sched = json.loads(sched_in) - sched_dum = Scheduler( - job_name='default', - account='DUMDUM', - walltime='00:01:00', - queue='none', - nproc=1, - ppn=1 - ) - sched_dum.__dict__.update(sched) - return sched_dum + reference_sim = _make_sim(domain_dir = domain_dir, + source_dir= reference_dir, + configuration=configuration, + ncores = ncores, + scheduler = scheduler, + account = account) + return reference_sim @pytest.fixture(scope="session") -def output_dir(request,configuration): +def output_dir(request): + configuration = request.config.getoption("--config") output_dir = request.config.getoption("--output_dir") + output_dir = pathlib.Path(output_dir) output_dir = output_dir / configuration if output_dir.is_dir() is False: diff --git a/tests/test_1_fundamental.py b/tests/test_1_fundamental.py index fe0c9d351..f10fcd1e2 100644 --- a/tests/test_1_fundamental.py +++ b/tests/test_1_fundamental.py @@ -1,10 +1,7 @@ import copy import datetime as dt -import json import pickle import pytest -import shutil -import time import warnings import wrfhydropy @@ -20,7 +17,7 @@ ###Compile questions compiler, def test_compile_candidate( - candidate_setup, + candidate_sim, output_dir, compiler, capsys @@ -31,19 +28,19 @@ def test_compile_candidate( compile_dir = output_dir / 'compile_candidate' # Compile the model - candidate_setup.model.compile( + candidate_sim.model.compile( compiler=compiler, compile_dir=compile_dir, overwrite=True ) # Check compilation status - assert candidate_setup.model.compile_log.returncode == 0, \ + assert candidate_sim.model.compile_log.returncode == 0, \ "Candidate code did not compile correctly." def test_compile_reference( - reference_setup, + reference_sim, output_dir, compiler, capsys @@ -54,14 +51,14 @@ def test_compile_reference( compile_dir = output_dir / 'compile_reference' # Compile the model - reference_setup.model.compile( + reference_sim.model.compile( compiler=compiler, compile_dir=compile_dir, overwrite=True ) # Check compilation status - assert reference_setup.model.compile_log.returncode == 0, \ + assert reference_sim.model.compile_log.returncode == 0, \ "Reference code did not compile correctly" diff --git a/trunk/NDHMS/compile_options.json b/trunk/NDHMS/compile_options.json index 5b76ab4be..b0472e4bd 100644 --- a/trunk/NDHMS/compile_options.json +++ b/trunk/NDHMS/compile_options.json @@ -1,5 +1,5 @@ { - "NWM": { + "nwm": { "WRF_HYDRO": 1, "HYDRO_D": 0, "SPATIAL_SOIL": 1, @@ -8,7 +8,7 @@ "NCEP_WCOSS": 0, "WRF_HYDRO_NUDGING": 1 }, - "Gridded": { + "gridded": { "WRF_HYDRO": 1, "HYDRO_D": 0, "SPATIAL_SOIL": 1, @@ -17,7 +17,7 @@ "NCEP_WCOSS": 0, "WRF_HYDRO_NUDGING": 0 }, - "Gridded_NO_LAKES": { + "gridded_no_lakes": { "WRF_HYDRO": 1, "HYDRO_D": 0, "SPATIAL_SOIL": 1, @@ -26,7 +26,7 @@ "NCEP_WCOSS": 0, "WRF_HYDRO_NUDGING": 0 }, - "Reach": { + "reach": { "WRF_HYDRO": 1, "HYDRO_D": 0, "SPATIAL_SOIL": 1, diff --git a/trunk/NDHMS/hrldas_namelists.json b/trunk/NDHMS/hrldas_namelists.json index e39d5a765..d9e6275ca 100644 --- a/trunk/NDHMS/hrldas_namelists.json +++ b/trunk/NDHMS/hrldas_namelists.json @@ -21,7 +21,7 @@ "rst_bi_in": 0, "rst_bi_out": 0, "runoff_option": 3, - "snow_albedo_option": 2, + "snow_albedo_option": 1, "soil_thick_input": [0.1, 0.3, 0.6, 1.0], "spatial_filename": "NULL_specified_in_domain.json", "split_output_count": 1, @@ -41,48 +41,36 @@ "forc_typ": "NULL_specified_in_domain.json" } }, - "NWM_ana": { + "nwm_ana": { "noahlsm_offline": {}, "wrf_hydro_offline": {} }, - "NWM_short_range" : { - "noahlsm_offline": { - "khour": 18, - "restart_frequency_hours": 600 - }, + "nwm_short_range" : { + "noahlsm_offline": {}, "wrf_hydro_offline": {} }, - "NWM_medium_range" : { - "noahlsm_offline": { - "khour": 240, - "output_timestep": 10800, - "restart_frequency_hours": 6000 - }, + "nwm_medium_range" : { + "noahlsm_offline": {}, "wrf_hydro_offline": {} }, - "NWM_long_range" : { - "noahlsm_offline": { - "forcing_timestep": 10800, - "khour": 720, - "output_timestep": 86400, - "restart_frequency_hours": 6000 - }, + "nwm_long_range" : { + "noahlsm_offline": {}, "wrf_hydro_offline": {} }, - "NWM_channel_only" : { + "nwm_channel_only" : { "noahlsm_offline": {}, "wrf_hydro_offline": {} }, - "Gridded": { + "gridded": { "noahlsm_offline": {}, "wrf_hydro_offline": {} }, - "Reach": { + "reach": { "noahlsm_offline": {}, "wrf_hydro_offline": {} } diff --git a/trunk/NDHMS/hydro_namelists.json b/trunk/NDHMS/hydro_namelists.json index d194aadcc..3d51c61e8 100644 --- a/trunk/NDHMS/hydro_namelists.json +++ b/trunk/NDHMS/hydro_namelists.json @@ -1,7 +1,7 @@ { "base": { "hydro_nlist": { - "aggfactrt": 4, + "aggfactrt": 1, "channel_option": 2, "chanobs_domain": 1, "chanrtswcrt": 1, @@ -9,11 +9,11 @@ "chrtout_grid": 0, "dtrt_ch": 300, "dtrt_ter": 10, - "dxrt": 250.0, + "dxrt": 1000, "frxst_pts_out": 0, "geo_finegrid_flnm": "NULL_specified_in_domain.json", "geo_static_flnm": "NULL_specified_in_domain.json", - "gw_restart": 1, + "gw_restart": 0, "gwbaseswcrt": 1, "gwbuckparm_file": "NULL_specified_in_domain.json", "hydrotbl_f": "NULL_specified_in_domain.json", @@ -26,7 +26,7 @@ "order_to_write": 1, "out_dt": 60, "outlake": 1, - "output_channelbucket_influx": 1, + "output_channelbucket_influx": 2, "output_gw": 1, "ovrtswcrt": 1, "restart_file": "NULL_specified_in_domain.json", @@ -35,7 +35,7 @@ "rst_bi_in": 0, "rst_bi_out": 0, "rst_dt": 60, - "rst_typ": 1, + "rst_typ": 0, "rstrt_swc": 1, "rt_option": 1, "rtout_domain": 1, @@ -62,14 +62,8 @@ "timeslicepath": "NULL_specified_in_domain.json" } }, - "NWM_ana": { - "hydro_nlist": { - "chrtout_domain": 1, - "outlake": 1, - "output_channelbucket_influx": 1, - "output_gw": 0, - "lsmout_domain": 0 - }, + "nwm_ana": { + "hydro_nlist": {}, "nudging_nlist": { "biaswindowbeforet0": false, "invdisttimeweightbias": true, @@ -86,15 +80,8 @@ } }, - "NWM_short_range" : { - "hydro_nlist": { - "chrtout_domain": 1, - "outlake": 1, - "output_channelbucket_influx": 1, - "output_gw": 0, - "lsmout_domain": 0, - "rst_dt": 999990 - }, + "nwm_short_range" : { + "hydro_nlist": {}, "nudging_nlist": { "biaswindowbeforet0": true, "invdisttimeweightbias": true, @@ -111,16 +98,8 @@ } }, - "NWM_medium_range": { - "hydro_nlist": { - "chrtout_domain": 1, - "outlake": 1, - "output_gw": 0, - "lsmout_domain": 0, - "out_dt": 180, - "output_channelbucket_influx": 1, - "rst_dt": 999990 - }, + "nwm_medium_range": { + "hydro_nlist": {}, "nudging_nlist": { "biaswindowbeforet0": true, "invdisttimeweightbias": true, @@ -137,19 +116,9 @@ } }, - "NWM_long_range": { + "nwm_long_range": { "hydro_nlist": { - "chrtout_domain": 1, - "outlake": 1, - "output_gw": 0, - "lsmout_domain": 0, - "aggfactrt": 1, - "dxrt": 1000.0, - "out_dt": 360, - "output_channelbucket_influx": 0, "ovrtswcrt": 0, - "rst_dt": 999990, - "rst_typ": 0, "rtout_domain": 0, "subrtswcrt": 0 }, @@ -169,11 +138,8 @@ } }, - "NWM_channel-only": { + "nwm_channel-only": { "hydro_nlist": { - "chrtout_domain": 1, - "outlake": 1, - "output_channelbucket_influx": 1, "output_gw": 0, "lsmout_domain": 0 }, @@ -193,7 +159,7 @@ } }, - "Gridded": { + "gridded": { "hydro_nlist": { "gwbasmskfil": "NULL_specified_in_domain.json", "channel_option": 3, @@ -205,7 +171,7 @@ "nudging_nlist": {} }, - "Reach": { + "reach": { "hydro_nlist": { "gwbasmskfil": "NULL_specified_in_domain.json", "dtrt_ch": 10,