Disable GroupID hashing for acceptance tests (#1094)

* Update expected acceptance tests * Revert "Update expected acceptance tests" This reverts commit ad0907f. * Default "hashed_group_id" to false in exposure run * Move hashed_group_id=F default from "RunExposure" to "RunFmTest" * Fix/pip compile (#1097) * Only install pip-tools before pip-compile * Try pinning flake8 * Revert "Try pinning flake8" This reverts commit d845d5b. * Try pinning virtualenv * add --upgrade to pip install pip-tools * Fix test_get_dataframe__from_csv_file__set_col_defaults_option_and_use_defaults_ and run with falsifying example * Remove falsifying example Co-authored-by: Marco Tazzari <6020226+mtazzari@users.noreply.github.com>
OasisLMF · sambles · Oct 3, 2022 · Jul 5, 2022 · Jul 5, 2022 · Jul 6, 2022
commit 82fb79c01861ef77dedbf5f60c9712c14bdbb2d5
diff --git a/.github/workflows/oasislmf-unittest.yml b/.github/workflows/oasislmf-unittest.yml
@@ -35,8 +35,7 @@ jobs:
       - name: Install pip-tools
         run: |
           python -m pip install --upgrade pip
-          pip install pip-tools
-
+          pip install --upgrade pip-tools
       - name: Pip Compile
         run: |
           rm -f requirements.txt

diff --git a/oasislmf/computation/run/exposure.py b/oasislmf/computation/run/exposure.py
@@ -54,6 +54,7 @@ class RunExposure(ComputationStep):
         {'name': 'fmpy_low_memory',      'default': False, 'type': str2bool, 'const':True, 'nargs':'?', 'help': 'use memory map instead of RAM to store loss array (may decrease performance but reduce RAM usage drastically)'},
         {'name': 'fmpy_sort_output',     'default': True, 'type': str2bool, 'const': True, 'nargs': '?', 'help': 'order fmpy output by item_id'},
         {'name': 'stream_type',          'flag':'-t', 'default': 2,  'type':int,  'help': 'Set the IL input stream type, 2 = default loss stream, 1 = deprecated cov/item stream'},
+        {"name": "hashed_group_id",      "default": True, "type": str2bool, "const": False, 'nargs': '?',  "help": "Hashes the group_id in the items.bin"},
         {'name': 'net_ri', 'default': True},
         {'name': 'include_loss_factor', 'default': True},
         {'name': 'print_summary', 'default': True},
@@ -120,6 +121,7 @@ def run(self):
             oed_info_csv=ri_info_fp,
             oed_scope_csv=ri_scope_fp,
             keys_data_csv=keys_fp,
+            hashed_group_id=self.hashed_group_id,
         ).run()
 
         # 3. Run Deterministic Losses
@@ -312,6 +314,7 @@ class RunFmTest(ComputationStep):
         {'name': 'fmpy_low_memory',     'default': False,   'type': str2bool, 'const': True, 'nargs': '?', 'help': 'use memory map instead of RAM to store loss array (may decrease performance but reduce RAM usage drastically)'},
         {'name': 'fmpy_sort_output',    'default': True,    'type': str2bool, 'const': True, 'nargs': '?', 'help': 'order fmpy output by item_id'},
         {'name': 'update_expected',     'default': False},
+        {'name': 'hashed_group_id',     'default': False},
         {'name': 'expected_output_dir', 'default': "expected"},
     ]
 
@@ -408,7 +411,8 @@ def execute_test_case(self, test_case):
             num_subperils=self.num_subperils,
             fmpy=self.fmpy,
             fmpy_low_memory=self.fmpy_low_memory,
-            fmpy_sort_output=self.fmpy_sort_output
+            fmpy_sort_output=self.fmpy_sort_output,
+            hashed_group_id=self.hashed_group_id,
         ).run()
 
         expected_data_dir = os.path.join(test_dir, self.expected_output_dir)

diff --git a/tests/fm/test_fm.py b/tests/fm/test_fm.py
@@ -15,6 +15,7 @@ def setUp(self):
         self.test_cases_fp = os.path.join(sys.path[0], 'validation')
         self.update_expected = False
         self.keep_output = True
+        self.hashed_group_id = False
 
     def run_test(self, test_case, fmpy=False, subperils=1, expected_dir="expected"):
         with tempfile.TemporaryDirectory() as tmp_run_dir:
@@ -35,6 +36,7 @@ def run_test(self, test_case, fmpy=False, subperils=1, expected_dir="expected"):
                 num_subperils=subperils,
                 test_tolerance=0.001,
                 expected_output_dir=expected_dir,
+                hashed_group_id=self.hashed_group_id,
             )
 
         self.assertTrue(result)

diff --git a/tests/fm/test_fmpy.py b/tests/fm/test_fmpy.py
@@ -15,6 +15,7 @@ def setUp(self):
         self.test_cases_fp = os.path.join(sys.path[0], 'validation')
         self.update_expected = False
         self.keep_output = True
+        self.hashed_group_id = False
 
     def run_test(self, test_case, fmpy=False, subperils=1, expected_dir="expected"):
         with tempfile.TemporaryDirectory() as tmp_run_dir:
@@ -37,6 +38,7 @@ def run_test(self, test_case, fmpy=False, subperils=1, expected_dir="expected"):
                 num_subperils=subperils,
                 test_tolerance=0.001,
                 expected_output_dir=expected_dir,
+                hashed_group_id=self.hashed_group_id,
             )
         self.assertTrue(result)
 
@@ -91,4 +93,4 @@ def test_issues_2_subperils(self):
         self.run_test('issues', fmpy=True, subperils=2, expected_dir="expected_subperils")
 
     def test_insurance_policy_coverage_2_subperils(self):
-        self.run_test('insurance_policy_coverage',fmpy=True, subperils=2, expected_dir="expected_subperils")
+        self.run_test('insurance_policy_coverage',fmpy=True, subperils=2, expected_dir="expected_subperils")
diff --git a/tests/utils/test_data.py b/tests/utils/test_data.py
@@ -39,6 +39,7 @@
     get_timestamp,
     get_utctimestamp,
     get_location_df,
+    PANDAS_DEFAULT_NULL_VALUES,
 )
 
 from oasislmf.utils.defaults import (
@@ -541,10 +542,12 @@ def test_get_dataframe__from_csv_file_with_mixed_case_cols__set_col_defaults_opt
         try:
             df = pd.DataFrame(data)
             df.to_csv(path_or_buf=fp, columns=df.columns, encoding='utf-8', index=False)
+            df['STR_COL'] = df['STR_COL'].map(lambda x: np.nan if x in PANDAS_DEFAULT_NULL_VALUES  else x)
             fp.close()
 
             expected = df.copy(deep=True)
             expected.columns = expected.columns.str.lower()
+
             for col, default in defaults.items():
                 expected.loc[:, col.lower()].fillna(defaults[col], inplace=True)