Collects failed assets on LyTestTools test failures

Signed-off-by: evanchia <evanchia@amazon.com>
o3de · Kadino · Feb 10, 2022 · Feb 3, 2022 · Feb 3, 2022 · Feb 4, 2022
commit f45ff6488bb716350de374e7f8df7bc3ee13f638
@@ -175,13 +175,20 @@ def asset_processor(self):
         return os.path.join(self.build_directory(), 'AssetProcessor')
 
     def asset_processor_batch(self):
-        """"
+        """
         Return path for the AssetProcessorBatch compatible with this build platform and configuration
         ex. engine_root/dev/mac/bin/profile/AssetProcessorBatch
         :return: path to AssetProcessorBatch
         """
         return os.path.join(self.build_directory(), 'AssetProcessorBatch')
 
+    def ap_job_logs(self):
+        """
+        Return path to the Asset Processor JobLogs directory.
+        :return: path to <project>/user/log/JobLogs
+        """
+        return os.path.join(self.project_log(), 'JobLogs')
+
     def editor(self):
         """
         Return path to the editor executable compatible with the current build.

@@ -777,6 +777,7 @@ def _exec_editor_test(self, request: Request, workspace: AbstractWorkspace, edit
             if return_code == 0:
                 test_result = Result.Pass.create(test_spec, output, editor_log_content)
             else:
+                editor_utils.save_failed_asset_joblogs(workspace)
                 has_crashed = return_code != EditorTestSuite._TEST_FAIL_RETCODE
                 if has_crashed:
                     test_result = Result.Crash.create(test_spec, output, return_code, editor_utils.retrieve_crash_output
@@ -855,6 +856,7 @@ def _exec_editor_multitest(self, request: Request, workspace: AbstractWorkspace,
                 for test_spec in test_spec_list:
                     results[test_spec.__name__] = Result.Pass.create(test_spec, output, editor_log_content)
             else:
+                editor_utils.save_failed_asset_joblogs(workspace)
                 # Scrap the output to attempt to find out which tests failed.
                 # This function should always populate the result list, if it didn't find it, it will have "Unknown" type of result
                 results = self._get_results_using_output(test_spec_list, output, editor_log_content)

@@ -10,6 +10,7 @@
 import os
 import time
 import logging
+import re
 
 import ly_test_tools.environment.process_utils as process_utils
 import ly_test_tools.environment.waiter as waiter
@@ -151,3 +152,42 @@ def retrieve_last_run_test_index_from_output(test_spec_list: list[EditorTestBase
         else:
             index += 1
     return index
+
+def save_failed_asset_joblogs(workspace: AbstractWorkspace) -> None:
+    """
+    Checks all asset logs in the JobLogs directory to see if the asset has any warnings or errors. If so, the asset is
+    saved via ArtifactManager.
+
+    :param workspace: The AbstractWorkspace to access the JobLogs path
+    :return: None
+    """
+    # Tuples are in the form [this directory, [child directories], [files in this directory]]
+    for walk_tuple in os.walk(workspace.paths.ap_job_logs()):
+        for log_file in walk_tuple[2]:
+            full_log_path = os.path.join(walk_tuple[0], log_file)
+            # Only save asset logs that contain errors or warnings
+            if _check_log_errors_warnings(full_log_path):
+                workspace.artifact_manager.save_artifact(full_log_path)
+
+def _check_log_errors_warnings(log_path: str) -> bool:
+    """
+    Checks to see if the asset log contains any errors or warnings by reading the last line of the file.
+    Example log line: '~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 0 errors, 1 warnings'
+
+    :param log_path: The pull path to the asset log file to read
+    :return: True if the last line regex finds an error or warning, else False
+    """
+    log_regex = "(\\d+) errors, (\\d+) warnings"
+    with open(log_path, 'r') as opened_asset_log:
+        # Read the last line
+        for log_line in opened_asset_log:
+            pass
+        last_line = log_line
+    # Use regex to see if there are any non zero values for errors or warnings
+    regex_match = re.search(log_regex, last_line)
+    if regex_match is None:
+        logger.warning(f"Regex for asset log is expected to be in the form:\nn errors, n warnings\nActual: {last_line}")
+        return False
+    if (int)(regex_match.group(1)) != 0 or (int)(regex_match.group(2)) != 0:
+        return True
+    return False
@@ -170,3 +170,59 @@ def test_RetrieveLastRunTestIndexFromOutput_FirstItemFailed_Returns0(self):
             mock_test_list.append(mock_test)
 
         assert 0 == editor_test_utils.retrieve_last_run_test_index_from_output(mock_test_list, mock_editor_output)
+
+    @mock.patch('ly_test_tools.o3de.editor_test_utils._check_log_errors_warnings')
+    @mock.patch('os.walk')
+    def test_SaveFailedAssetJoblogs_ManyValidLogs_SavesCorrectly(self, mock_walk, mock_check_log):
+        mock_workspace = mock.MagicMock()
+        mock_walk.return_value = [['MockDirectory', None, ['mock_log.log']],
+                                  ['MockDirectory2', None, ['mock_log2.log']]]
+        mock_check_log.return_value = True
+
+        editor_test_utils.save_failed_asset_joblogs(mock_workspace)
+
+        assert mock_workspace.artifact_manager.save_artifact.call_count == 2
+
+    @mock.patch('ly_test_tools.o3de.editor_test_utils._check_log_errors_warnings')
+    @mock.patch('os.walk')
+    def test_SaveFailedAssetJoblogs_ManyInvalidLogs_NoSaves(self, mock_walk, mock_check_log):
+        mock_workspace = mock.MagicMock()
+        mock_walk.return_value = [['MockDirectory', None, ['mock_log.log']],
+                                  ['MockDirectory2', None, ['mock_log2.log']]]
+        mock_check_log.return_value = False
+
+        editor_test_utils.save_failed_asset_joblogs(mock_workspace)
+
+        assert mock_workspace.artifact_manager.save_artifact.call_count == 0
+
+    def test_CheckLogErrorWarnings_ValidLine_ReturnsTrue(self):
+        mock_log = '~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 0 errors, 1 warnings'
+        mock_log_path = mock.MagicMock()
+
+        with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
+            expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
+        assert expected
+
+    def test_CheckLogErrorWarnings_MultipleValidLine_ReturnsTrue(self):
+        mock_log = '~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 1 errors, 1 warnings'
+        mock_log_path = mock.MagicMock()
+
+        with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
+            expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
+        assert expected
+
+    def test_CheckLogErrorWarnings_InvalidLine_ReturnsFalse(self):
+        mock_log = '~~1643759303647~~1~~00000000000009E0~~AssetBuilder~~S: 0 errors, 0 warnings'
+        mock_log_path = mock.MagicMock()
+
+        with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
+            expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
+        assert not expected
+
+    def test_CheckLogErrorWarnings_InvalidRegex_ReturnsFalse(self):
+        mock_log = 'Invalid last line'
+        mock_log_path = mock.MagicMock()
+
+        with mock.patch('builtins.open', mock.mock_open(read_data=mock_log)) as mock_file:
+            expected = editor_test_utils._check_log_errors_warnings(mock_log_path)
+        assert not expected
@@ -617,6 +617,7 @@ def test_ExecEditorTest_TestSucceeds_ReturnsPass(self, mock_cycle_crash, mock_ge
     @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
+    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
     @mock.patch('os.path.join', mock.MagicMock())
     def test_ExecEditorTest_TestFails_ReturnsFail(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                      mock_retrieve_log, mock_retrieve_editor_log,
@@ -646,6 +647,7 @@ def test_ExecEditorTest_TestFails_ReturnsFail(self, mock_cycle_crash, mock_get_t
     @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
+    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
     @mock.patch('os.path.join', mock.MagicMock())
     @mock.patch('os.path.basename', mock.MagicMock())
     def test_ExecEditorTest_TestCrashes_ReturnsCrash(self, mock_cycle_crash, mock_get_testcase_filepath,
@@ -735,6 +737,7 @@ def test_ExecEditorMultitest_AllTestsPass_ReturnsPasses(self, mock_cycle_crash,
     @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
+    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
     @mock.patch('os.path.join', mock.MagicMock())
     def test_ExecEditorMultitest_OneFailure_CallsCorrectFunc(self, mock_cycle_crash, mock_get_testcase_filepath,
                                                            mock_retrieve_log, mock_retrieve_editor_log, mock_get_results):
@@ -761,6 +764,7 @@ def test_ExecEditorMultitest_OneFailure_CallsCorrectFunc(self, mock_cycle_crash,
     @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
+    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
     @mock.patch('os.path.join', mock.MagicMock())
     @mock.patch('os.path.basename', mock.MagicMock())
     def test_ExecEditorMultitest_OneCrash_ReportsOnUnknownResult(self, mock_cycle_crash, mock_get_testcase_filepath,
@@ -798,6 +802,7 @@ def test_ExecEditorMultitest_OneCrash_ReportsOnUnknownResult(self, mock_cycle_cr
     @mock.patch('ly_test_tools.o3de.editor_test_utils.retrieve_log_path')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.get_testcase_module_filepath')
     @mock.patch('ly_test_tools.o3de.editor_test_utils.cycle_crash_report')
+    @mock.patch('ly_test_tools.o3de.editor_test_utils.save_failed_asset_joblogs', mock.MagicMock())
     @mock.patch('os.path.join', mock.MagicMock())
     @mock.patch('os.path.basename', mock.MagicMock())
     def test_ExecEditorMultitest_ManyUnknown_ReportsUnknownResults(self, mock_cycle_crash, mock_get_testcase_filepath,