Report passes and rerun non flaky e2e tests

oppia · U8NWXD · Dec 19, 2020 · Nov 25, 2020 · Nov 25, 2020 · Nov 25, 2020
commit f203b294e6c08064c192eefed07cd4ff48b7f41a
@@ -24,10 +24,13 @@
 
 import requests
 
-FLAKE_REPORT_URL = (
+FLAKE_CHECK_AND_REPORT_URL = (
     'https://oppia-e2e-test-results-logger.herokuapp.com'
     '/check-flake-and-report')
-FLAKE_REPORT_API_KEY = '7Ccp062JVjv9LUYwnLMqcm5Eu5gYqqhpl3zQmcO3cDQ'
+PASS_REPORT_URL = (
+    'https://oppia-e2e-test-results-logger.herokuapp.com'
+    '/check-flake-and-report')
+REPORT_API_KEY = '7Ccp062JVjv9LUYwnLMqcm5Eu5gYqqhpl3zQmcO3cDQ'
 
 CI_INFO = {
     'circleCI': {
@@ -98,6 +101,25 @@ def _get_build_info():
     raise Exception('Unknown build environment.')
 
 
+def report_pass(suite_name):
+    """Report a passing test to the logging server."""
+    metadata = _get_build_info()
+    payload = {
+        'suite': suite_name,
+        'metadata': metadata,
+    }
+    try:
+        response = requests.post(
+            PASS_REPORT_URL, json=payload,
+            allow_redirects=False,
+            headers={'report_key': REPORT_API_KEY})
+    except Exception as e:
+        _print_color_message((
+            'Failed to contact E2E test logging server at %s.'
+            'Please report to E2E team in case server is down.'
+            'Exception: %s') % (FLAKE_CHECK_AND_REPORT_URL, e))
+
+
 def is_test_output_flaky(output_lines, suite_name):
     """Returns whether the test output matches any flaky test log."""
     build_info = _get_build_info()
@@ -109,13 +131,14 @@ def is_test_output_flaky(output_lines, suite_name):
     response = None
     try:
         response = requests.post(
-            FLAKE_REPORT_URL, json=payload, allow_redirects=False,
-            headers={'report_key': FLAKE_REPORT_API_KEY})
+            FLAKE_CHECK_AND_REPORT_URL, json=payload,
+            allow_redirects=False,
+            headers={'report_key': REPORT_API_KEY})
     except Exception as e:
         _print_color_message((
             'Failed to contact E2E test logging server at %s.'
             'Please report to E2E team in case server is down.'
-            'Exception: %s') % (FLAKE_REPORT_URL, e))
+            'Exception: %s') % (FLAKE_CHECK_AND_REPORT_URL, e))
 
         return False
 
@@ -131,8 +154,9 @@ def is_test_output_flaky(output_lines, suite_name):
         _print_color_message('Unable to convert json response: %s' % e)
 
     if 'log' in report:
+        log_str = '\n'.join(report['log'])
         _print_color_message(
-            'Logs from test result logging server:\n %s' % report['log'])
+            'Logs from test result logging server:\n %s' % log_str)
 
     flaky = report['result'] if 'result' in report else False
     if flaky:

@@ -34,6 +34,7 @@
 from scripts import install_third_party_libs
 
 MAX_RETRY_COUNT = 3
+RERUN_NON_FLAKY = True
 WEB_DRIVER_PORT = 4444
 GOOGLE_APP_ENGINE_PORT = 9001
 OPPIA_SERVER_PORT = 8181
@@ -572,21 +573,26 @@ def main(args=None):
     portserver_process = start_portserver()
     atexit.register(cleanup_portserver, portserver_process)
 
-    return_code = 1
-
     for attempt_num in python_utils.RANGE(MAX_RETRY_COUNT):
         python_utils.PRINT('***Attempt %s.***' % (attempt_num + 1))
-        # Only rerun tests on CI.
+        output, return_code = run_tests(parsed_args)
+        # Don't rerun passing tests.
+        if return_code == 0:
+            flake_checker.report_pass(parsed_args.suite)
+            break
+        flaky = flake_checker.is_test_output_flaky(
+            output, parsed_args.suite)
+        # Don't rerun if the test was non-flaky or if we are not
+        # rerunning non-flaky tests.
+        if not flaky or not RERUN_NON_FLAKY:
+            break
+        # Don't rerun off of CI.
         if not flake_checker.check_if_on_ci():
             python_utils.PRINT('No reruns because not running on CI.')
             break
-        output, return_code = run_tests(parsed_args)
-        if return_code == 0 or not flake_checker.is_test_output_flaky(
-                output, parsed_args.suite):
-            break
-        else:
-            cleanup_portserver(portserver_process)
-            cleanup()
+        # Prepare for rerun.
+        cleanup_portserver(portserver_process)
+        cleanup()
 
     sys.exit(return_code)