Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds flaky e2e report and restart functionality #11263

Merged
merged 31 commits into from
Dec 19, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
fae8a15
Reverted old changes
DubeySandeep Nov 25, 2020
c6c9a8a
Adds checks for reporting.
DubeySandeep Nov 25, 2020
9562edd
Added url
DubeySandeep Nov 25, 2020
43f2ffd
Merge branch 'develop' of github.com:oppia/oppia into flake-e2e
DubeySandeep Nov 27, 2020
6353ea8
Fixes exit error issues.
DubeySandeep Nov 28, 2020
ea3af12
Adds checker file.
DubeySandeep Nov 28, 2020
d2bfd87
Adds correct url.
DubeySandeep Nov 28, 2020
7e6bf00
Adds correct url format.
DubeySandeep Nov 28, 2020
917c92f
Fixes test issues
DubeySandeep Nov 28, 2020
1f0544a
Address review comments.
DubeySandeep Dec 1, 2020
d33937e
Merge branch 'develop' into flake-e2e
U8NWXD Dec 11, 2020
f10e084
Update with review comments and for new server
U8NWXD Dec 11, 2020
58753c1
lint fixes
U8NWXD Dec 11, 2020
4f68414
Catch non-unicode characters and ignore them
U8NWXD Dec 11, 2020
4e28982
Merge branch 'develop' into flake-e2e
U8NWXD Dec 14, 2020
f203b29
Report passes and rerun non flaky e2e tests
U8NWXD Dec 15, 2020
2b66886
Fix e2e backend tests
U8NWXD Dec 15, 2020
17c4179
Fix lint errors
U8NWXD Dec 15, 2020
f55fff8
Fix backend tests
U8NWXD Dec 16, 2020
6209a6a
Silence pylint
U8NWXD Dec 16, 2020
426aace
Reach 100% coverage of run_e2e_tests.py
U8NWXD Dec 16, 2020
02210e2
Add tests for flake_checker.py
U8NWXD Dec 17, 2020
1826229
Lint fixes
U8NWXD Dec 17, 2020
545b794
Merge branch 'develop' into flake-e2e
U8NWXD Dec 18, 2020
19f43c1
Include unicode character directly for testing
U8NWXD Dec 18, 2020
8203f85
Make cleanup code more robust
U8NWXD Dec 18, 2020
a84928c
Improve unicode handling
U8NWXD Dec 18, 2020
49d3f05
Fix backend code coverage
U8NWXD Dec 18, 2020
c87d667
Fix CI errors
U8NWXD Dec 19, 2020
8c219b0
More robustly shut down processes
U8NWXD Dec 19, 2020
a2f3b67
Make killing reluctant processes more robust
U8NWXD Dec 19, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Lint fixes
  • Loading branch information
U8NWXD committed Dec 17, 2020
commit 1826229dd22bc2ffabee1febc98723a1ed1b5021
6 changes: 3 additions & 3 deletions scripts/flake_checker.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
}
}

RequestExceptions = (
REQUEST_EXCEPTIONS = (
requests.RequestException, requests.ConnectionError,
requests.HTTPError, requests.TooManyRedirects, requests.Timeout)

Expand Down Expand Up @@ -117,7 +117,7 @@ def report_pass(suite_name):
PASS_REPORT_URL, json=payload,
allow_redirects=False,
headers={'report_key': REPORT_API_KEY})
except RequestExceptions as e:
except REQUEST_EXCEPTIONS as e:
_print_color_message((
'Failed to contact E2E test logging server at %s.'
'Please report to E2E team in case server is down.'
Expand All @@ -138,7 +138,7 @@ def is_test_output_flaky(output_lines, suite_name):
FLAKE_CHECK_AND_REPORT_URL, json=payload,
allow_redirects=False,
headers={'report_key': REPORT_API_KEY})
except RequestExceptions as e:
except REQUEST_EXCEPTIONS as e:
_print_color_message((
'Failed to contact E2E test logging server at %s.'
'Please report to E2E team in case server is down.'
Expand Down
50 changes: 22 additions & 28 deletions scripts/flake_checker_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,15 @@
from __future__ import absolute_import # pylint: disable=import-only-modules
from __future__ import unicode_literals # pylint: disable=import-only-modules

import atexit
import contextlib
import functools
import os
import re
import signal
import subprocess
import sys
import time
import requests
import datetime
import os

from core.tests import test_utils
import feconf
import python_utils

from scripts import build
from scripts import common
from scripts import flake_checker
from scripts import install_third_party_libs

import requests


class CheckIfOnCITests(test_utils.GenericTestBase):
Expand All @@ -58,7 +47,7 @@ def mock_getenv(variable):

def test_returns_false_when_off_ci(self):

def mock_getenv(variable):
def mock_getenv(unused_variable):
return False

getenv_swap = self.swap_with_checks(
Expand All @@ -72,12 +61,13 @@ def mock_getenv(variable):
self.assertFalse(on_ci)


class MockDatetime:
class MockDatetime(python_utils.OBJECT):

def __init__(self, date):
self.date = date

def utcnow(self):
"""Get datetime.datetime object."""
return self.date


Expand All @@ -97,7 +87,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
pass

expected_payload = {
Expand Down Expand Up @@ -136,7 +126,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
pass

expected_payload = {
Expand Down Expand Up @@ -175,7 +165,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
raise requests.HTTPError()

expected_payload = {
Expand Down Expand Up @@ -210,32 +200,36 @@ def mock_getenv(variable):
environment_vars = dict()
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
raise AssertionError('requests.post called.')

getenv_swap = self.swap(os, 'getenv', mock_getenv)
post_swap = self.swap(requests, 'post', mock_post)

with getenv_swap, post_swap:
with self.assertRaisesRegexp(
Exception, 'Unknown build environment.'):
Exception, 'Unknown build environment.'):
flake_checker.report_pass('suiteName')


class MockResponse:
class MockResponse(python_utils.OBJECT):

def __init__(
self, ok=True, json=dict(), status_code=200, reason='foo'):
self, ok=True, json=None, status_code=200, reason='foo'):
if json is None:
json = dict()
self.ok = ok
self.json_dict = json
self.status_code = status_code
self.reason = reason

def json(self):
"""Get json dict or raise ValueError if json_dict not a dict."""
if not isinstance(self.json_dict, dict):
raise ValueError('Payload not JSON.')
return self.json_dict


class IsTestOutputFlakyTests(test_utils.GenericTestBase):

def setUp(self):
Expand All @@ -252,7 +246,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
response = {
'log': ['log1', 'log2'],
'result': True,
Expand Down Expand Up @@ -303,7 +297,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
response = {
'log': ['log1', 'log2'],
'result': True,
Expand Down Expand Up @@ -354,7 +348,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
raise requests.HTTPError()

expected_payload = {
Expand Down Expand Up @@ -396,7 +390,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
return MockResponse(False, None)

expected_payload = {
Expand Down Expand Up @@ -438,7 +432,7 @@ def mock_getenv(variable):
}
return environment_vars.get(variable)

def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-args
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
return MockResponse(True, None)

expected_payload = {
Expand Down
4 changes: 2 additions & 2 deletions scripts/run_e2e_tests_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1009,7 +1009,7 @@ def mock_get_chrome_driver_version():
],),
],
)
args = run_e2e_tests._PARSER.parse_args([])
args = run_e2e_tests._PARSER.parse_args(args=[]) # pylint: disable=protected-access
with check_swap, setup_and_install_swap, register_swap, cleanup_swap:
with build_swap, start_webdriver_swap:
with start_google_app_engine_server_swap:
Expand Down Expand Up @@ -1251,7 +1251,7 @@ def mock_start_portserver():
with register_swap, run_swap, is_test_output_flaky_swap:
with start_portserver_swap, cleanup_portserver_swap:
with on_ci_swap, cleanup_swap, exit_swap:
run_e2e_tests.main(args=['--suite', 'mySuite'])
run_e2e_tests.main(args=['--suite', 'mySuite'])

def test_start_tests_skip_build(self):

Expand Down