Skip to content

Commit

Permalink
Fix #21284 and puts in groundwork for #21112: Refactor acceptance tes…
Browse files Browse the repository at this point in the history 
…t configuration and split tests by Docker and Python environments (#21144)

* Refactor acceptance test configuration and split tests by Docker and Python environments

* Uncomment code

* Add install-chrome actiona and remove unnecessary comments

* Refactor acceptance test configuration and split tests by Docker and Python environments

* Remove unnecessary file root-files-mapping.json

* Fixed few backend and mypy tests

* added 'environment' in GenericTestSuiteDict

* fixed basic backed-end failure

* fixed backed-tests

* Fixed linting issues

* Added test for split_test_by_docker

* Update backend-test to cover check_ci_test_suites_to_run fully.

* Added Fix for Issue#21284

* Fixed lint issue

---------

Co-authored-by: Jayam <jayampatel04@gmail.com>
  • Loading branch information
@gp201 @jayam04
gp201 and jayam04 authored Nov 25, 2024
1 parent adc4db2 commit c75c613
Show file tree
Hide file tree
Showing 8 changed files with 281 additions and 74 deletions.
59 changes: 56 additions & 3 deletions .github/workflows/e2e_lighthouse_performance_acceptance_tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ jobs:
needs: check_test_suites_to_run
if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).e2e.count > 0 ||
fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).lighthouse_performance.count > 0 ||
fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.count > 0 }}
fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.docker.count > 0 ||
fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.python.count > 0 }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
Expand Down Expand Up @@ -259,12 +260,12 @@ jobs:
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
acceptance_test:
needs: [check_test_suites_to_run, build]
if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.count > 0 }}
if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.python.count > 0 }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
suite: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.suites }}
suite: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.python.suites }}
name: acceptance_test (${{ matrix.os }}, ${{ matrix.suite.name }})
steps:
- name: Checkout repository
Expand Down Expand Up @@ -353,6 +354,58 @@ jobs:
with:
message: "An acceptance test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
acceptance_test_with_docker:
needs: [check_test_suites_to_run]
if: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.docker.count > 0 }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-22.04]
suite: ${{ fromJson(needs.check_test_suites_to_run.outputs.test_suites_to_run).acceptance.docker.suites }}
name: acceptance_test (${{ matrix.os }}, ${{ matrix.suite.name }})
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Setup Python 3.8
uses: actions/setup-python@v3
with:
python-version: '3.8.15'
architecture: 'x64'
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- name: Merge develop branch into the current branch
uses: ./.github/actions/merge
- name: Install Chrome
uses: ./.github/actions/install-chrome
- name: Initializing Containers
run: |
make build
docker compose up angular-build -d
# This is required as lighthouse tests are run on the host machine and not in the docker container.
sudo docker cp oppia-angular-build:/app/oppia/node_modules .
- name: Run Desktop Acceptance Test ${{ matrix.suite.name }}
run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" make run_tests.acceptance suite=${{ matrix.suite.name }}
- name: Uploading generated test to angular module mapping as an artifact
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: generated-test-to-angular-module-mapping
path: /home/runner/work/oppia/oppia/core/tests/test-modules-mappings/acceptance/${{ matrix.suite.name }}.txt
- name: Run Mobile Acceptance Test ${{ matrix.suite.name }}
run: xvfb-run -a --server-args="-screen 0, 1285x1000x24" make run_tests.acceptance suite=${{ matrix.suite.name }} MOBILE=true
- name: Report failure if failed on oppia/oppia develop branch
if: ${{
failure() &&
github.event_name == 'push' &&
github.repository == 'oppia/oppia' &&
github.ref == 'refs/heads/develop'
}}
uses: ./.github/actions/send-webhook-notification
with:
message: "An acceptance test failed on the upstream develop branch."
webhook-url: ${{ secrets.BUILD_FAILURE_ROOM_WEBHOOK_URL }}
check_e2e_lighthouse_performance_acceptance_workflow_status:
needs: [check_test_suites_to_run, build, e2e_test, lighthouse_performance, acceptance_test]
if: always()
Expand Down
16 changes: 8 additions & 8 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,13 +134,13 @@ restart.%: ## Restarts the given docker service. Example: make restart.datastore
docker compose restart $*

run_tests.prettier: ## Runs the prettier checks
docker compose run --no-deps --entrypoint "npx prettier --check ." dev-server
docker compose run --no-deps --rm --entrypoint "npx prettier --check ." dev-server

run_tests.third_party_size_check: ## Runs the third party size check
docker compose run --no-deps --entrypoint "python -m scripts.third_party_size_check" dev-server
docker compose run --no-deps --rm --entrypoint "python -m scripts.third_party_size_check" dev-server

run_tests.lint: ## Runs the linter tests
docker compose run --no-deps --entrypoint "/bin/sh -c 'git config --global --add safe.directory /app/oppia && python -m scripts.linters.run_lint_checks $(PYTHON_ARGS)'" dev-server
docker compose run --no-deps --rm --entrypoint "/bin/sh -c 'git config --global --add safe.directory /app/oppia && python -m scripts.linters.run_lint_checks $(PYTHON_ARGS)'" dev-server

run_tests.backend: ## Runs the backend tests
@echo 'Shutting down any previously started server.'
Expand All @@ -161,19 +161,19 @@ run_tests.check_overall_backend_test_coverage: ## Runs the check for overall bac
$(MAKE) stop

run_tests.frontend: ## Runs the frontend unit tests
docker compose run --no-deps --entrypoint "python -m scripts.run_frontend_tests $(PYTHON_ARGS) --skip_install" dev-server
docker compose run --no-deps --rm --entrypoint "python -m scripts.run_frontend_tests $(PYTHON_ARGS) --skip_install" dev-server

run_tests.typescript: ## Runs the typescript checks
docker compose run --no-deps --entrypoint "python -m scripts.run_typescript_checks $(PYTHON_ARGS)" dev-server
docker compose run --no-deps --rm --entrypoint "python -m scripts.run_typescript_checks $(PYTHON_ARGS)" dev-server

run_tests.custom_eslint: ## Runs the custome eslint tests
docker compose run --no-deps --entrypoint "python -m scripts.run_custom_eslint_tests" dev-server
docker compose run --no-deps --rm --entrypoint "python -m scripts.run_custom_eslint_tests" dev-server

run_tests.mypy: ## Runs mypy checks
docker compose run --no-deps --entrypoint "python -m scripts.run_mypy_checks" dev-server
docker compose run --no-deps --rm --entrypoint "python -m scripts.run_mypy_checks" dev-server

run_tests.check_backend_associated_tests: ## Runs the backend associate tests
docker compose run --no-deps --entrypoint "/bin/sh -c 'git config --global --add safe.directory /app/oppia && python -m scripts.check_backend_associated_test_file'" dev-server
docker compose run --no-deps --rm --entrypoint "/bin/sh -c 'git config --global --add safe.directory /app/oppia && python -m scripts.check_backend_associated_test_file'" dev-server

run_tests.acceptance: ## Runs the acceptance tests for the parsed suite
## Flag for Acceptance tests
Expand Down
10 changes: 6 additions & 4 deletions core/tests/ci-test-suite-configs/acceptance.json
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
{
"suites": [
{
"name": "blog-admin/assign-roles-to-users-and-change-tag-properties",
"module": "core/tests/puppeteer-acceptance-tests/specs/blog-admin/assign-roles-to-users-and-change-tag-properties.spec.ts"
},
{
"name": "blog-editor/create-and-publish-a-blog-post-with-required-details",
"module": "core/tests/puppeteer-acceptance-tests/specs/blog-editor/create-and-publish-a-blog-post-with-required-details.spec.ts"
Expand Down Expand Up @@ -284,5 +280,11 @@
"name": "topic-manager/create-and-delete-subtopic-and-story",
"module": "core/tests/puppeteer-acceptance-tests/specs/topic-manager/create-and-delete-subtopic-and-story.spec.ts"
}
],
"docker_suites": [
{
"name": "blog-admin/assign-roles-to-users-and-change-tag-properties",
"module": "core/tests/puppeteer-acceptance-tests/specs/blog-admin/assign-roles-to-users-and-change-tag-properties.spec.ts"
}
]
}
9 changes: 6 additions & 3 deletions core/tests/test-dependencies/root-files-mapping-generator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,9 +626,12 @@ class RootFilesMappingGenerator {
.map(file => path.join(CI_TEST_SUITE_CONFIGS_DIRECTORY, file));
for (const testSuiteConfig of testSuiteConfigFiles) {
const config = JSON.parse(fs.readFileSync(testSuiteConfig, 'utf-8'));
const suites = config.suites;
for (const suite of suites) {
testSuiteModules.push(suite.module);
const keys = Object.keys(config);
for (const key of keys) {
const suites = config[key];
for (const suite of suites) {
testSuiteModules.push(suite.module);
}
}
}
return testSuiteModules;
Expand Down
2 changes: 1 addition & 1 deletion docker/pre_commit_hook.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ GIT_USEREMAIL=$(git config user.email)
# We need to pass git username and email to the container, so that it can
# configure git user.name and user.email for the commit which is checked in
# pre-commit hook.
docker compose run -T --no-deps --entrypoint "/bin/sh -c \
docker compose run -T --no-deps --rm --entrypoint "/bin/sh -c \
'git config user.name $GIT_USERNAME && git config user.email $GIT_USEREMAIL \
&& python3 $PYTHON_PRE_COMMIT_SYMLINK $@'" dev-server

Expand Down
42 changes: 40 additions & 2 deletions scripts/check_ci_test_suites_to_run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ class GenericTestSuiteDict(TypedDict):

name: str
module: str
environment: str


class LighthouseTestSuiteDict(GenericTestSuiteDict):
Expand Down Expand Up @@ -105,6 +106,8 @@ class RootFilesConfigDict(TypedDict):
'core', 'tests', 'lighthouse-pages.json')
CI_TEST_SUITE_CONFIGS_DIRECTORY: Final = os.path.join(
'core', 'tests', 'ci-test-suite-configs')
DEFAULT_SUITE = 'suites'
DOCKER_SUITE = 'docker_suites'
TEST_MODULES_MAPPING_DIRECTORY: Final = os.path.join(
'core', 'tests', 'test-modules-mappings')

Expand Down Expand Up @@ -235,6 +238,32 @@ def output_variable_to_github_workflow(
print(f'{output_variable}={output_value}', file=o)


def split_tests_by_docker(
test_suites: CITestSuitesDict
) -> dict[str, CITestSuitesDict]:
"""Splits the test suites into Docker and Python environments.
Args:
test_suites: list(dict). The test suites to split.
Returns:
tuple(list(dict), list(dict)). The test suites split into Docker and
Python environments.
"""
docker_test_suites = []
python_test_suites = []
for test_suite in test_suites.get('suites', []):
print(test_suite)
if test_suite.get('environment', 'python') == 'docker':
docker_test_suites.append(test_suite)
else:
python_test_suites.append(test_suite)
return {
'docker': create_ci_test_suites_dict(docker_test_suites),
'python': create_ci_test_suites_dict(python_test_suites)
}


def output_test_suites_to_run_to_github_workflow(
test_suites_to_run: CITestSuitesToRunDict
) -> None:
Expand All @@ -246,7 +275,9 @@ def output_test_suites_to_run_to_github_workflow(
"""
test_suites_to_run_output = {
'e2e': test_suites_to_run['e2e'],
'acceptance': test_suites_to_run['acceptance'],
'acceptance': split_tests_by_docker(
test_suites_to_run['acceptance']
),
'lighthouse_performance':
test_suites_to_run['lighthouse_performance'],
'lighthouse_accessibility':
Expand Down Expand Up @@ -275,7 +306,13 @@ def get_test_suites_from_config(
list(dict). The test suites from the configuration file.
"""
with open(test_suites_config_file_path, 'r', encoding='utf-8') as f:
suites: List[GenericTestSuiteDict] = json.load(f)['suites']
test_suites = json.load(f)
suites: List[GenericTestSuiteDict] = test_suites[DEFAULT_SUITE]
if DOCKER_SUITE in test_suites.keys():
docker_suites = test_suites[DOCKER_SUITE]
for docker_suite in docker_suites:
docker_suite['environment'] = 'docker'
suites.extend(docker_suites)
return suites


Expand Down Expand Up @@ -321,6 +358,7 @@ def partition_lighthouse_pages_into_test_suites(
str(i // LIGHTHOUSE_PAGES_PER_SHARD + 1)
),
'module': lighthouse_module,
'environment': 'python',
'pages_to_run': []
}
assert current_lighthouse_test_suite is not None
Expand Down
Loading

0 comments on commit c75c613

Please sign in to comment.