ci: Post comment after e2e smoke (MetaMask#8495)

## **Description**  The purpose of these changes is to add a Github check for checking E2E smoke test run status (for PRs that have `Run Smoke E2E` label applied). Under the hood, the action does the following: - Take the latest Bitrise comment, which is posted after the `Run Smoke E2E` label is applied - Extract the commit hash from the comment body - Compare the hash against a list of commits - The list of commits to check against - Starting with the latest commit and moving backwards, it includes all merge from main (from pressing the update with main button) up to the latest non-merge from main commit (a commit that the contributor pushed up) - If the comment commit hash does not match any from the list, the check fails since there is no comment associated with the list of commits - If the comment commit hash does match something from the list, the action checks the build status listed in the comment - If it's pending or failed, then the action fails and the PR is blocked from merge. If it's a success status, then the action passes and the PR is unblocked for merge ## **Related issues** Fixes: ## **Manual testing steps** Scenario `Check Bitrise E2E status` will fail if Bitrise status comment with last non-merge commit hash either doesn't exist, is pending, or failed - GIVEN The Bitrise E2E status check runs - AND The `Run Smoke E2E` label is applied - AND A Bitrise status comment with last non-merge commit hash either doesn't exist, is pending, or failed - THEN the Bitrise E2E status check will fail Scenario `Check Bitrise E2E status` will pass if Bitrise E2E smoke comment with last non-merge commit hash is posted as a successful build - GIVEN The Bitrise E2E status check runs - AND The `Run Smoke E2E` label is applied - AND A Bitrise status comment with last non-merge commit hash shows passed - THEN the E2E comment check will pass and PR merge should be unblocked Scenario `Check Bitrise E2E status` will pass if E2E smoke label is applied - GIVEN The Bitrise E2E status check runs - AND The `Run E2E Smoke` label is not applied to the PR - THEN the E2E comment check will pass and PR merge should be unblocked ## **Screenshots/Recordings**  ### **Before**  ### **After**  Check fails when comment shows pending https://github.com/MetaMask/metamask-mobile/assets/10508597/01be00e3-a0e8-4c14-96eb-e884d9829507 Check fails since no Bitrise comment exists with latest commit https://github.com/MetaMask/metamask-mobile/assets/10508597/ab0ac40d-322f-4f1f-9bff-ee2cc3ef436e Check passes with Bitrise comment showing pass on latest commit https://github.com/MetaMask/metamask-mobile/assets/10508597/9ffbd0c5-4b3b-426d-9396-cfb271da8422 Check passes with Bitrise comment showing pass on latest non-main merge commit https://github.com/MetaMask/metamask-mobile/assets/10508597/415fb89c-a873-4a4f-86c5-afc06c0d6a90 Comment shows failed when Bitrise build fails. Check will also detect this and fail. https://github.com/MetaMask/metamask-mobile/assets/10508597/aee06b95-06b8-4861-8611-d2cdda219fd4 Check passes with Bitrise comment showing pass on a later merge from main commit https://github.com/MetaMask/metamask-mobile/assets/10508597/dacc0c99-e2a4-434c-be7a-00e63cb34c14 ## **Pre-merge author checklist** - [ ] I’ve followed [MetaMask Coding Standards](https://github.com/MetaMask/metamask-mobile/blob/main/.github/guidelines/CODING_GUIDELINES.md). - [ ] I've clearly explained what problem this PR is solving and how it is solved. - [ ] I've linked related issues - [ ] I've included manual testing steps - [ ] I've included screenshots/recordings if applicable - [ ] I’ve included tests if applicable - [ ] I’ve documented my code using [JSDoc](https://jsdoc.app/) format if applicable - [ ] I’ve applied the right labels on the PR (see [labeling guidelines](https://github.com/MetaMask/metamask-mobile/blob/main/.github/guidelines/LABELING_GUIDELINES.md)). Not required for external contributors. - [ ] I’ve properly set the pull request status: - [ ] In case it's not yet "ready for review", I've set it to "draft". - [ ] In case it's "ready for review", I've changed it from "draft" to "non-draft". ## **Pre-merge reviewer checklist** - [ ] I've manually tested the PR (e.g. pull and build branch, run the app, test code being changed). - [ ] I confirm that this PR addresses all acceptance criteria described in the ticket it closes and includes the necessary testing evidence such as recordings and or screenshots.
housepwn · Feb 28, 2024 · a386010 · a386010
1 parent 0d875fb
commit a386010
Show file tree

Hide file tree

Showing 9 changed files with 469 additions and 140 deletions.
diff --git a/.github/scripts/bitrise/check-bitrise-e2e-status.ts b/.github/scripts/bitrise/check-bitrise-e2e-status.ts
@@ -0,0 +1,175 @@
+import * as core from '@actions/core';
+import { context, getOctokit } from '@actions/github';
+import { GitHub } from '@actions/github/lib/utils';
+import { PullRequestTriggerType } from '../scripts.types';
+
+main().catch((error: Error): void => {
+  console.error(error);
+  process.exit(1);
+});
+
+async function main(): Promise<void> {
+  const githubToken = process.env.GITHUB_TOKEN;
+  const e2eLabel = process.env.E2E_LABEL;
+  const triggerAction = context.payload.action as PullRequestTriggerType;
+  const removeAndApplyInstructions = `Remove and re-apply the "${e2eLabel}" label to trigger a E2E smoke test on Bitrise.`;
+  const mergeFromMainCommitMessagePrefix = `Merge branch 'main' into`;
+
+  if (!githubToken) {
+    core.setFailed('GITHUB_TOKEN not found');
+    process.exit(1);
+  }
+
+  if (!e2eLabel) {
+    core.setFailed('E2E_LABEL not found');
+    process.exit(1);
+  }
+
+  const { owner, repo, number: issue_number } = context.issue;
+  const octokit: InstanceType<typeof GitHub> = getOctokit(githubToken);
+
+  // Get PR information
+  const { data: prData } = await octokit.rest.pulls.get({
+    owner,
+    repo,
+    pull_number: issue_number,
+  });
+
+  // Check if the e2e smoke label is applied
+  const labels = prData.labels;
+  const hasSmokeTestLabel = labels.some((label) => label.name === e2eLabel);
+
+  // Pass check since e2e smoke label is not applied
+  if (!hasSmokeTestLabel) {
+    console.log(
+      `"${e2eLabel}" label not applied. Skipping Bitrise status check.`,
+    );
+    return;
+  }
+
+  // Define Bitrise comment tags
+  const bitriseTag = '<!-- BITRISE_TAG -->';
+  const bitrisePendingTag = '<!-- BITRISE_PENDING_TAG -->';
+  const bitriseSuccessTag = '<!-- BITRISE_SUCCESS_TAG -->';
+  const bitriseFailTag = '<!-- BITRISE_FAIL_TAG -->';
+
+  // Get at least the last 30 comments
+  const numberOfTotalComments = prData.comments;
+  const numberOfCommentsToCheck = 30;
+  const lastCommentPage = Math.ceil(
+    numberOfTotalComments / numberOfCommentsToCheck,
+  );
+  const { data: latestCommentBatch } = await octokit.rest.issues.listComments({
+    owner,
+    repo,
+    issue_number: issue_number,
+    page: lastCommentPage,
+    per_page: numberOfCommentsToCheck,
+  });
+  let comments = [...latestCommentBatch];
+  if (
+    numberOfTotalComments % numberOfCommentsToCheck !== 0 &&
+    lastCommentPage > 1
+  ) {
+    // Also fetch previous 30 comments
+    const { data: previousCommentBatch } =
+      await octokit.rest.issues.listComments({
+        owner,
+        repo,
+        issue_number: issue_number,
+        page: lastCommentPage - 1,
+        per_page: numberOfCommentsToCheck,
+      });
+    comments = [...previousCommentBatch, ...comments];
+  }
+
+  const bitriseComment = comments
+    .reverse()
+    .find(({ body }) => body?.includes(bitriseTag));
+
+  // Bitrise comment doesn't exist
+  if (!bitriseComment) {
+    core.setFailed(
+      `No Bitrise build status comment found. ${removeAndApplyInstructions}`,
+    );
+    process.exit(1);
+  }
+
+  // This regex matches a 40-character hexadecimal string enclosed within <!-- and -->
+  let bitriseCommentBody = bitriseComment.body || '';
+  const commitTagRegex = /<!--\s*([0-9a-f]{40})\s*-->/i;
+  const hashMatch = bitriseCommentBody.match(commitTagRegex);
+  let bitriseCommentCommitHash = hashMatch && hashMatch[1] ? hashMatch[1] : '';
+
+  // Get at least the last 10 commits
+  const numberOfTotalCommits = prData.commits;
+  const numberOfCommitsToCheck = 10;
+  const lastCommitPage = Math.ceil(
+    numberOfTotalCommits / numberOfCommitsToCheck,
+  );
+  const { data: latestCommitBatch } = await octokit.rest.pulls.listCommits({
+    owner,
+    repo,
+    pull_number: issue_number,
+    page: lastCommitPage,
+    per_page: numberOfCommitsToCheck,
+  });
+  let commits = [...latestCommitBatch];
+  if (
+    numberOfTotalCommits % numberOfCommitsToCheck !== 0 &&
+    lastCommitPage > 1
+  ) {
+    // Also fetch previous 10 commits
+    const { data: previousCommitBatch } = await octokit.rest.pulls.listCommits({
+      owner,
+      repo,
+      pull_number: issue_number,
+      page: lastCommitPage - 1,
+      per_page: numberOfCommitsToCheck,
+    });
+    commits = [...previousCommitBatch, ...commits];
+  }
+
+  // Relevant hashes include both merge from main commits and the last non-merge from main commit
+  const relevantCommitHashes: string[] = [];
+  for (const commit of commits.reverse()) {
+    const commitMessage = commit.commit.message;
+    relevantCommitHashes.push(commit.sha);
+    if (!commitMessage.includes(mergeFromMainCommitMessagePrefix)) {
+      break;
+    }
+  }
+
+  if (triggerAction === PullRequestTriggerType.Labeled) {
+    // A Bitrise build was triggered for the last commit
+    bitriseCommentCommitHash = relevantCommitHashes[0];
+    bitriseCommentBody = bitrisePendingTag;
+  }
+
+  // Check if Bitrise comment hash matches any of the relevant commit hashes
+  if (relevantCommitHashes.includes(bitriseCommentCommitHash)) {
+    // Check Bitrise build status from comment
+    const bitriseCommentPrefix = `Bitrise build status comment for commit ${bitriseCommentCommitHash}`;
+    if (bitriseCommentBody.includes(bitrisePendingTag)) {
+      core.setFailed(`${bitriseCommentPrefix} is pending.`);
+      process.exit(1);
+    } else if (bitriseCommentBody.includes(bitriseFailTag)) {
+      core.setFailed(`${bitriseCommentPrefix} has failed.`);
+      process.exit(1);
+    } else if (bitriseCommentBody.includes(bitriseSuccessTag)) {
+      console.log(`${bitriseCommentPrefix} has passed.`);
+      return;
+    } else {
+      core.setFailed(
+        `${bitriseCommentPrefix} does not contain any build status. Please verify that the build status tag exists in the comment body.`,
+      );
+      process.exit(1);
+    }
+  } else {
+    // No build comment found for relevant commits
+    core.setFailed(
+      `No Bitrise build comment exists for latest commits. ${removeAndApplyInstructions}`,
+    );
+    process.exit(1);
+  }
+}
diff --git a/...rise-e2e-smoke/check-e2e-smoke-trigger.ts → .github/scripts/bitrise/detect-e2e-label.ts b/...rise-e2e-smoke/check-e2e-smoke-trigger.ts → .github/scripts/bitrise/detect-e2e-label.ts
@@ -1,11 +1,7 @@
 import * as core from '@actions/core';
 import { context, getOctokit } from '@actions/github';
 import { GitHub } from '@actions/github/lib/utils';
-
-enum PullRequestTriggerType {
-  ReadyForReview = 'ready_for_review',
-  Labeled = 'labeled',
-}
+import { PullRequestTriggerType } from '../scripts.types';
 
 main().catch((error: Error): void => {
   console.error(error);
@@ -78,5 +74,5 @@ async function main(): Promise<void> {
   }
 
   // Set the output for the next step to use.
-  core.setOutput("shouldTriggerE2E", shouldTriggerE2E);
+  core.setOutput('shouldTriggerE2E', shouldTriggerE2E);
 }
diff --git a/.github/scripts/bitrise/run-bitrise-e2e-tests.ts b/.github/scripts/bitrise/run-bitrise-e2e-tests.ts
@@ -0,0 +1,166 @@
+import * as core from '@actions/core';
+import { context, getOctokit } from '@actions/github';
+import { GitHub } from '@actions/github/lib/utils';
+import axios from 'axios';
+
+main().catch((error: Error): void => {
+  console.error(error);
+  process.exit(1);
+});
+
+async function main(): Promise<void> {
+  const e2eLabel = process.env.E2E_LABEL;
+  const githubToken = process.env.GITHUB_TOKEN;
+  const e2ePipeline = process.env.E2E_PIPELINE;
+  const workflowName = process.env.WORKFLOW_NAME;
+  const pullRequestNumber = context.issue.number;
+  const repoOwner = context.repo.owner;
+  const repo = context.repo.repo;
+  const pullRequestLink = `https://github.com/MetaMask/metamask-mobile/pull/${pullRequestNumber}`;
+
+  if (!githubToken) {
+    core.setFailed('GITHUB_TOKEN not found');
+    process.exit(1);
+  }
+
+  if (!e2ePipeline) {
+    core.setFailed('E2E_PIPELINE not found');
+    process.exit(1);
+  }
+
+  const octokit: InstanceType<typeof GitHub> = getOctokit(githubToken);
+
+  // Get the latest commit hash
+  const pullRequestResponse = await octokit.rest.pulls.get({
+    owner: repoOwner,
+    repo,
+    pull_number: pullRequestNumber,
+  });
+
+  const latestCommitHash = pullRequestResponse.data.head.sha;
+
+  // Configure Bitrise configuration for API call
+  const data = {
+    build_params: {
+      branch: process.env.GITHUB_HEAD_REF,
+      pipeline_id: e2ePipeline,
+      environments: [
+        {
+          mapped_to: 'GITHUB_PR_NUMBER',
+          value: `${pullRequestNumber}`,
+          is_expand: true,
+        },
+        {
+          mapped_to: 'TRIGGERED_BY_PR_LABEL',
+          value: `true`,
+          is_expand: true,
+        },
+        {
+          mapped_to: 'GITHUB_PR_HASH',
+          value: `${latestCommitHash}`,
+          is_expand: true,
+        },
+      ],
+      commit_message: `Triggered by (${workflowName}) workflow in ${pullRequestLink}`,
+    },
+    hook_info: {
+      type: 'bitrise',
+      build_trigger_token: process.env.BITRISE_BUILD_TRIGGER_TOKEN,
+    },
+    triggered_by: workflowName,
+  };
+
+  const bitriseProjectUrl = `https://app.bitrise.io/app/${process.env.BITRISE_APP_ID}`;
+  const bitriseBuildStartUrl = `${bitriseProjectUrl}/build/start.json`;
+
+  // Start Bitrise build.
+  const bitriseBuildResponse = await axios.post(bitriseBuildStartUrl, data, {
+    headers: {
+      'Content-Type': 'application/json',
+    },
+  });
+
+  if (!bitriseBuildResponse.data.build_slug) {
+    core.setFailed(`Bitrise build slug not found`);
+    process.exit(1);
+  }
+
+  const bitriseTag = '<!-- BITRISE_TAG -->';
+  const bitrisePendingTag = '<!-- BITRISE_PENDING_TAG -->';
+  const latestCommitTag = `<!-- ${latestCommitHash} -->`;
+  const buildLink = `${bitriseProjectUrl}/pipelines/${bitriseBuildResponse.data.build_slug}`;
+  const message = `## [<img alt="https://bitrise.io/" src="https://assets-global.website-files.com/5db35de024bb983af1b4e151/5e6f9ccc3e129dfd8a205e4e_Bitrise%20Logo%20-%20Eggplant%20Bg.png" height="20">](${buildLink}) **Bitrise**\n\n🔄🔄🔄 \`${e2ePipeline}\` started on Bitrise...🔄🔄🔄\n\nCommit hash: ${latestCommitHash}\nBuild link: ${buildLink}\n\n>[!NOTE]\n>- This comment will auto-update when build completes\n>- You can kick off another \`${e2ePipeline}\` on Bitrise by removing and re-applying the \`${e2eLabel}\` label on the pull request\n${bitriseTag}\n${bitrisePendingTag}\n\n${latestCommitTag}`;
+
+  if (bitriseBuildResponse.status === 201) {
+    console.log(`Started Bitrise build at ${buildLink}`);
+  } else {
+    core.setFailed(
+      `Bitrise build request returned with status code ${bitriseBuildResponse.status}`,
+    );
+    process.exit(1);
+  }
+
+  // Reopen conversation in case it's locked
+  const unlockConvoResponse = await octokit.rest.issues.unlock({
+    owner: repoOwner,
+    repo,
+    issue_number: pullRequestNumber,
+  });
+
+  if (unlockConvoResponse.status === 204) {
+    console.log(`Unlocked conversation for PR ${pullRequestLink}`);
+  } else {
+    core.setFailed(
+      `Unlock conversation request returned with status code ${unlockConvoResponse.status}`,
+    );
+    process.exit(1);
+  }
+
+  // Look for existing Bitrise comment.
+  const { data: comments } = await octokit.rest.issues.listComments({
+    owner: repoOwner,
+    repo,
+    issue_number: pullRequestNumber,
+  });
+
+  const bitriseComment = comments.find(({ body }) =>
+    body?.includes(latestCommitTag),
+  );
+
+  // Existing comment exists for commit hash. Update comment with pending status.
+  if (bitriseComment) {
+    const updateCommentResponse = await octokit.rest.issues.updateComment({
+      owner: repoOwner,
+      repo,
+      issue_number: pullRequestNumber,
+      body: message,
+      comment_id: bitriseComment.id,
+    });
+
+    if (updateCommentResponse.status === 200) {
+      console.log(`Updating comment in pull request ${pullRequestLink}`);
+    } else {
+      core.setFailed(
+        `Update comment request returned with status code ${updateCommentResponse.status}`,
+      );
+      process.exit(1);
+    }
+  } else {
+    // Post new Bitrise comment in PR.
+    const postCommentResponse = await octokit.rest.issues.createComment({
+      owner: repoOwner,
+      repo,
+      issue_number: pullRequestNumber,
+      body: message,
+    });
+
+    if (postCommentResponse.status === 201) {
+      console.log(`Posting comment in pull request ${pullRequestLink}`);
+    } else {
+      core.setFailed(
+        `Post comment request returned with status code ${postCommentResponse.status}`,
+      );
+      process.exit(1);
+    }
+  }
+}