Added GitHub init (twentyhq#5317)

- Added github:init to allow full import, as opposed to gitHub:sync which allows partial sync and therefore respecting Github API Limit quota. --------- Co-authored-by: Ady Beraud <a.beraud96@gmail.com>
synapsenet-arena · May 13, 2024 · 4a7aabd · 4a7aabd
1 parent 321ce72
commit 4a7aabd
Show file tree

Hide file tree

Showing 12 changed files with 50 additions and 27 deletions.
diff --git a/packages/twenty-website/package.json b/packages/twenty-website/package.json
@@ -8,7 +8,8 @@
     "build": "npx next build",
     "start": "npx next start",
     "lint": "npx next lint",
-    "github:sync": "npx tsx src/github-sync/github-sync.ts",
+    "github:sync": "npx tsx src/github/github-sync.ts --pageLimit 1",
+    "github:init": "npx tsx src/github/github-sync.ts",
     "database:migrate": "npx tsx src/database/migrate-database.ts",
     "database:generate:pg": "npx drizzle-kit generate:pg --config=src/database/drizzle-posgres.config.ts"
   },

diff --git a/packages/twenty-website/src/github-sync/github-sync.ts b/packages/twenty-website/src/github-sync/github-sync.ts
diff --git a/...c/contributors/fetch-assignable-users.tsx → ...b/contributors/fetch-assignable-users.tsx b/...c/contributors/fetch-assignable-users.tsx → ...b/contributors/fetch-assignable-users.tsx
@@ -1,6 +1,6 @@
 import { graphql } from '@octokit/graphql';
 
-import { Repository } from '@/github-sync/contributors/types';
+import { Repository } from '@/github/contributors/types';
 
 export async function fetchAssignableUsers(
   query: typeof graphql,

diff --git a/...ub-sync/contributors/fetch-issues-prs.tsx → .../github/contributors/fetch-issues-prs.tsx b/...ub-sync/contributors/fetch-issues-prs.tsx → .../github/contributors/fetch-issues-prs.tsx
@@ -4,19 +4,24 @@ import {
   IssueNode,
   PullRequestNode,
   Repository,
-} from '@/github-sync/contributors/types';
+} from '@/github/contributors/types';
+
+// TODO: We should implement a true partial sync instead of using pageLimit.
+// Check search-issues-prs.tsx and modify "updated:>2024-02-27" to make it dynamic
 
 export async function fetchIssuesPRs(
   query: typeof graphql,
   cursor: string | null = null,
   isIssues = false,
   accumulatedData: Array<PullRequestNode | IssueNode> = [],
+  pageLimit: number,
+  currentPage = 0,
 ): Promise<Array<PullRequestNode | IssueNode>> {
   const { repository } = await query<Repository>(
     `
       query ($cursor: String) {
         repository(owner: "twentyhq", name: "twenty") {
-          pullRequests(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) {
+          pullRequests(first: 30, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) {
             nodes {
               id
               title
@@ -89,12 +94,16 @@ export async function fetchIssuesPRs(
     ? repository.issues.pageInfo
     : repository.pullRequests.pageInfo;
 
-  if (pageInfo.hasNextPage) {
+  const newCurrentPage = currentPage + 1;
+
+  if ((!pageLimit || newCurrentPage < pageLimit) && pageInfo.hasNextPage) {
     return fetchIssuesPRs(
       query,
       pageInfo.endCursor,
       isIssues,
       newAccumulatedData,
+      pageLimit,
+      currentPage + 1,
     );
   } else {
     return newAccumulatedData;

diff --git a/...b-sync/contributors/save-issues-to-db.tsx → ...github/contributors/save-issues-to-db.tsx b/...b-sync/contributors/save-issues-to-db.tsx → ...github/contributors/save-issues-to-db.tsx
@@ -5,7 +5,7 @@ import {
   labelModel,
   userModel,
 } from '@/database/model';
-import { IssueNode } from '@/github-sync/contributors/types';
+import { IssueNode } from '@/github/contributors/types';
 
 export async function saveIssuesToDB(
   issues: Array<IssueNode>,

diff --git a/...thub-sync/contributors/save-prs-to-db.tsx → ...rc/github/contributors/save-prs-to-db.tsx b/...thub-sync/contributors/save-prs-to-db.tsx → ...rc/github/contributors/save-prs-to-db.tsx
@@ -5,7 +5,7 @@ import {
   pullRequestModel,
   userModel,
 } from '@/database/model';
-import { PullRequestNode } from '@/github-sync/contributors/types';
+import { PullRequestNode } from '@/github/contributors/types';
 
 export async function savePRsToDB(
   prs: Array<PullRequestNode>,

diff --git a/...b-sync/contributors/search-issues-prs.tsx → ...github/contributors/search-issues-prs.tsx b/...b-sync/contributors/search-issues-prs.tsx → ...github/contributors/search-issues-prs.tsx
@@ -4,7 +4,7 @@ import {
   IssueNode,
   PullRequestNode,
   SearchIssuesPRsQuery,
-} from '@/github-sync/contributors/types';
+} from '@/github/contributors/types';
 
 export async function searchIssuesPRs(
   query: typeof graphql,

diff --git a/...te/src/github-sync/contributors/types.tsx → ...website/src/github/contributors/types.tsx b/...te/src/github-sync/contributors/types.tsx → ...website/src/github/contributors/types.tsx
diff --git a/...github-sync/fetch-and-save-github-data.ts → .../src/github/fetch-and-save-github-data.ts b/...github-sync/fetch-and-save-github-data.ts → .../src/github/fetch-and-save-github-data.ts
@@ -1,15 +1,19 @@
 import { global } from '@apollo/client/utilities/globals';
 import { graphql } from '@octokit/graphql';
 
-import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users';
-import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs';
-import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db';
-import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db';
-import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types';
-import { fetchAndSaveGithubReleases } from '@/github-sync/github-releases/fetch-and-save-github-releases';
-import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars';
-
-export const fetchAndSaveGithubData = async () => {
+import { fetchAssignableUsers } from '@/github/contributors/fetch-assignable-users';
+import { fetchIssuesPRs } from '@/github/contributors/fetch-issues-prs';
+import { saveIssuesToDB } from '@/github/contributors/save-issues-to-db';
+import { savePRsToDB } from '@/github/contributors/save-prs-to-db';
+import { IssueNode, PullRequestNode } from '@/github/contributors/types';
+import { fetchAndSaveGithubReleases } from '@/github/github-releases/fetch-and-save-github-releases';
+import { fetchAndSaveGithubStars } from '@/github/github-stars/fetch-and-save-github-stars';
+
+export const fetchAndSaveGithubData = async ({
+  pageLimit,
+}: {
+  pageLimit: number;
+}) => {
   if (!global.process.env.GITHUB_TOKEN) {
     return new Error('No GitHub token provided');
   }
@@ -31,12 +35,14 @@ export const fetchAndSaveGithubData = async () => {
     null,
     false,
     [],
+    pageLimit,
   )) as Array<PullRequestNode>;
   const fetchedIssues = (await fetchIssuesPRs(
     query,
     null,
     true,
     [],
+    pageLimit,
   )) as Array<IssueNode>;
 
   await savePRsToDB(fetchedPRs, assignableUsers);

diff --git a/...leases/fetch-and-save-github-releases.tsx → ...leases/fetch-and-save-github-releases.tsx b/...leases/fetch-and-save-github-releases.tsx → ...leases/fetch-and-save-github-releases.tsx
@@ -2,7 +2,7 @@ import { graphql } from '@octokit/graphql';
 
 import { insertMany } from '@/database/database';
 import { githubReleasesModel } from '@/database/model';
-import { Repository } from '@/github-sync/contributors/types';
+import { Repository } from '@/github/contributors/types';
 
 export const fetchAndSaveGithubReleases = async (
   query: typeof graphql,

diff --git a/...hub-stars/fetch-and-save-github-stars.tsx → ...hub-stars/fetch-and-save-github-stars.tsx b/...hub-stars/fetch-and-save-github-stars.tsx → ...hub-stars/fetch-and-save-github-stars.tsx
@@ -2,7 +2,7 @@ import { graphql } from '@octokit/graphql';
 
 import { insertMany } from '@/database/database';
 import { githubStarsModel } from '@/database/model';
-import { Repository } from '@/github-sync/contributors/types';
+import { Repository } from '@/github/contributors/types';
 
 export const fetchAndSaveGithubStars = async (
   query: typeof graphql,

diff --git a/packages/twenty-website/src/github/github-sync.ts b/packages/twenty-website/src/github/github-sync.ts
@@ -0,0 +1,15 @@
+import { fetchAndSaveGithubData } from '@/github/fetch-and-save-github-data';
+
+export const githubSync = async () => {
+  const pageLimitFlagIndex = process.argv.indexOf('--pageLimit');
+  let pageLimit = 0;
+
+  if (pageLimitFlagIndex > -1) {
+    pageLimit = parseInt(process.argv[pageLimitFlagIndex + 1], 10);
+  }
+
+  await fetchAndSaveGithubData({ pageLimit });
+  process.exit(0);
+};
+
+githubSync();