diff --git a/packages/twenty-website/package.json b/packages/twenty-website/package.json index 3870679a5c8b..427e22680f9b 100644 --- a/packages/twenty-website/package.json +++ b/packages/twenty-website/package.json @@ -8,7 +8,8 @@ "build": "npx next build", "start": "npx next start", "lint": "npx next lint", - "github:sync": "npx tsx src/github-sync/github-sync.ts", + "github:sync": "npx tsx src/github/github-sync.ts --pageLimit 1", + "github:init": "npx tsx src/github/github-sync.ts", "database:migrate": "npx tsx src/database/migrate-database.ts", "database:generate:pg": "npx drizzle-kit generate:pg --config=src/database/drizzle-posgres.config.ts" }, diff --git a/packages/twenty-website/src/github-sync/github-sync.ts b/packages/twenty-website/src/github-sync/github-sync.ts deleted file mode 100644 index 5abb0e3f0ed9..000000000000 --- a/packages/twenty-website/src/github-sync/github-sync.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { fetchAndSaveGithubData } from '@/github-sync/fetch-and-save-github-data'; - -export const githubSync = async () => { - await fetchAndSaveGithubData(); - process.exit(0); -}; - -githubSync(); diff --git a/packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx b/packages/twenty-website/src/github/contributors/fetch-assignable-users.tsx similarity index 88% rename from packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx rename to packages/twenty-website/src/github/contributors/fetch-assignable-users.tsx index 63d5ebc45bd0..faf922ab2469 100644 --- a/packages/twenty-website/src/github-sync/contributors/fetch-assignable-users.tsx +++ b/packages/twenty-website/src/github/contributors/fetch-assignable-users.tsx @@ -1,6 +1,6 @@ import { graphql } from '@octokit/graphql'; -import { Repository } from '@/github-sync/contributors/types'; +import { Repository } from '@/github/contributors/types'; export async function fetchAssignableUsers( query: typeof graphql, diff --git a/packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx b/packages/twenty-website/src/github/contributors/fetch-issues-prs.tsx similarity index 81% rename from packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx rename to packages/twenty-website/src/github/contributors/fetch-issues-prs.tsx index 0127eb6c2825..0e8802a36d9b 100644 --- a/packages/twenty-website/src/github-sync/contributors/fetch-issues-prs.tsx +++ b/packages/twenty-website/src/github/contributors/fetch-issues-prs.tsx @@ -4,19 +4,24 @@ import { IssueNode, PullRequestNode, Repository, -} from '@/github-sync/contributors/types'; +} from '@/github/contributors/types'; + +// TODO: We should implement a true partial sync instead of using pageLimit. +// Check search-issues-prs.tsx and modify "updated:>2024-02-27" to make it dynamic export async function fetchIssuesPRs( query: typeof graphql, cursor: string | null = null, isIssues = false, accumulatedData: Array = [], + pageLimit: number, + currentPage = 0, ): Promise> { const { repository } = await query( ` query ($cursor: String) { repository(owner: "twentyhq", name: "twenty") { - pullRequests(first: 100, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) { + pullRequests(first: 30, after: $cursor, orderBy: {field: CREATED_AT, direction: DESC}) @skip(if: ${isIssues}) { nodes { id title @@ -89,12 +94,16 @@ export async function fetchIssuesPRs( ? repository.issues.pageInfo : repository.pullRequests.pageInfo; - if (pageInfo.hasNextPage) { + const newCurrentPage = currentPage + 1; + + if ((!pageLimit || newCurrentPage < pageLimit) && pageInfo.hasNextPage) { return fetchIssuesPRs( query, pageInfo.endCursor, isIssues, newAccumulatedData, + pageLimit, + currentPage + 1, ); } else { return newAccumulatedData; diff --git a/packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx b/packages/twenty-website/src/github/contributors/save-issues-to-db.tsx similarity index 95% rename from packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx rename to packages/twenty-website/src/github/contributors/save-issues-to-db.tsx index f56d24784346..3d7b91d15b52 100644 --- a/packages/twenty-website/src/github-sync/contributors/save-issues-to-db.tsx +++ b/packages/twenty-website/src/github/contributors/save-issues-to-db.tsx @@ -5,7 +5,7 @@ import { labelModel, userModel, } from '@/database/model'; -import { IssueNode } from '@/github-sync/contributors/types'; +import { IssueNode } from '@/github/contributors/types'; export async function saveIssuesToDB( issues: Array, diff --git a/packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx b/packages/twenty-website/src/github/contributors/save-prs-to-db.tsx similarity index 95% rename from packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx rename to packages/twenty-website/src/github/contributors/save-prs-to-db.tsx index 27ebf2e984a3..4be5f5821050 100644 --- a/packages/twenty-website/src/github-sync/contributors/save-prs-to-db.tsx +++ b/packages/twenty-website/src/github/contributors/save-prs-to-db.tsx @@ -5,7 +5,7 @@ import { pullRequestModel, userModel, } from '@/database/model'; -import { PullRequestNode } from '@/github-sync/contributors/types'; +import { PullRequestNode } from '@/github/contributors/types'; export async function savePRsToDB( prs: Array, diff --git a/packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx b/packages/twenty-website/src/github/contributors/search-issues-prs.tsx similarity index 98% rename from packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx rename to packages/twenty-website/src/github/contributors/search-issues-prs.tsx index a2f5ed7bc332..03028e3041be 100644 --- a/packages/twenty-website/src/github-sync/contributors/search-issues-prs.tsx +++ b/packages/twenty-website/src/github/contributors/search-issues-prs.tsx @@ -4,7 +4,7 @@ import { IssueNode, PullRequestNode, SearchIssuesPRsQuery, -} from '@/github-sync/contributors/types'; +} from '@/github/contributors/types'; export async function searchIssuesPRs( query: typeof graphql, diff --git a/packages/twenty-website/src/github-sync/contributors/types.tsx b/packages/twenty-website/src/github/contributors/types.tsx similarity index 100% rename from packages/twenty-website/src/github-sync/contributors/types.tsx rename to packages/twenty-website/src/github/contributors/types.tsx diff --git a/packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts b/packages/twenty-website/src/github/fetch-and-save-github-data.ts similarity index 56% rename from packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts rename to packages/twenty-website/src/github/fetch-and-save-github-data.ts index 434cdb4ad774..a0a41f024a0c 100644 --- a/packages/twenty-website/src/github-sync/fetch-and-save-github-data.ts +++ b/packages/twenty-website/src/github/fetch-and-save-github-data.ts @@ -1,15 +1,19 @@ import { global } from '@apollo/client/utilities/globals'; import { graphql } from '@octokit/graphql'; -import { fetchAssignableUsers } from '@/github-sync/contributors/fetch-assignable-users'; -import { fetchIssuesPRs } from '@/github-sync/contributors/fetch-issues-prs'; -import { saveIssuesToDB } from '@/github-sync/contributors/save-issues-to-db'; -import { savePRsToDB } from '@/github-sync/contributors/save-prs-to-db'; -import { IssueNode, PullRequestNode } from '@/github-sync/contributors/types'; -import { fetchAndSaveGithubReleases } from '@/github-sync/github-releases/fetch-and-save-github-releases'; -import { fetchAndSaveGithubStars } from '@/github-sync/github-stars/fetch-and-save-github-stars'; - -export const fetchAndSaveGithubData = async () => { +import { fetchAssignableUsers } from '@/github/contributors/fetch-assignable-users'; +import { fetchIssuesPRs } from '@/github/contributors/fetch-issues-prs'; +import { saveIssuesToDB } from '@/github/contributors/save-issues-to-db'; +import { savePRsToDB } from '@/github/contributors/save-prs-to-db'; +import { IssueNode, PullRequestNode } from '@/github/contributors/types'; +import { fetchAndSaveGithubReleases } from '@/github/github-releases/fetch-and-save-github-releases'; +import { fetchAndSaveGithubStars } from '@/github/github-stars/fetch-and-save-github-stars'; + +export const fetchAndSaveGithubData = async ({ + pageLimit, +}: { + pageLimit: number; +}) => { if (!global.process.env.GITHUB_TOKEN) { return new Error('No GitHub token provided'); } @@ -31,12 +35,14 @@ export const fetchAndSaveGithubData = async () => { null, false, [], + pageLimit, )) as Array; const fetchedIssues = (await fetchIssuesPRs( query, null, true, [], + pageLimit, )) as Array; await savePRsToDB(fetchedPRs, assignableUsers); diff --git a/packages/twenty-website/src/github-sync/github-releases/fetch-and-save-github-releases.tsx b/packages/twenty-website/src/github/github-releases/fetch-and-save-github-releases.tsx similarity index 91% rename from packages/twenty-website/src/github-sync/github-releases/fetch-and-save-github-releases.tsx rename to packages/twenty-website/src/github/github-releases/fetch-and-save-github-releases.tsx index 7237e912bec3..59029c1113c8 100644 --- a/packages/twenty-website/src/github-sync/github-releases/fetch-and-save-github-releases.tsx +++ b/packages/twenty-website/src/github/github-releases/fetch-and-save-github-releases.tsx @@ -2,7 +2,7 @@ import { graphql } from '@octokit/graphql'; import { insertMany } from '@/database/database'; import { githubReleasesModel } from '@/database/model'; -import { Repository } from '@/github-sync/contributors/types'; +import { Repository } from '@/github/contributors/types'; export const fetchAndSaveGithubReleases = async ( query: typeof graphql, diff --git a/packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx b/packages/twenty-website/src/github/github-stars/fetch-and-save-github-stars.tsx similarity index 90% rename from packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx rename to packages/twenty-website/src/github/github-stars/fetch-and-save-github-stars.tsx index 46bdfd992f65..0fac2d5df535 100644 --- a/packages/twenty-website/src/github-sync/github-stars/fetch-and-save-github-stars.tsx +++ b/packages/twenty-website/src/github/github-stars/fetch-and-save-github-stars.tsx @@ -2,7 +2,7 @@ import { graphql } from '@octokit/graphql'; import { insertMany } from '@/database/database'; import { githubStarsModel } from '@/database/model'; -import { Repository } from '@/github-sync/contributors/types'; +import { Repository } from '@/github/contributors/types'; export const fetchAndSaveGithubStars = async ( query: typeof graphql, diff --git a/packages/twenty-website/src/github/github-sync.ts b/packages/twenty-website/src/github/github-sync.ts new file mode 100644 index 000000000000..1c766687e93e --- /dev/null +++ b/packages/twenty-website/src/github/github-sync.ts @@ -0,0 +1,15 @@ +import { fetchAndSaveGithubData } from '@/github/fetch-and-save-github-data'; + +export const githubSync = async () => { + const pageLimitFlagIndex = process.argv.indexOf('--pageLimit'); + let pageLimit = 0; + + if (pageLimitFlagIndex > -1) { + pageLimit = parseInt(process.argv[pageLimitFlagIndex + 1], 10); + } + + await fetchAndSaveGithubData({ pageLimit }); + process.exit(0); +}; + +githubSync();