diff --git a/backend/src/database/migrations/U1772043927__removeReposFromGithubSettings.sql b/backend/src/database/migrations/U1772043927__removeReposFromGithubSettings.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/database/migrations/V1772043927__removeReposFromGithubSettings.sql b/backend/src/database/migrations/V1772043927__removeReposFromGithubSettings.sql new file mode 100644 index 0000000000..c1295f972d --- /dev/null +++ b/backend/src/database/migrations/V1772043927__removeReposFromGithubSettings.sql @@ -0,0 +1,30 @@ +-- Backup settings before any modifications +CREATE TABLE integration.integrations_settings_backup_02_28_2026 AS +SELECT id, settings FROM integrations; + +-- Strip repos from orgs in settings for github and github-nango integrations +-- Repos now live in public.repositories table and are populated into API responses +-- via the compatibility layer in integrationRepository._populateRelations +UPDATE integrations +SET settings = jsonb_set( + settings, + '{orgs}', + ( + SELECT coalesce(jsonb_agg( + org - 'repos' + ), '[]'::jsonb) + FROM jsonb_array_elements(settings->'orgs') org + ) +) +WHERE platform IN ('github', 'github-nango') + AND settings->'orgs' IS NOT NULL + AND "deletedAt" IS NULL + AND status != 'mapping'; + +-- Also clean up top-level repos/unavailableRepos if present +UPDATE integrations +SET settings = settings - 'repos' - 'unavailableRepos' +WHERE platform IN ('github', 'github-nango') + AND (settings ? 'repos' OR settings ? 'unavailableRepos') + AND "deletedAt" IS NULL + AND status != 'mapping'; diff --git a/backend/src/database/repositories/integrationRepository.ts b/backend/src/database/repositories/integrationRepository.ts index a70499a700..45ba7680d2 100644 --- a/backend/src/database/repositories/integrationRepository.ts +++ b/backend/src/database/repositories/integrationRepository.ts @@ -585,10 +585,9 @@ class IntegrationRepository { const output = record.get({ plain: true }) - // For github-nango integrations, populate settings.nangoMapping from the dedicated table - // so the API contract remains unchanged for frontend consumers + // For github-nango integrations, populate settings.nangoMapping from dedicated table if (output.platform === PlatformType.GITHUB_NANGO) { - const rows = await record.sequelize.query( + const nangoRows = await record.sequelize.query( `SELECT "connectionId", owner, "repoName" FROM integration.nango_mapping WHERE "integrationId" = :integrationId`, { replacements: { integrationId: output.id }, @@ -596,15 +595,71 @@ class IntegrationRepository { }, ) - if (rows.length > 0) { + if (nangoRows.length > 0) { const nangoMapping: Record = {} - for (const row of rows as { connectionId: string; owner: string; repoName: string }[]) { + for (const row of nangoRows as { + connectionId: string + owner: string + repoName: string + }[]) { nangoMapping[row.connectionId] = { owner: row.owner, repoName: row.repoName } } output.settings = { ...output.settings, nangoMapping } } } + // For both github and github-nango, populate orgs[].repos from repositories table + if ( + (output.platform === PlatformType.GITHUB || output.platform === PlatformType.GITHUB_NANGO) && + output.settings?.orgs?.length > 0 + ) { + const repoRows = (await record.sequelize.query( + `SELECT url, split_part(url, '/', -1) as name, split_part(url, '/', -2) as owner, "forkedFrom", "updatedAt" + FROM public.repositories + WHERE "sourceIntegrationId" = :integrationId AND "deletedAt" IS NULL + ORDER BY url`, + { + replacements: { integrationId: output.id }, + type: QueryTypes.SELECT, + }, + )) as { + url: string + name: string + owner: string + forkedFrom: string | null + updatedAt: string + }[] + + // Only overwrite orgs[].repos from the repositories table if there are rows. + // During the 'mapping' phase (legacy github connect), repos live in settings + // before being written to the repositories table via mapGithubRepos. + if (repoRows.length > 0) { + const reposByOwner: Record = {} + for (const repo of repoRows) { + if (!reposByOwner[repo.owner]) reposByOwner[repo.owner] = [] + reposByOwner[repo.owner].push(repo) + } + + output.settings = { + ...output.settings, + orgs: output.settings.orgs.map((org) => ({ + ...org, + repos: (reposByOwner[org.name] || []).map((r) => ({ + url: r.url, + name: r.name, + owner: r.owner, + forkedFrom: r.forkedFrom, + updatedAt: r.updatedAt, + })), + })), + } + } + + // Strip legacy top-level keys that may still exist in the DB column + delete output.settings.repos + delete output.settings.unavailableRepos + } + return output } } diff --git a/backend/src/services/collectionService.ts b/backend/src/services/collectionService.ts index 51be9693d1..74bfd1d84e 100644 --- a/backend/src/services/collectionService.ts +++ b/backend/src/services/collectionService.ts @@ -27,6 +27,7 @@ import { } from '@crowd/data-access-layer/src/collections' import { fetchIntegrationsForSegment } from '@crowd/data-access-layer/src/integrations' import { QueryFilter } from '@crowd/data-access-layer/src/query' +import { getReposForGithubIntegration } from '@crowd/data-access-layer/src/repositories' import { ICreateRepositoryGroup, IRepositoryGroup, @@ -533,13 +534,8 @@ export class CollectionService extends LoggerBase { return listRepositoryGroups(qx, { insightsProjectId }) } - static isSingleRepoOrg(orgs: GithubIntegrationSettings['orgs']): boolean { - return ( - Array.isArray(orgs) && - orgs.length === 1 && - Array.isArray(orgs[0]?.repos) && - orgs[0].repos.length === 1 - ) + static isSingleRepoOrg(orgs: GithubIntegrationSettings['orgs'], repoCount: number): boolean { + return Array.isArray(orgs) && orgs.length === 1 && repoCount === 1 } /** @@ -613,13 +609,12 @@ export class CollectionService extends LoggerBase { const settings = githubIntegration.settings as GithubIntegrationSettings // The orgs must have at least one repo - if ( - !settings?.orgs || - !Array.isArray(settings.orgs) || - settings.orgs.length === 0 || - !Array.isArray(settings.orgs[0].repos) || - settings.orgs[0].repos.length === 0 - ) { + if (!settings?.orgs || !Array.isArray(settings.orgs) || settings.orgs.length === 0) { + return null + } + + const repos = await getReposForGithubIntegration(qx, githubIntegration.id) + if (repos.length === 0) { return null } @@ -633,11 +628,8 @@ export class CollectionService extends LoggerBase { return null } - const details = CollectionService.isSingleRepoOrg(settings.orgs) - ? await GithubIntegrationService.findRepoDetails( - mainOrg.name, - settings.orgs[0].repos[0].name, - ) + const details = CollectionService.isSingleRepoOrg(settings.orgs, repos.length) + ? await GithubIntegrationService.findRepoDetails(mainOrg.name, repos[0].name) : { ...(await GithubIntegrationService.findOrgDetails(mainOrg.name)), topics: mainOrg.topics, diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 570a36bd13..fac1822a88 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -21,11 +21,13 @@ import { IRepository, IRepositoryMapping, getIntegrationReposMapping, + getReposForGithubIntegration, getRepositoriesBySourceIntegrationId, getRepositoriesByUrl, insertRepositories, restoreRepositories, softDeleteRepositories, + stripReposFromGithubSettings, } from '@crowd/data-access-layer/src/repositories' import { getMappedAllWithSegmentName, @@ -801,6 +803,24 @@ export default class IntegrationService { const txService = new IntegrationService(txOptions) try { + // Extract repos from orgs and build forkedFrom map, then store settings without repos + const forkedFromMap = new Map() + if (settings?.orgs) { + for (const org of settings.orgs) { + for (const repo of org.repos || []) { + if (repo.url) { + forkedFromMap.set(repo.url, repo.forkedFrom || null) + } + } + } + } + + // Strip repos from orgs before storing in settings + const settingsToStore = { + ...settings, + orgs: (settings?.orgs || []).map(({ repos: _repos, ...org }) => org), + } + let integration if (!integrationId) { @@ -808,26 +828,26 @@ export default class IntegrationService { integration = await txService.createOrUpdate( { platform: PlatformType.GITHUB_NANGO, - settings, + settings: settingsToStore, status: 'done', }, transaction, ) // create github mapping - this also creates git integration - await txService.mapGithubRepos(integration.id, mapping, false) + await txService.mapGithubRepos(integration.id, mapping, false, forkedFromMap) } else { // update existing integration integration = await txService.findById(integrationId) // create github mapping - this also creates git integration - await txService.mapGithubRepos(integrationId, mapping, false) + await txService.mapGithubRepos(integrationId, mapping, false, forkedFromMap) integration = await txService.createOrUpdate( { id: integrationId, platform: PlatformType.GITHUB_NANGO, - settings, + settings: settingsToStore, }, transaction, ) @@ -858,7 +878,12 @@ export default class IntegrationService { } } - async mapGithubRepos(integrationId, mapping, fireOnboarding = true) { + async mapGithubRepos( + integrationId, + mapping, + fireOnboarding = true, + forkedFromMap?: Map, + ) { this.options.log.info(`Mapping GitHub repos for integration ${integrationId}!`) const transaction = await SequelizeRepository.createTransaction(this.options) @@ -866,6 +891,7 @@ export default class IntegrationService { ...this.options, transaction, } + let onboardingIntegration try { // add the repos to the git integration const repos: Record = Object.entries(mapping).reduce( @@ -880,9 +906,29 @@ export default class IntegrationService { ) // Note: Repos are synced to public.repositories via mapUnifiedRepositories at the end of this method - // Get integration settings to access forkedFrom data from all orgs const integration = await IntegrationRepository.findById(integrationId, txOptions) - const allReposInSettings = integration.settings?.orgs?.flatMap((org) => org.repos || []) || [] + + // Build forkedFrom map from repositories table if not provided + if (!forkedFromMap) { + forkedFromMap = new Map() + const qx = SequelizeRepository.getQueryExecutor(txOptions) + const existingRepos = await getReposForGithubIntegration(qx, integrationId) + if (existingRepos.length > 0) { + for (const repo of existingRepos) { + forkedFromMap.set(repo.url, repo.forkedFrom) + } + } else { + // On first mapping, repositories table is empty — read forkedFrom from settings + const orgs = integration.settings?.orgs || [] + for (const org of orgs) { + for (const repo of org.repos || []) { + if (repo.url) { + forkedFromMap.set(repo.url, repo.forkedFrom || null) + } + } + } + } + } for (const [segmentId, urls] of Object.entries(repos)) { let isGitintegrationConfigured @@ -904,6 +950,9 @@ export default class IntegrationService { isGitintegrationConfigured = false } + const buildRemotes = (urlList: string[]) => + urlList.map((url) => ({ url, forkedFrom: forkedFromMap.get(url) || null })) + if (isGitintegrationConfigured) { this.options.log.info(`Finding Git integration for segment ${segmentId}!`) const gitInfo = await this.gitGetRemotes(segmentOptions) @@ -911,24 +960,14 @@ export default class IntegrationService { const allUrls = Array.from(new Set([...gitRemotes, ...urls])) this.options.log.info(`Updating Git integration for segment ${segmentId}!`) await this.gitConnectOrUpdate( - { - remotes: allUrls.map((url) => { - const repoInSettings = allReposInSettings.find((r) => r.url === url) - return { url, forkedFrom: repoInSettings?.forkedFrom || null } - }), - }, + { remotes: buildRemotes(allUrls) }, segmentOptions, PlatformType.GITHUB, ) } else { this.options.log.info(`Updating Git integration for segment ${segmentId}!`) await this.gitConnectOrUpdate( - { - remotes: urls.map((url) => { - const repoInSettings = allReposInSettings.find((r) => r.url === url) - return { url, forkedFrom: repoInSettings?.forkedFrom || null } - }), - }, + { remotes: buildRemotes(urls) }, segmentOptions, PlatformType.GITHUB, ) @@ -937,19 +976,25 @@ export default class IntegrationService { // sync to public.repositories const txService = new IntegrationService(txOptions) - await txService.mapUnifiedRepositories(integration.platform, integrationId, mapping) + await txService.mapUnifiedRepositories( + integration.platform, + integrationId, + mapping, + true, + forkedFromMap, + ) + + // Now that repos are in the repositories table, strip them from settings + const qxTx = SequelizeRepository.getQueryExecutor(txOptions) + await stripReposFromGithubSettings(qxTx, integrationId) if (fireOnboarding) { this.options.log.info('Updating integration status to in-progress!') - const integration = await IntegrationRepository.update( + onboardingIntegration = await IntegrationRepository.update( integrationId, { status: 'in-progress' }, txOptions, ) - - this.options.log.info('Sending GitHub message to int-run-worker!') - const emitter = await getIntegrationRunWorkerEmitter() - await emitter.triggerIntegrationRun(integration.platform, integration.id, true) } await SequelizeRepository.commitTransaction(transaction) @@ -962,6 +1007,17 @@ export default class IntegrationService { } throw err } + + // Trigger the run AFTER commit so the worker can see the repositories rows + if (onboardingIntegration) { + this.options.log.info('Sending GitHub message to int-run-worker!') + const emitter = await getIntegrationRunWorkerEmitter() + await emitter.triggerIntegrationRun( + onboardingIntegration.platform, + onboardingIntegration.id, + true, + ) + } } /** @@ -2388,13 +2444,8 @@ export default class IntegrationService { const githubToken = await getGithubInstallationToken() - const repos = integration.settings.orgs.flatMap((org) => org.repos) as { - url: string - name: string - updatedAt: string - }[] - const qx = SequelizeRepository.getQueryExecutor(this.options) + const repos = await getReposForGithubIntegration(qx, integrationId) const githubRepos = await getRepositoriesBySourceIntegrationId(qx, integrationId) const mappedSegments = githubRepos.map((repo) => repo.segmentId) @@ -2841,46 +2892,13 @@ export default class IntegrationService { const repos = await getInstalledRepositories(installToken) this.options.log.info(`Fetched ${repos.length} installed repositories`) - // Update integration settings - const currentSettings: { - orgs: Array<{ - name: string - logo: string - url: string - fullSync: boolean - updatedAt: string - repos: Array<{ - name: string - url: string - updatedAt: string - }> - }> - } = integration.settings || { orgs: [] } - - if (currentSettings.orgs.length !== 1) { - throw new Error('Integration settings must have exactly one organization') - } - - const currentRepos = currentSettings.orgs[0].repos || [] - const newRepos = repos.filter((repo) => !currentRepos.some((r) => r.url === repo.url)) - this.options.log.info(`Found ${newRepos.length} new repositories`) + // Get current repos from repositories table + const qx = SequelizeRepository.getQueryExecutor(this.options) + const currentRepoRows = await getReposForGithubIntegration(qx, integration.id) + const currentRepoUrls = new Set(currentRepoRows.map((r) => r.url)) - const updatedSettings = { - ...currentSettings, - orgs: [ - { - ...currentSettings.orgs[0], - repos: [ - ...currentRepos, - ...newRepos.map((repo) => ({ - name: repo.name, - url: repo.url, - updatedAt: repo.updatedAt || new Date().toISOString(), - })), - ], - }, - ], - } + const newRepos = repos.filter((repo) => !currentRepoUrls.has(repo.url)) + this.options.log.info(`Found ${newRepos.length} new repositories`) this.options = { ...this.options, @@ -2891,20 +2909,17 @@ export default class IntegrationService { ], } - // Update the integration with new settings - await this.update(integration.id, { settings: updatedSettings }) - - this.options.log.info(`Updated integration settings for integration id: ${integration.id}`) - - // Update GitHub repos mapping + // Update GitHub repos mapping — new repos are added to repositories table via mapGithubRepos const defaultSegmentId = integration.segmentId const mapping = {} + const forkedFromMap = new Map() for (const repo of newRepos) { mapping[repo.url] = defaultSegmentId + forkedFromMap.set(repo.url, repo.forkedFrom || null) } if (Object.keys(mapping).length > 0) { // false - not firing onboarding - await this.mapGithubRepos(integration.id, mapping, false) + await this.mapGithubRepos(integration.id, mapping, false, forkedFromMap) this.options.log.info(`Updated GitHub repos mapping for integration id: ${integration.id}`) } else { this.options.log.info(`No new repos to map for integration id: ${integration.id}`) @@ -3002,6 +3017,7 @@ export default class IntegrationService { sourcePlatform: PlatformType, sourceIntegrationId: string, txOptions: IRepositoryOptions, + existingForkedFromMap?: Map, ): Promise { if (urls.length === 0) { return [] @@ -3045,19 +3061,19 @@ export default class IntegrationService { } } - // Build forkedFrom map from integration settings (for GITHUB repositories) - const forkedFromMap = new Map() - const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes( - sourcePlatform, - ) - const sourceIntegration = isGitHubPlatform - ? await IntegrationRepository.findById(sourceIntegrationId, txOptions) - : null - if (sourceIntegration?.settings?.orgs) { - const allRepos = sourceIntegration.settings.orgs.flatMap((org: any) => org.repos || []) - for (const repo of allRepos) { - if (repo.url && repo.forkedFrom) { - forkedFromMap.set(repo.url, repo.forkedFrom) + // Build forkedFrom map from existing repositories (for GITHUB platforms) + // Use the passed map if available (on first mapping, repositories table is empty) + const forkedFromMap = existingForkedFromMap ?? new Map() + if (!existingForkedFromMap) { + const isGitHubPlatform = [PlatformType.GITHUB, PlatformType.GITHUB_NANGO].includes( + sourcePlatform, + ) + if (isGitHubPlatform) { + const existingRepos = await getReposForGithubIntegration(qx, sourceIntegrationId) + for (const repo of existingRepos) { + if (repo.forkedFrom) { + forkedFromMap.set(repo.url, repo.forkedFrom) + } } } } @@ -3089,6 +3105,7 @@ export default class IntegrationService { sourceIntegrationId: string, mapping: { [url: string]: string }, skipMirroredRepos = true, + forkedFromMap?: Map, ) { // Check for existing transaction to support nested calls within outer transactions const existingTransaction = SequelizeRepository.getTransaction(this.options) @@ -3155,6 +3172,7 @@ export default class IntegrationService { sourcePlatform, sourceIntegrationId, txOptions, + forkedFromMap, ) if (payloads.length > 0) { await insertRepositories(qx, payloads) @@ -3172,6 +3190,7 @@ export default class IntegrationService { sourcePlatform, sourceIntegrationId, txOptions, + forkedFromMap, ) if (restorePayloads.length > 0) { await restoreRepositories(qx, restorePayloads) diff --git a/frontend/src/modules/admin/modules/integration/pages/integration-list.page.vue b/frontend/src/modules/admin/modules/integration/pages/integration-list.page.vue index bfb16905a9..efb44fb303 100644 --- a/frontend/src/modules/admin/modules/integration/pages/integration-list.page.vue +++ b/frontend/src/modules/admin/modules/integration/pages/integration-list.page.vue @@ -128,7 +128,22 @@ const { array, loadingFetch } = mapGetters('integration'); const { id, grandparentId } = route.params; -const useGitHubNango = ref(false); // true for v2, false for v1 +const authStore = useAuthStore(); +const userId = computed(() => authStore.user?.id); +const teamUserIds = computed(() => config.permissions.teamUserIds); +const env = computed(() => config.env); + +const isTeamUser = computed(() => env.value !== 'production' || teamUserIds.value?.includes(userId.value)); + +const useGitHubNango = computed(() => { + const githubIntegration = array.value.find( + (integration: any) => integration.platform === 'github', + ); + if (githubIntegration) { + return !!githubIntegration.isNango; + } + return !!isTeamUser.value; +}); const subproject = ref(); @@ -153,34 +168,15 @@ const platformsByStatus = computed(() => { return all.filter((platform) => matching.includes(platform)); }); -const authStore = useAuthStore(); -const userId = computed(() => authStore.user?.id); -const teamUserIds = computed(() => config.permissions.teamUserIds); -const env = computed(() => config.env); - -const isTeamUser = computed(() => env.value !== 'production' || teamUserIds.value?.includes(userId.value)); - onMounted(() => { localStorage.setItem('segmentId', id as string); localStorage.setItem('segmentGrandparentId', grandparentId as string); - doFetch().then(() => { - useGitHubNango.value = updateGithubVersion(); - }); + doFetch(); findSubProject(id).then((res) => { subproject.value = res; }); }); - -const updateGithubVersion = () => { - const githubIntegration = array.value.find( - (integration: any) => integration.platform === 'github', - ); - if (githubIntegration) { - return !!githubIntegration.isNango; - } - return !!isTeamUser.value; -};