diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b2ecc3c1..233826fed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Migrated connection syncing, repository indexing, permission syncing, and background pruning from in-process managers and pollers to BullMQ workloads with retries and per-resource execution locking. [#1427](https://github.com/sourcebot-dev/sourcebot/pull/1427) +- Refactored the repository and connection list view to display useful debugging information to owners, such as failure reasons and logs. [#1608](https://github.com/sourcebot-dev/sourcebot/pull/1608) ### Fixed - Upgraded Next.js to 16.3.1 to bound memory retained by high-cardinality dynamic route cache entries. [#1594](https://github.com/sourcebot-dev/sourcebot/pull/1594) diff --git a/CLAUDE.md b/CLAUDE.md index db943715c..9f2c8623a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,8 +32,9 @@ Use the workload system in `packages/backend` for background work. Define the qu ### Lifecycle state -- In `onStarted`, upsert the workload-specific job row as `IN_PROGRESS`. If the parent resource tracks a `latest...JobId`, update that pointer in the same database transaction. -- Completion and terminal-failure hooks must always update their own historical job row by job ID. Do not condition that update on the job still being latest. Every job row should record its actual outcome. +- BullMQ is the default source of truth for workload lifecycle state. Only persist a separate database job row when the product explicitly requires durable history beyond the queue's retention policy. +- If a parent resource tracks a `latest...JobId`, update that pointer when the job starts so consumers can resolve its state from BullMQ. +- For workloads that persist historical job rows, completion and terminal-failure hooks must always update their own row by job ID. Do not condition that update on the job still being latest. Every persisted job row should record its actual outcome. - `onTerminalFailure` only runs after the job exhausts all retry attempts. Intermediate failures are retried without marking the lifecycle row as terminally failed. - If a completion or failure hook publishes state onto the parent resource, use a conditional `updateMany` keyed by both the resource ID and its `latest...JobId`. This prevents an older hook from overwriting state belonging to a newer job after the execution lock has been released. - Parent-resource state written inside `process` is already serialized by the execution lock. It does not need a latest-job conditional merely because the resource tracks the latest job ID. diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 3cb61ebb8..0e22f75e6 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -7,7 +7,7 @@ import { } from '@bull-board/metrics'; import { Octokit } from '@octokit/rest'; import * as Sentry from "@sentry/node"; -import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; +import { PrismaClient } from '@sourcebot/db'; import { createLogger, env, JOB_PRIORITIES } from '@sourcebot/shared'; import express, { NextFunction, Request, Response } from 'express'; import 'express-async-errors'; @@ -176,7 +176,6 @@ const scheduleAndTriggerRepoIndexing = async ({ reindexIntervalMs, { repoId, - type: RepoIndexingJobType.INDEX, }, { priority: JOB_PRIORITIES.SCHEDULED }, ); @@ -185,7 +184,6 @@ const scheduleAndTriggerRepoIndexing = async ({ "repo-index", { repoId, - type: RepoIndexingJobType.INDEX, }, { priority: JOB_PRIORITIES.INTERACTIVE }, ); diff --git a/packages/backend/src/attachmentPruneWorkload.ts b/packages/backend/src/attachmentPruneWorkload.ts index f9ddf862e..a21a60dd2 100644 --- a/packages/backend/src/attachmentPruneWorkload.ts +++ b/packages/backend/src/attachmentPruneWorkload.ts @@ -18,12 +18,6 @@ interface Props { storage?: StorageBackend; } -interface AttachmentPruneResult { - pendingClaimed: number; - committedClaimed: number; - reclaimed: number; -} - /** * Reclaims orphaned attachment blobs using the `DELETING` tombstone protocol: * an orphan is first atomically flipped to `DELETING`, then its bytes are @@ -47,7 +41,7 @@ export const createAttachmentPruneWorkload = ({ db, ttlHours, storage = getStorageBackend(), -}: Props): Workload<"attachment-prune", AttachmentPruneResult> => ({ +}: Props): Workload<"attachment-prune"> => ({ queueSpec: ATTACHMENT_PRUNE_QUEUE, concurrency: 1, ...(ttlHours > 0 diff --git a/packages/backend/src/azuredevops.test.ts b/packages/backend/src/azuredevops.test.ts new file mode 100644 index 000000000..234eaa55c --- /dev/null +++ b/packages/backend/src/azuredevops.test.ts @@ -0,0 +1,137 @@ +import type { AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/azuredevops.type'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + getProjects: vi.fn(), + getRepositories: vi.fn(), + getRepository: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...await importOriginal(), + getTokenFromConfig: vi.fn(async () => "token"), +})); + +vi.mock("azure-devops-node-api", () => ({ + getPersonalAccessTokenHandler: vi.fn(() => ({})), + WebApi: class { + getCoreApi = vi.fn(async () => ({ + getProjects: mocks.getProjects, + })); + getGitApi = vi.fn(async () => ({ + getRepositories: mocks.getRepositories, + getRepository: mocks.getRepository, + })); + }, +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { getAzureDevOpsReposFromConfig } from './azuredevops'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; + +const config = (overrides: Partial): AzureDevOpsConnectionConfig => ({ + type: "azuredevops", + deploymentType: "cloud", + token: { env: "AZURE_DEVOPS_TOKEN" }, + ...overrides, +}); + +beforeEach(() => { + vi.clearAllMocks(); + const notFound = Object.assign(new Error("Not Found"), { statusCode: 404 }); + mocks.getProjects.mockRejectedValue(notFound); + mocks.getRepositories.mockRejectedValue(notFound); + mocks.getRepository.mockRejectedValue(notFound); +}); + +describe("Azure DevOps repository discovery", () => { + test("reports inaccessible configured targets as partial successes", async () => { + const result = await collectRepositoryDiscoveryIssues(() => + getAzureDevOpsReposFromConfig(config({ + orgs: ["missing-org"], + projects: ["org/missing-project"], + repos: ["org/project/missing-repo"], + })) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: "missing-org", + }, + message: "Azure DevOps organization was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: "org/missing-project", + }, + message: "Azure DevOps project was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "org/project/missing-repo", + }, + message: "Azure DevOps repository was not found or is inaccessible.", + }, + ], + }); + }); + + test("reports incomplete project enumeration within an organization", async () => { + mocks.getProjects.mockResolvedValue([ + { name: "missing-id" }, + { id: "broken-project-id", name: "broken-project" }, + ]); + mocks.getRepositories.mockRejectedValue(new Error("Service unavailable")); + + const result = await collectRepositoryDiscoveryIssues(() => + getAzureDevOpsReposFromConfig(config({ orgs: ["my-org"] })) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: "my-org/missing-id", + }, + message: "Azure DevOps returned a project without an ID, so its repositories were skipped.", + }, + { + code: "ENUMERATION_FAILED", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: "my-org/broken-project", + }, + message: "Azure DevOps repository enumeration did not complete for this project.", + }, + ], + }); + }); +}); diff --git a/packages/backend/src/azuredevops.ts b/packages/backend/src/azuredevops.ts index 23b8edd62..864a80d86 100644 --- a/packages/backend/src/azuredevops.ts +++ b/packages/backend/src/azuredevops.ts @@ -7,6 +7,7 @@ import * as Sentry from "@sentry/node"; import * as azdev from "azure-devops-node-api"; import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js"; import { getTokenFromConfig } from "@sourcebot/shared"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; const logger = createLogger('azuredevops'); const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com"; @@ -42,39 +43,32 @@ export const getAzureDevOpsReposFromConfig = async ( const useTfsPath = config.useTfsPath || false; let allRepos: GitRepository[] = []; - let allWarnings: string[] = []; if (config.orgs) { - const { repos, warnings } = await getReposForOrganizations( + allRepos = allRepos.concat(await getReposForOrganizations( config.orgs, baseUrl, token, useTfsPath - ); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + )); } if (config.projects) { - const { repos, warnings } = await getReposForProjects( + allRepos = allRepos.concat(await getReposForProjects( config.projects, baseUrl, token, useTfsPath - ); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + )); } if (config.repos) { - const { repos, warnings } = await getRepos( + allRepos = allRepos.concat(await getRepos( config.repos, baseUrl, token, useTfsPath - ); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + )); } let repos = allRepos @@ -89,10 +83,7 @@ export const getAzureDevOpsReposFromConfig = async ( logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; }; export const shouldExcludeRepo = ({ @@ -180,6 +171,15 @@ async function getReposForOrganizations( for (const project of projects) { if (!project.id) { logger.warn(`Encountered project in org ${org} with no id: ${project.name}`); + reportRepositoryDiscoveryIssue({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: `${org}/${project.name ?? "unknown"}`, + }, + message: "Azure DevOps returned a project without an ID, so its repositories were skipped.", + }); continue; } @@ -188,6 +188,15 @@ async function getReposForOrganizations( allRepos.push(...repos); } catch (error) { logger.warn(`Failed to fetch repositories for project ${project.name}: ${error}`); + reportRepositoryDiscoveryIssue({ + code: "ENUMERATION_FAILED", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: `${org}/${project.name ?? project.id}`, + }, + message: "Azure DevOps repository enumeration did not complete for this project.", + }); } } @@ -198,10 +207,7 @@ async function getReposForOrganizations( }); logger.debug(`Found ${data.length} repositories in organization ${org} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for organization ${org}.`, error); @@ -210,22 +216,23 @@ async function getReposForOrganizations( if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { const warning = `Organization ${org} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: org, + }, + message: "Azure DevOps organization was not found or is inaccessible.", + }); + return []; } throw error; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } async function getReposForProjects( @@ -253,10 +260,7 @@ async function getReposForProjects( }); logger.debug(`Found ${data.length} repositories in project ${project} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for project ${project}.`, error); @@ -264,22 +268,23 @@ async function getReposForProjects( if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { const warning = `Project ${project} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Azure DevOps project was not found or is inaccessible.", + }); + return []; } throw error; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } async function getRepos( @@ -307,10 +312,7 @@ async function getRepos( }); logger.debug(`Found info for repository ${repo} in ${durationMs}ms`); - return { - type: 'valid' as const, - data: [result] - }; + return [result]; } catch (error) { Sentry.captureException(error); @@ -319,20 +321,21 @@ async function getRepos( if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { const warning = `Repository ${repo} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Azure DevOps repository was not found or is inaccessible.", + }); + return []; } throw error; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; -} \ No newline at end of file + return processPromiseResults(results); +} diff --git a/packages/backend/src/bitbucket.test.ts b/packages/backend/src/bitbucket.test.ts index 5d2fb55e4..2177e7d1b 100644 --- a/packages/backend/src/bitbucket.test.ts +++ b/packages/backend/src/bitbucket.test.ts @@ -1,8 +1,163 @@ -import { expect, test, describe } from 'vitest'; -import { cloudShouldExcludeRepo, serverShouldExcludeRepo, BitbucketRepository } from './bitbucket'; -import { BitbucketConnectionConfig } from '@sourcebot/schemas/v3/bitbucket.type'; -import { SchemaRepository as CloudRepository } from '@coderabbitai/bitbucket/cloud/openapi'; -import { SchemaRestRepository as ServerRepository } from '@coderabbitai/bitbucket/server/openapi'; +import { beforeEach, expect, test, describe, vi } from 'vitest'; +import type { BitbucketConnectionConfig } from '@sourcebot/schemas/v3/bitbucket.type'; +import type { SchemaRepository as CloudRepository } from '@coderabbitai/bitbucket/cloud/openapi'; +import type { SchemaRestRepository as ServerRepository } from '@coderabbitai/bitbucket/server/openapi'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { status: 404 }); + + return { + cloudGet: vi.fn(async () => { + throw notFound; + }), + serverGet: vi.fn(async () => { + throw notFound; + }), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@coderabbitai/bitbucket/cloud", () => ({ + createBitbucketCloudClient: () => ({ + use: vi.fn(), + GET: mocks.cloudGet, + }), +})); + +vi.mock("@coderabbitai/bitbucket/server", () => ({ + createBitbucketServerClient: () => ({ + use: vi.fn(), + GET: mocks.serverGet, + }), +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; +import { + cloudShouldExcludeRepo, + getBitbucketReposFromConfig, + serverShouldExcludeRepo, + type BitbucketRepository, +} from './bitbucket'; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Bitbucket repository discovery", () => { + test("reports Bitbucket Cloud discovery gaps", async () => { + const config = { + type: "bitbucket", + deploymentType: "cloud", + all: true, + workspaces: ["missing-workspace"], + projects: ["workspace/missing-project", "invalid-project"], + repos: ["workspace/missing-repo", "invalid-repo"], + exclude: { archived: true }, + } satisfies BitbucketConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getBitbucketReposFromConfig(config) + ); + + expect(result.value).toEqual([]); + expect(result.issues).toHaveLength(7); + expect(result.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { kind: "configuration", value: "all" }, + }), + expect.objectContaining({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "exclude.archived", + }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { kind: "workspace", value: "missing-workspace" }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { + kind: "project", + value: "workspace/missing-project", + }, + }), + expect.objectContaining({ + code: "INVALID_TARGET", + subject: { kind: "project", value: "invalid-project" }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { + kind: "repository", + value: "workspace/missing-repo", + }, + }), + expect.objectContaining({ + code: "INVALID_TARGET", + subject: { kind: "repository", value: "invalid-repo" }, + }), + ])); + }); + + test("reports Bitbucket Server discovery gaps", async () => { + const config = { + type: "bitbucket", + deploymentType: "server", + url: "https://bitbucket.example.com", + workspaces: ["unsupported-workspace"], + projects: ["missing-project"], + repos: ["PROJ/missing-repo", "invalid-repo"], + } satisfies BitbucketConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getBitbucketReposFromConfig(config) + ); + + expect(result.value).toEqual([]); + expect(result.issues).toHaveLength(4); + expect(result.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "workspace", + value: "unsupported-workspace", + }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { kind: "project", value: "missing-project" }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { + kind: "repository", + value: "PROJ/missing-repo", + }, + }), + expect.objectContaining({ + code: "INVALID_TARGET", + subject: { kind: "repository", value: "invalid-repo" }, + }), + ])); + }); +}); const makeCloudRepo = (overrides: Partial = {}): BitbucketRepository => ({ type: 'repository', diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 84f85d023..525eb336d 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -11,9 +11,9 @@ import { SchemaRepositoryUserPermission as CloudRepositoryUserPermission, } from "@coderabbitai/bitbucket/cloud/openapi"; import { SchemaRestRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi"; -import { processPromiseResults } from "./connectionUtils.js"; -import { throwIfAnyFailed } from "./connectionUtils.js"; +import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import { getTokenFromConfig } from "@sourcebot/shared"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; const logger = createLogger('bitbucket'); const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org'; @@ -29,9 +29,9 @@ interface BitbucketClient { apiClient: any; baseUrl: string; gitUrl: string; - getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; - getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; - getRepos: (client: BitbucketClient, repos: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; + getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise; + getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise; + getRepos: (client: BitbucketClient, repos: string[]) => Promise; shouldExcludeRepo: (repo: BitbucketRepository, config: BitbucketConnectionConfig) => boolean; } @@ -91,46 +91,63 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon createBitbucketCloudClient(config.user, token); let allRepos: BitbucketRepository[] = []; - let allWarnings: string[] = []; if (config.all === true) { if (client.deploymentType === BITBUCKET_SERVER) { - const { repos, warnings } = await serverGetAllRepos(client); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await serverGetAllRepos(client)); } else { const warning = `Ignoring option all:true in config: not supported for Bitbucket Cloud`; logger.warn(warning); - allWarnings = allWarnings.concat(warning); + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "all", + }, + message: "The all option is not supported for Bitbucket Cloud.", + }); } } if (config.workspaces) { - const { repos, warnings } = await client.getReposForWorkspace(client, config.workspaces); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat( + await client.getReposForWorkspace(client, config.workspaces), + ); } if (config.projects) { - const { repos, warnings } = await client.getReposForProjects(client, config.projects); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat( + await client.getReposForProjects(client, config.projects), + ); } if (config.repos) { - const { repos, warnings } = await client.getRepos(client, config.repos); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await client.getRepos(client, config.repos)); + } + + if ( + client.deploymentType === BITBUCKET_CLOUD + && config.exclude?.archived + ) { + const warning = "Bitbucket Cloud does not support filtering archived repositories. Ignoring exclude.archived."; + logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "exclude.archived", + }, + message: "The exclude.archived option is not supported for Bitbucket Cloud.", + }); } const filteredRepos = allRepos.filter((repo) => { return !client.shouldExcludeRepo(repo, config); }); - return { - repos: filteredRepos, - warnings: allWarnings, - }; + return filteredRepos; } export function createBitbucketCloudClient(user: string | undefined, token: string | undefined): BitbucketClient { @@ -210,7 +227,7 @@ function parseUrl(url: string): { path: string; query: Record; } } -async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { +async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise { const results = await Promise.allSettled(workspaces.map(async (workspace) => { try { logger.debug(`Fetching all repos for workspace ${workspace}...`); @@ -231,10 +248,7 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st }); logger.debug(`Found ${data.length} repos for workspace ${workspace} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: data, - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to get repos for workspace ${workspace}: ${e}`); @@ -242,33 +256,41 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st if (e?.status === 404) { const warning = `Workspace ${workspace} not found or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - } + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "workspace", + value: workspace, + }, + message: "Bitbucket workspace was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings, - }; + return processPromiseResults(results); } -async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { +async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise { const results = await Promise.allSettled(projects.map(async (project) => { const [workspace, project_name] = project.split('/'); if (!workspace || !project_name) { const warning = `Invalid project ${project}`; logger.warn(warning); - return { - type: 'warning' as const, - warning - } + reportRepositoryDiscoveryIssue({ + code: "INVALID_TARGET", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Bitbucket Cloud projects must use the workspace/project format.", + }); + return []; } logger.debug(`Fetching all repos for project ${project} for workspace ${workspace}...`); @@ -292,10 +314,7 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin }); logger.debug(`Found ${repos.length} repos for project ${project_name} for workspace ${workspace} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: repos - } + return repos; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch repos for project ${project_name}: ${e}`); @@ -303,33 +322,41 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin if (e?.status === 404) { const warning = `Project ${project_name} not found in ${workspace} or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - } + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Bitbucket Cloud project was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - } + return processPromiseResults(results); } -async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { +async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise { const results = await Promise.allSettled(repoList.map(async (repo) => { const [workspace, repo_slug] = repo.split('/'); if (!workspace || !repo_slug) { const warning = `Invalid repo ${repo}`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "INVALID_TARGET", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Cloud repositories must use the workspace/repository format.", + }); + return []; } logger.debug(`Fetching repo ${repo_slug} for workspace ${workspace}...`); @@ -339,10 +366,7 @@ async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promi const { data } = await client.apiClient.GET(path); return data; }, `repo ${repo}`, logger); - return { - type: 'valid' as const, - data: [data] - }; + return [data]; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch repo ${repo}: ${e}`); @@ -350,21 +374,23 @@ async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promi if (e?.status === 404) { const warning = `Repo ${repo} not found in ${workspace} or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Cloud repository was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - }; + return processPromiseResults(results); } export function cloudShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean { @@ -381,10 +407,6 @@ export function cloudShouldExcludeRepo(repo: BitbucketRepository, config: Bitbuc } } - if (!!config.exclude?.archived) { - logger.warn(`Exclude archived repos flag provided in config but Bitbucket Cloud does not support archived repos. Ignoring...`); - } - if (!!config.exclude?.forks && cloudRepo.parent !== undefined) { reason = `\`exclude.forks\` is true`; return true; @@ -464,16 +486,23 @@ const getPaginatedServer = async ( return results; } -async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { - const warnings = workspaces.map(workspace => `Workspaces are not supported in Bitbucket Server: ${workspace}`); +async function serverGetReposForWorkspace(_client: BitbucketClient, workspaces: string[]): Promise { logger.debug('Workspaces are not supported in Bitbucket Server'); - return { - repos: [], - warnings - }; + for (const workspace of workspaces) { + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "workspace", + value: workspace, + }, + message: "Workspaces are not supported for Bitbucket Server.", + }); + } + return []; } -async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { +async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise { const results = await Promise.allSettled(projects.map(async (project) => { try { logger.debug(`Fetching all repos for project ${project}...`); @@ -495,10 +524,7 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri }); logger.debug(`Found ${data.length} repos for project ${project} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: data, - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to get repos for project ${project}: ${e}`); @@ -506,33 +532,41 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri if (e?.status === 404) { const warning = `Project ${project} not found or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Bitbucket Server project was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - }; + return processPromiseResults(results); } -async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { +async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise { const results = await Promise.allSettled(repoList.map(async (repo) => { const [project, repo_slug] = repo.split('/'); if (!project || !repo_slug) { const warning = `Invalid repo ${repo}`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "INVALID_TARGET", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Server repositories must use the project/repository format.", + }); + return []; } logger.debug(`Fetching repo ${repo_slug} for project ${project}...`); @@ -542,10 +576,7 @@ async function serverGetRepos(client: BitbucketClient, repoList: string[]): Prom const { data } = await client.apiClient.GET(path); return data; }, `repo ${repo}`, logger); - return { - type: 'valid' as const, - data: [data] - }; + return [data]; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch repo ${repo}: ${e}`); @@ -553,24 +584,26 @@ async function serverGetRepos(client: BitbucketClient, repoList: string[]): Prom if (e?.status === 404) { const warning = `Repo ${repo} not found in project ${project} or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Server repository was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - }; + return processPromiseResults(results); } -async function serverGetAllRepos(client: BitbucketClient): Promise<{repos: ServerRepository[], warnings: string[]}> { +async function serverGetAllRepos(client: BitbucketClient): Promise { logger.debug(`Fetching all repos from Bitbucket Server...`); const path = `/rest/api/1.0/repos` as ServerGetRequestPath; const { durationMs, data } = await measure(async () => { @@ -583,7 +616,7 @@ async function serverGetAllRepos(client: BitbucketClient): Promise<{repos: Serve return fetchWithRetry(fetchFn, `all repos`, logger); }); logger.debug(`Found ${data.length} total repos in ${durationMs}ms.`); - return { repos: data, warnings: [] }; + return data; } export function serverShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean { diff --git a/packages/backend/src/configManager.test.ts b/packages/backend/src/configManager.test.ts index 9f28771b9..2d625ced0 100644 --- a/packages/backend/src/configManager.test.ts +++ b/packages/backend/src/configManager.test.ts @@ -217,13 +217,13 @@ describe("ConfigManager", () => { "repo-index-v1-2", ); expect(mocks.trigger).toHaveBeenCalledWith( - "repo-index", - { repoId: 1, type: "CLEANUP" }, + "repo-cleanup", + { repoId: 1 }, { priority: 10 }, ); expect(mocks.trigger).not.toHaveBeenCalledWith( - "repo-index", - { repoId: 2, type: "CLEANUP" }, + "repo-cleanup", + { repoId: 2 }, expect.anything(), ); expect(mocks.removeJobScheduler).toHaveBeenCalledWith( diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index bf6d984d8..a3326ca8b 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -12,7 +12,7 @@ import { reconcileRepoIndexWork, reconcileRepoPermissionSyncWork, replaceConnectionRepositories, -} from "./connectionWorkload.js"; +} from "./connectionSyncWorkload.js"; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts similarity index 77% rename from packages/backend/src/connectionWorkload.test.ts rename to packages/backend/src/connectionSyncWorkload.test.ts index 3e5f8c6f6..3b78b780a 100644 --- a/packages/backend/src/connectionWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -6,8 +6,7 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), - connectionSyncJobUpsert: vi.fn(), - connectionSyncJobUpdate: vi.fn(), + connectionUpdateMany: vi.fn(), compileGithubConfig: vi.fn(), loadConfig: vi.fn(), syncSearchContexts: vi.fn(), @@ -37,8 +36,9 @@ vi.mock("./entitlements.js", () => ({ vi.mock("@sourcebot/shared", () => ({ CONNECTION_QUEUE: { name: "connection-sync", - dedupKey: ({ connectionId }: { connectionId: number }) => - `connection:${connectionId}`, + deduplication: ({ connectionId }: { connectionId: number }) => ({ + id: `connection:${connectionId}`, + }), jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, @@ -71,6 +71,9 @@ vi.mock("@sourcebot/shared", () => ({ ], createLogger: vi.fn(() => mocks.logger), loadConfig: mocks.loadConfig, + repositoryDiscoveryIssueSchema: { + parse: (issue: unknown) => issue, + }, })); vi.mock("./repoCompileUtils.js", () => ({ @@ -88,34 +91,19 @@ vi.mock("./ee/syncSearchContexts.js", () => ({ })); import { - createConnectionWorkload, + createConnectionSyncWorkload as createConnectionWorkload, replaceConnectionRepositories, reconcileRepoIndexWork, reconcileRepoPermissionSyncWork, -} from "./connectionWorkload.js"; +} from "./connectionSyncWorkload.js"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; -const transactionClient = { - connection: { - update: mocks.connectionUpdate, - }, - connectionSyncJob: { - upsert: mocks.connectionSyncJobUpsert, - }, -}; -const transaction = vi.fn( - (callback: (tx: typeof transactionClient) => Promise) => - callback(transactionClient), -); - const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, - }, - connectionSyncJob: { - upsert: mocks.connectionSyncJobUpsert, - update: mocks.connectionSyncJobUpdate, + updateMany: mocks.connectionUpdateMany, }, repo: { findMany: mocks.repoFindMany, @@ -124,7 +112,6 @@ const db = { repoToConnection: { deleteMany: mocks.repoToConnectionDeleteMany, }, - $transaction: transaction, } as unknown as PrismaClient; const jobManager = { @@ -159,6 +146,7 @@ describe("connectionWorkload", () => { vi.restoreAllMocks(); vi.clearAllMocks(); mocks.connectionUpdate.mockResolvedValue({}); + mocks.connectionUpdateMany.mockResolvedValue({ count: 1 }); mocks.repoFindMany.mockResolvedValue([]); mocks.getJobSchedulerIds.mockResolvedValue([]); mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); @@ -168,7 +156,7 @@ describe("connectionWorkload", () => { mocks.syncSearchContexts.mockResolvedValue(undefined); }); - test("declares database-backed lifecycle hooks", () => { + test("declares the connection lifecycle hooks", () => { expect(connectionWorkload.onStarted).toBeTypeOf("function"); expect(connectionWorkload.onCompleted).toBeTypeOf("function"); expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); @@ -200,26 +188,9 @@ describe("connectionWorkload", () => { expect(mocks.connectionFindUniqueOrThrow).not.toHaveBeenCalled(); }); - test("marks the connection sync job as in progress when started", async () => { + test("records the latest connection sync job ID when started", async () => { await connectionWorkload.onStarted?.(lifecycleContext); - expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - warningMessages: [], - }, - create: { - id: "job-1", - connectionId: 42, - status: "IN_PROGRESS", - warningMessages: [], - }, - }); expect(mocks.connectionUpdate).toHaveBeenCalledWith({ where: { id: 42, @@ -228,41 +199,37 @@ describe("connectionWorkload", () => { latestSyncJobId: "job-1", }, }); - expect(transaction).toHaveBeenCalledOnce(); }); - test("marks the connection sync job as completed", async () => { + test("records the first successful sync job terminal state", async () => { await connectionWorkload.onCompleted?.(lifecycleContext, { - reposToCleanup: [], - reposToIndex: [], + outcome: "SUCCESS", }); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + expect(mocks.connectionUpdateMany).toHaveBeenCalledWith({ where: { - id: "job-1", + id: 42, + firstSyncJobFinishedAt: null, }, data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, + firstSyncJobFinishedAt: expect.any(Date), }, }); }); - test("marks the connection sync job as failed after terminal failure", async () => { + test("records the first failed sync job terminal state", async () => { await connectionWorkload.onTerminalFailure?.( lifecycleContext, new Error("Connection credentials expired"), ); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ + expect(mocks.connectionUpdateMany).toHaveBeenCalledWith({ where: { - id: "job-1", + id: 42, + firstSyncJobFinishedAt: null, }, data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Connection credentials expired", + firstSyncJobFinishedAt: expect.any(Date), }, }); }); @@ -281,10 +248,7 @@ describe("connectionWorkload", () => { orgId: 7, config, }); - mocks.compileGithubConfig.mockResolvedValue({ - repoData: [discoveredRepo], - warnings: ["Repository was archived"], - }); + mocks.compileGithubConfig.mockResolvedValue([discoveredRepo]); mocks.repoUpsert.mockResolvedValue({ id: 4, name: "github.com/sourcebot/repo-4", @@ -305,23 +269,18 @@ describe("connectionWorkload", () => { 42, expect.any(AbortSignal), ); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { id: "job-1" }, - data: { warningMessages: ["Repository was archived"] }, - }); expect(mocks.repoUpsert).toHaveBeenCalledOnce(); expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( "repo-index", "repo-index-v1-4", 3_600_000, - { repoId: 4, type: "INDEX" }, + { repoId: 4 }, { priority: 10 }, ); expect(trigger).toHaveBeenCalledWith( "repo-index", { repoId: 4, - type: "INDEX", }, { priority: 5 }, ); @@ -333,31 +292,138 @@ describe("connectionWorkload", () => { orgId: 7, contexts: undefined, }); - expect(result).toEqual({ - reposToCleanup: [], - reposToIndex: [ - { id: 4, name: "github.com/sourcebot/repo-4" }, - ], - }); + expect(result).toEqual({ outcome: "SUCCESS" }); expect(updateProgress).not.toHaveBeenCalled(); }); - test("does not mark the connection synced when repo work reconciliation fails", async () => { + test("returns partial success reasons reported during discovery", async () => { + const reason = { + code: "NOT_FOUND_OR_INACCESSIBLE" as const, + effect: "TARGET_SKIPPED" as const, + subject: { + kind: "repository" as const, + value: "sourcebot-dev/legacy", + }, + message: "Repository was not found or is inaccessible.", + }; + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config: { type: "github" }, + }); + mocks.compileGithubConfig.mockImplementation(async () => { + reportRepositoryDiscoveryIssue(reason); + return []; + }); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }), + ).resolves.toEqual({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }); + }); + + test("preserves missing repositories when discovery is incomplete", async () => { + const reason = { + code: "ENUMERATION_FAILED" as const, + effect: "DISCOVERY_INCOMPLETE" as const, + subject: { + kind: "organization" as const, + value: "sourcebot-dev", + }, + message: "Repository enumeration did not complete.", + }; + const indexedAt = new Date("2026-07-30T12:00:00.000Z"); mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, name: "github", orgId: 7, config: { type: "github" }, }); - mocks.compileGithubConfig.mockResolvedValue({ - repoData: [ + mocks.compileGithubConfig.mockImplementation(async () => { + reportRepositoryDiscoveryIssue(reason); + return [ { - external_id: "repo-4", + external_id: "repo-1", external_codeHostUrl: "https://github.com", }, - ], - warnings: [], + ]; }); + mocks.repoFindMany + .mockResolvedValueOnce([ + { + id: 2, + name: "github.com/sourcebot/repo-2", + indexedAt, + }, + ]) + .mockResolvedValueOnce([]); + mocks.repoUpsert.mockResolvedValue({ + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt, + }); + const trigger = vi.fn(); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + }), + ).resolves.toEqual({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }); + + expect(mocks.repoToConnectionDeleteMany).not.toHaveBeenCalled(); + expect(mocks.logger.debug).toHaveBeenCalledWith( + "Preserving repositories omitted from a DISCOVERY_INCOMPLETE result", + { + connectionId: 42, + repositories: [ + { + id: 2, + name: "github.com/sourcebot/repo-2", + }, + ], + }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-2", + 3_600_000, + { repoId: 2 }, + { priority: 10 }, + ); + expect(trigger).not.toHaveBeenCalledWith( + "repo-cleanup", + { repoId: 2 }, + expect.anything(), + ); + }); + + test("does not mark the connection synced when repo work reconciliation fails", async () => { + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config: { type: "github" }, + }); + mocks.compileGithubConfig.mockResolvedValue([ + { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + }, + ]); mocks.repoUpsert.mockResolvedValue({ id: 4, name: "github.com/sourcebot/repo-4", @@ -517,7 +583,7 @@ describe("connectionWorkload repo sync helpers", () => { "repo-index", "repo-index-v1-1", 3_600_000, - { repoId: 1, type: "INDEX" }, + { repoId: 1 }, { priority: 10 }, ); expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( @@ -525,7 +591,7 @@ describe("connectionWorkload repo sync helpers", () => { "repo-index", "repo-index-v1-4", 3_600_000, - { repoId: 4, type: "INDEX" }, + { repoId: 4 }, { priority: 10 }, ); expect(mocks.removeJobScheduler).toHaveBeenCalledWith( @@ -534,10 +600,9 @@ describe("connectionWorkload repo sync helpers", () => { ); expect(trigger).toHaveBeenNthCalledWith( 1, - "repo-index", + "repo-cleanup", { repoId: 2, - type: "CLEANUP", }, { priority: 10 }, ); @@ -546,7 +611,6 @@ describe("connectionWorkload repo sync helpers", () => { "repo-index", { repoId: 4, - type: "INDEX", }, { priority: 5 }, ); diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts similarity index 82% rename from packages/backend/src/connectionWorkload.ts rename to packages/backend/src/connectionSyncWorkload.ts index 2ed1fe944..3eb896b2e 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -1,5 +1,5 @@ import * as Sentry from "@sentry/node"; -import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; +import { PrismaClient } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { CONNECTION_QUEUE, @@ -21,6 +21,7 @@ import { compileGitlabConfig, } from "./repoCompileUtils.js"; import type { RepoData } from "./repoCompileUtils.js"; +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; import { JobManager, ProcessContext, Settings, Workload } from "./types.js"; const CONNECTION_SYNC_LOCK_DURATION_MS = 60_000; @@ -32,16 +33,11 @@ interface Props { settings: Settings; } -interface ConnectionSyncResult { - reposToCleanup: { id: number; name: string }[]; - reposToIndex: { id: number; name: string }[]; -} - -export const createConnectionWorkload = ({ +export const createConnectionSyncWorkload = ({ db, jobManager, settings, -}: Props): Workload<"connection-sync", ConnectionSyncResult> => ({ +}: Props): Workload<"connection-sync"> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, executionLock: { @@ -52,7 +48,6 @@ export const createConnectionWorkload = ({ process: async ({ data: { connectionId }, signal, - jobId, trigger, }) => { signal.throwIfAborted(); @@ -69,33 +64,31 @@ export const createConnectionWorkload = ({ orgId, }); - const { repoData, warnings } = await discoverConnectionRepositories({ - config: connection.config as unknown as ConnectionConfig, - connectionId, - signal, - }); - - signal.throwIfAborted(); - await db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, - }); + const { value: repoData, issues } = + await collectRepositoryDiscoveryIssues(() => + discoverConnectionRepositories({ + config: connection.config as unknown as ConnectionConfig, + connectionId, + signal, + }) + ); logger.debug(`Discovered ${repoData.length} repositories`, { connectionId, repositoryCount: repoData.length, }); + const discoveryIsComplete = !issues.some( + ({ effect }) => effect === "DISCOVERY_INCOMPLETE", + ); + signal.throwIfAborted(); const repoChanges = await replaceConnectionRepositories({ db, connectionId, orgId, discoveredRepos: repoData, + removeMissing: discoveryIsComplete, }); signal.throwIfAborted(); @@ -159,66 +152,43 @@ export const createConnectionWorkload = ({ connectionId, }); - return { - reposToCleanup: repoChanges.orphanedRepos, - reposToIndex: repoChanges.unindexedRepos, - }; + return issues.length === 0 + ? { outcome: "SUCCESS" } + : { outcome: "PARTIAL_SUCCESS", reasons: issues }; }, onStarted: async ({ data: { connectionId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.connectionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - warningMessages: [], - }, - create: { - id: jobId, - connectionId, - status: ConnectionSyncJobStatus.IN_PROGRESS, - warningMessages: [], - }, - }); - await tx.connection.update({ - where: { - id: connectionId, - }, - data: { - latestSyncJobId: jobId, - }, - }); - }); - }, - onCompleted: async ({ jobId }) => { - await db.connectionSyncJob.update({ + await db.connection.update({ where: { - id: jobId, + id: connectionId, }, data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, + latestSyncJobId: jobId, }, }); }, - onTerminalFailure: async ({ jobId }, error) => { - await db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); + onCompleted: async ({ data: { connectionId } }) => { + await markFirstSyncJobFinished(db, connectionId); + }, + onTerminalFailure: async ({ data: { connectionId } }) => { + await markFirstSyncJobFinished(db, connectionId); }, }); +const markFirstSyncJobFinished = async ( + db: PrismaClient, + connectionId: number, +) => { + await db.connection.updateMany({ + where: { + id: connectionId, + firstSyncJobFinishedAt: null, + }, + data: { + firstSyncJobFinishedAt: new Date(), + }, + }); +}; + export interface CurrentRepo { id: number; name: string; @@ -251,11 +221,13 @@ export const replaceConnectionRepositories = async ({ connectionId, orgId, discoveredRepos, + removeMissing = true, }: { db: PrismaClient; connectionId: number; orgId: number; discoveredRepos: RepoData[]; + removeMissing?: boolean; }): Promise => { const previouslyAssociatedRepos = await db.repo.findMany({ where: { @@ -267,12 +239,14 @@ export const replaceConnectionRepositories = async ({ }, select: { id: true, + name: true, + indexedAt: true, }, }); - const currentRepos: CurrentRepo[] = []; + const discoveredCurrentRepos: CurrentRepo[] = []; for (const repo of deduplicateRepos(discoveredRepos)) { - currentRepos.push( + discoveredCurrentRepos.push( await db.repo.upsert({ where: { external_id_external_codeHostUrl_orgId: { @@ -302,10 +276,31 @@ export const replaceConnectionRepositories = async ({ ); } + const discoveredRepoIds = new Set( + discoveredCurrentRepos.map(({ id }) => id), + ); + const missingRepos = previouslyAssociatedRepos.filter( + ({ id }) => !discoveredRepoIds.has(id), + ); + if (!removeMissing && missingRepos.length > 0) { + logger.debug( + "Preserving repositories omitted from a DISCOVERY_INCOMPLETE result", + { + connectionId, + repositories: missingRepos.map(({ id, name }) => ({ + id, + name, + })), + }, + ); + } + const currentRepos = removeMissing + ? discoveredCurrentRepos + : [...discoveredCurrentRepos, ...missingRepos]; const currentRepoIds = new Set(currentRepos.map(({ id }) => id)); - const staleRepoIds = previouslyAssociatedRepos - .map(({ id }) => id) - .filter((id) => !currentRepoIds.has(id)); + const staleRepoIds = removeMissing + ? missingRepos.map(({ id }) => id) + : []; if (staleRepoIds.length > 0) { await db.repoToConnection.deleteMany({ @@ -367,7 +362,7 @@ export const reconcileRepoIndexWork = async ({ "repo-index", `repo-index-v1-${id}`, intervalMs, - { repoId: id, type: "INDEX" }, + { repoId: id }, { priority: JOB_PRIORITIES.SCHEDULED }, ), ), @@ -385,10 +380,9 @@ export const reconcileRepoIndexWork = async ({ await Promise.all( orphanedRepos.map(({ id }) => trigger( - "repo-index", + "repo-cleanup", { repoId: id, - type: "CLEANUP", }, { priority: JOB_PRIORITIES.SCHEDULED }, ), @@ -401,7 +395,6 @@ export const reconcileRepoIndexWork = async ({ "repo-index", { repoId: id, - type: "INDEX", }, { priority: JOB_PRIORITIES.INITIAL }, ), diff --git a/packages/backend/src/connectionUtils.ts b/packages/backend/src/connectionUtils.ts index 074dfe7fe..69c1bd0eb 100644 --- a/packages/backend/src/connectionUtils.ts +++ b/packages/backend/src/connectionUtils.ts @@ -1,41 +1,11 @@ import * as Sentry from "@sentry/node"; -type ValidResult = { - type: 'valid'; - data: T[]; -}; - -type WarningResult = { - type: 'warning'; - warning: string; -}; - -type CustomResult = ValidResult | WarningResult; - export function processPromiseResults( - results: PromiseSettledResult>[], -): { - validItems: T[]; - warnings: string[]; -} { - const validItems: T[] = []; - const warnings: string[] = []; - - results.forEach(result => { - if (result.status === 'fulfilled') { - const value = result.value; - if (value.type === 'valid') { - validItems.push(...value.data); - } else { - warnings.push(value.warning); - } - } - }); - - return { - validItems, - warnings, - }; + results: PromiseSettledResult[], +): T[] { + return results.flatMap((result) => + result.status === "fulfilled" ? result.value : [] + ); } export function throwIfAnyFailed(results: PromiseSettledResult[]) { @@ -44,4 +14,4 @@ export function throwIfAnyFailed(results: PromiseSettledResult[]) { Sentry.captureException(failedResult.reason); throw failedResult.reason; } -} \ No newline at end of file +} diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts index befdb21af..5fcf9691d 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -102,18 +102,12 @@ const accountUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const repoFindMany = vi.fn().mockResolvedValue([]); const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); -const permissionSyncJobUpsert = vi.fn(); -const permissionSyncJobUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const transactionClient = { account: { findUnique: accountFindUnique, update: accountUpdate, updateMany: accountUpdateMany, }, - accountPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, }; const transaction = vi.fn( ( @@ -140,10 +134,6 @@ const db = { repo: { findMany: repoFindMany, }, - accountPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, $transaction: transaction, } as unknown as PrismaClient; @@ -189,7 +179,6 @@ beforeEach(() => { repoFindMany.mockResolvedValue([]); permissionCreateMany.mockResolvedValue({ count: 0 }); permissionDeleteMany.mockResolvedValue({ count: 95 }); - permissionSyncJobUpdateMany.mockResolvedValue({ count: 1 }); accountUpdateMany.mockResolvedValue({ count: 1 }); }); @@ -406,40 +395,19 @@ describe("accountPermissionSyncWorkload", () => { expect(transaction).not.toHaveBeenCalled(); }); - test("marks a job as in progress when started", async () => { + test("records the latest job when started", async () => { await createWorkload().onStarted?.(lifecycleContext); - expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ - where: { id: "job_1" }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - }, - create: { - id: "job_1", - accountId: "account_1", - status: "IN_PROGRESS", - }, - }); expect(accountUpdate).toHaveBeenCalledWith({ where: { id: "account_1" }, data: { latestPermissionSyncJobId: "job_1" }, }); - expect(transaction).toHaveBeenCalledOnce(); + expect(transaction).not.toHaveBeenCalled(); }); - test("marks a job completed and clears the account issue when it is still latest", async () => { + test("clears the account issue when the completed job is still latest", async () => { await createWorkload().onCompleted?.(lifecycleContext, undefined); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ - where: { id: "job_1" }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); expect(accountUpdateMany).toHaveBeenCalledWith({ where: { id: "account_1", @@ -458,21 +426,11 @@ describe("accountPermissionSyncWorkload", () => { expect(transaction).toHaveBeenCalledOnce(); }); - test("marks a job failed after terminal failure", async () => { + test("reports terminal failures", async () => { const error = new Error("Upstream unavailable"); await createWorkload().onTerminalFailure?.(lifecycleContext, error); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: "job_1" }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Upstream unavailable", - }, - }), - ); expect(mocks.captureException).toHaveBeenCalledWith(error, { tags: { jobId: "job_1", @@ -485,8 +443,7 @@ describe("accountPermissionSyncWorkload", () => { }); }); - test("completion tolerates a cascaded job and account row", async () => { - permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); + test("completion tolerates a deleted account", async () => { accountUpdateMany.mockResolvedValue({ count: 0 }); accountFindUnique.mockResolvedValue(null); @@ -495,8 +452,7 @@ describe("accountPermissionSyncWorkload", () => { ).resolves.toBeUndefined(); }); - test("terminal failure tolerates a cascaded job and account row", async () => { - permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); + test("terminal failure tolerates a deleted account", async () => { accountFindUnique.mockResolvedValue(null); await expect( diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts index 2f042250c..1aecaa29f 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -2,7 +2,6 @@ import * as Sentry from "@sentry/node"; import { Account, AccountPermissionSyncIssue, - AccountPermissionSyncJobStatus, PermissionSyncSource, PrismaClient, } from "@sourcebot/db"; @@ -247,30 +246,13 @@ export const createAccountPermissionSyncWorkload = ({ } }, onStarted: async ({ data: { accountId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.accountPermissionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - accountId, - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, - }); - await tx.account.update({ - where: { - id: accountId, - }, - data: { - latestPermissionSyncJobId: jobId, - }, - }); + await db.account.update({ + where: { + id: accountId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, }); }, onCompleted: async ({ @@ -278,16 +260,6 @@ export const createAccountPermissionSyncWorkload = ({ jobId, }) => { const account = await db.$transaction(async (tx) => { - await tx.accountPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); await tx.account.updateMany({ where: { id: accountId, @@ -326,16 +298,6 @@ export const createAccountPermissionSyncWorkload = ({ }, }); - await db.accountPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); const account = await db.account.findUnique({ where: { id: accountId, diff --git a/packages/backend/src/ee/auditLogPruneWorkload.ts b/packages/backend/src/ee/auditLogPruneWorkload.ts index bc8e0f18e..b70732ec5 100644 --- a/packages/backend/src/ee/auditLogPruneWorkload.ts +++ b/packages/backend/src/ee/auditLogPruneWorkload.ts @@ -16,15 +16,11 @@ interface Props { retentionDays: number; } -interface AuditLogPruneResult { - deleted: number; -} - export const createAuditLogPruneWorkload = ({ db, enabled, retentionDays, -}: Props): Workload<"audit-log-prune", AuditLogPruneResult> => ({ +}: Props): Workload<"audit-log-prune"> => ({ queueSpec: AUDIT_LOG_PRUNE_QUEUE, concurrency: 1, ...(enabled && retentionDays > 0 diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts index a66bdf240..ce1394bd1 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -87,17 +87,11 @@ const repoUpdate = vi.fn().mockResolvedValue(repo); const repoUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const accountFindMany = vi.fn().mockResolvedValue([]); const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); -const permissionSyncJobUpsert = vi.fn(); -const permissionSyncJobUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const transactionClient = { repo: { update: repoUpdate, updateMany: repoUpdateMany, }, - repoPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, }; const transaction = vi.fn( ( @@ -122,10 +116,6 @@ const db = { accountToRepoPermission: { createMany: permissionCreateMany, }, - repoPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, $transaction: transaction, } as unknown as PrismaClient; @@ -164,7 +154,6 @@ beforeEach(() => { mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); repoFindUniqueOrThrow.mockResolvedValue(repo); - permissionSyncJobUpdateMany.mockResolvedValue({ count: 1 }); accountFindMany.mockResolvedValue([]); repoUpdate.mockResolvedValue(repo); repoUpdateMany.mockResolvedValue({ count: 1 }); @@ -172,7 +161,7 @@ beforeEach(() => { }); describe("repoPermissionSyncWorkload", () => { - test("uses the configured concurrency and database-backed lifecycle hooks", () => { + test("uses the configured concurrency and lifecycle hooks", () => { const workload = createWorkload(); expect(workload.queueSpec.name).toBe("repo-permission-sync"); @@ -350,42 +339,21 @@ describe("repoPermissionSyncWorkload", () => { }); }); - test("marks a job as in progress when started", async () => { + test("records the latest job when started", async () => { await createWorkload().onStarted?.(lifecycleContext); - expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ - where: { id: "job_1" }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - }, - create: { - id: "job_1", - repoId: 42, - status: "IN_PROGRESS", - }, - }); expect(repoUpdate).toHaveBeenCalledWith({ where: { id: 42 }, data: { latestPermissionSyncJobId: "job_1" }, }); - expect(transaction).toHaveBeenCalledOnce(); + expect(transaction).not.toHaveBeenCalled(); }); - test("marks a job completed and updates the repo sync timestamp when it is still latest", async () => { + test("updates the repo sync timestamp when the completed job is still latest", async () => { await createWorkload().onCompleted?.(lifecycleContext, { repoName: "sourcebot-dev/sourcebot", }); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ - where: { id: "job_1" }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); expect(repoUpdateMany).toHaveBeenCalledWith({ where: { id: 42, @@ -395,11 +363,10 @@ describe("repoPermissionSyncWorkload", () => { permissionSyncedAt: expect.any(Date), }, }); - expect(transaction).toHaveBeenCalledOnce(); + expect(transaction).not.toHaveBeenCalled(); }); test("does not fail completion after the repo has been deleted", async () => { - permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); repoUpdateMany.mockResolvedValue({ count: 0 }); await expect( @@ -409,19 +376,11 @@ describe("repoPermissionSyncWorkload", () => { ).resolves.toBeUndefined(); }); - test("marks a job failed after terminal failure", async () => { + test("reports terminal failures", async () => { const error = new Error("Upstream unavailable"); await createWorkload().onTerminalFailure?.(lifecycleContext, error); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ - where: { id: "job_1" }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Upstream unavailable", - }, - }); expect(mocks.captureException).toHaveBeenCalledWith(error, { tags: { jobId: "job_1", diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts index 199f111ee..5e880433a 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -1,9 +1,5 @@ import * as Sentry from "@sentry/node"; -import { - PermissionSyncSource, - PrismaClient, - RepoPermissionSyncJobStatus, -} from "@sourcebot/db"; +import { PermissionSyncSource, PrismaClient } from "@sourcebot/db"; import { createLogger, REPO_PERMISSION_SYNC_QUEUE, @@ -39,20 +35,13 @@ interface RepoPermissionSyncWorkloadDependencies { settings: Settings; } -interface RepoPermissionSyncResult { - repoName: string; -} - const REPO_PERMISSION_SYNC_LOCK_DURATION_MS = 60_000; const logger = createLogger("repo-permission-sync-workload"); export const createRepoPermissionSyncWorkload = ({ db, settings, -}: RepoPermissionSyncWorkloadDependencies): Workload< - "repo-permission-sync", - RepoPermissionSyncResult -> => ({ +}: RepoPermissionSyncWorkloadDependencies): Workload<"repo-permission-sync"> => ({ queueSpec: REPO_PERMISSION_SYNC_QUEUE, concurrency: settings.maxRepoPermissionSyncJobConcurrency, executionLock: { @@ -133,56 +122,27 @@ export const createRepoPermissionSyncWorkload = ({ }; }, onStarted: async ({ data: { repoId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.repoPermissionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - repoId, - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, - }); - await tx.repo.update({ - where: { - id: repoId, - }, - data: { - latestPermissionSyncJobId: jobId, - }, - }); + await db.repo.update({ + where: { + id: repoId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, }); }, onCompleted: async ( { data: { repoId }, jobId }, { repoName }, ) => { - await db.$transaction(async (tx) => { - await tx.repoPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); - await tx.repo.updateMany({ - where: { - id: repoId, - latestPermissionSyncJobId: jobId, - }, - data: { - permissionSyncedAt: new Date(), - }, - }); + await db.repo.updateMany({ + where: { + id: repoId, + latestPermissionSyncJobId: jobId, + }, + data: { + permissionSyncedAt: new Date(), + }, }); logger.debug(`Permissions synced for repo ${repoName}`); @@ -195,17 +155,6 @@ export const createRepoPermissionSyncWorkload = ({ }, }); - await db.repoPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); - logger.error( `Repo permission sync job failed for repo ${repoId}: ${error.message}`, ); diff --git a/packages/backend/src/gitea.test.ts b/packages/backend/src/gitea.test.ts new file mode 100644 index 000000000..7a669dfa6 --- /dev/null +++ b/packages/backend/src/gitea.test.ts @@ -0,0 +1,132 @@ +import type { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { status: 404 }); + + return { + userListRepos: vi.fn(async () => { + throw notFound; + }), + orgListRepos: vi.fn(async () => { + throw notFound; + }), + repoGet: vi.fn(async () => { + throw notFound; + }), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("gitea-js", () => ({ + giteaApi: () => ({ + users: { + userListRepos: mocks.userListRepos, + }, + orgs: { + orgListRepos: mocks.orgListRepos, + }, + repos: { + repoGet: mocks.repoGet, + }, + }), +})); + +vi.mock("./utils.js", () => ({ + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { getGiteaReposFromConfig } from './gitea'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Gitea repository discovery", () => { + test("reports inaccessible configured targets as partial successes", async () => { + const config = { + type: "gitea", + orgs: ["missing-org"], + repos: ["missing-owner/missing-repo"], + users: ["missing-user"], + } satisfies GiteaConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getGiteaReposFromConfig(config) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: "missing-org", + }, + message: "Gitea organization was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "missing-owner/missing-repo", + }, + message: "Gitea repository was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: "missing-user", + }, + message: "Gitea user was not found or is inaccessible.", + }, + ], + }); + }); + + test("reports malformed repositories returned by Gitea", async () => { + mocks.orgListRepos.mockResolvedValueOnce({ + data: [null, { id: 123 }], + headers: new Headers({ "x-total-count": "2" }), + } as never); + + const result = await collectRepositoryDiscoveryIssues(() => + getGiteaReposFromConfig({ + type: "gitea", + orgs: ["my-org"], + }) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "Gitea returned a null repository, so it was skipped.", + }, + { + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "repository", + value: "123", + }, + message: "Gitea returned a repository without a full name, so it was skipped.", + }, + ], + }); + }); +}); diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index 4ebdc2033..af7ed80ba 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -7,6 +7,7 @@ import fetch from 'cross-fetch'; import { Api, giteaApi, Repository as GiteaRepository, HttpResponse } from 'gitea-js'; import micromatch from 'micromatch'; import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js'; +import { reportRepositoryDiscoveryIssue } from './repositoryDiscoveryIssueContext.js'; import { measure } from './utils.js'; const logger = createLogger('gitea'); @@ -45,33 +46,42 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig) => }); let allRepos: GiteaRepository[] = []; - let allWarnings: string[] = []; if (config.orgs) { - const { repos, warnings } = await getReposForOrgs(config.orgs, api); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await getReposForOrgs(config.orgs, api)); } if (config.repos) { - const { repos, warnings } = await getRepos(config.repos, api); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await getRepos(config.repos, api)); } if (config.users) { - const { repos, warnings } = await getReposOwnedByUsers(config.users, api); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await getReposOwnedByUsers(config.users, api)); } allRepos = allRepos.filter(repo => { if (repo === null || repo === undefined) { logger.warn(`Skipping null/undefined repository returned by the Gitea API`); + reportRepositoryDiscoveryIssue({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "Gitea returned a null repository, so it was skipped.", + }); return false; } if (repo.full_name === undefined) { logger.warn(`Repository with undefined full_name found: repoId=${repo.id}`); + reportRepositoryDiscoveryIssue({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + ...(repo.id !== undefined && repo.id !== null ? { + subject: { + kind: "repository" as const, + value: String(repo.id), + }, + } : {}), + message: "Gitea returned a repository without a full name, so it was skipped.", + }); return false; } return true; @@ -88,10 +98,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig) => }); logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; } const shouldExcludeRepo = ({ @@ -148,32 +155,30 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { ); logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); if (e?.status === 404) { const warning = `User ${user} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: user, + }, + message: "Gitea user was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } const getReposForOrgs = async (orgs: string[], api: Api) => { @@ -189,32 +194,30 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { ); logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); if (e?.status === 404) { const warning = `Organization ${org} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: org, + }, + message: "Gitea organization was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } const getRepos = async (repoList: string[], api: Api) => { @@ -232,32 +235,30 @@ const getRepos = async (repoList: string[], api: Api) => { } logger.debug(`Found repo ${repo} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: [response.data] - }; + return [response.data]; } catch (e: any) { Sentry.captureException(e); if (e?.status === 404) { const warning = `Repository ${repo} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Gitea repository was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } // @see : https://docs.gitea.com/development/api-usage#pagination @@ -284,4 +285,4 @@ const paginate = async (request: (page: number) => Promise { + const notFound = Object.assign(new Error("Not Found"), { status: 404 }); + + return { + reposGet: vi.fn(async () => { + throw notFound; + }), + paginateIterator: vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + throw notFound; + }, + })), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@octokit/rest", () => ({ + Octokit: class { + repos = { + get: mocks.reposGet, + listForOrg: vi.fn(), + }; + rest = { + search: { + repos: vi.fn(), + }, + users: { + getAuthenticated: vi.fn(), + }, + }; + paginate = { + iterator: mocks.paginateIterator, + }; + }, +})); + +vi.mock("./ee/githubAppManager.js", () => ({ + GithubAppManager: { + getInstance: () => ({ + ensureInitialized: vi.fn(), + appsConfigured: () => false, + }), + }, +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; import { OctokitRepository, shouldExcludeRepo, detectGitHubTokenType, supportsOAuthScopeIntrospection, + getGitHubReposFromConfig, } from './github'; +describe("GitHub repository discovery", () => { + test("reports inaccessible configured targets as partial successes", async () => { + const config = { + type: "github", + url: "https://github.example.com", + orgs: ["missing-org"], + repos: ["missing-owner/missing-repo"], + users: ["missing-user"], + } satisfies GithubConnectionConfig; + const result = await collectRepositoryDiscoveryIssues(() => + getGitHubReposFromConfig( + config, + new AbortController().signal, + ) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: "missing-org", + }, + message: "GitHub organization was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "missing-owner/missing-repo", + }, + message: "GitHub repository was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: "missing-user", + }, + message: "GitHub user was not found or is inaccessible.", + }, + ], + }); + }); +}); + describe('detectGitHubTokenType', () => { test('detects classic PAT (ghp_)', () => { expect(detectGitHubTokenType('ghp_abc123def456')).toBe('classic_pat'); diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 3b5101b7b..969de5438 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -9,6 +9,7 @@ import { hasEntitlement } from "./entitlements.js"; import micromatch from "micromatch"; import pLimit from "p-limit"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { GithubAppManager } from "./ee/githubAppManager.js"; import { fetchWithRetry, measure } from "./utils.js"; @@ -142,6 +143,16 @@ export const getOctokitWithGithubApp = async ( logger.warn( `No matching GitHub App installation found for ${context} on ${hostname}; falling back to legacy GitHub authentication.` ); + reportRepositoryDiscoveryIssue({ + code: "AUTHENTICATION_FALLBACK", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "configuration", + value: `GitHub App installation for ${owner} on ${hostname}`, + }, + message: + "No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.", + }); return octokit; } @@ -152,7 +163,7 @@ export const getOctokitWithGithubApp = async ( return octokitFromToken; } -export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, signal: AbortSignal): Promise<{ repos: OctokitRepository[], warnings: string[] }> => { +export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, signal: AbortSignal): Promise => { const hostname = config.url ? new URL(config.url).hostname : GITHUB_CLOUD_HOSTNAME; @@ -180,24 +191,20 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, s } let allRepos: OctokitRepository[] = []; - let allWarnings: string[] = []; if (config.orgs) { - const { repos, warnings } = await getReposForOrgs(config.orgs, octokit, signal, config.url); + const repos = await getReposForOrgs(config.orgs, octokit, signal, config.url); allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { repos, warnings } = await getRepos(config.repos, octokit, signal, config.url); + const repos = await getRepos(config.repos, octokit, signal, config.url); allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); } if (config.users) { - const { repos, warnings } = await getReposOwnedByUsers(config.users, octokit, signal, config.url); + const repos = await getReposOwnedByUsers(config.users, octokit, signal, config.url); allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -215,10 +222,7 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, s logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; } export const getRepoCollaborators = async (owner: string, repo: string, octokit: Octokit) => { @@ -337,10 +341,7 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A }); logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for user ${user}.`, error); @@ -348,22 +349,23 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A if (isHttpError(error, 404)) { const warning = `User ${user} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: user, + }, + message: "GitHub user was not found or is inaccessible.", + }); + return []; } throw error; } }))); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { @@ -400,10 +402,7 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi }); logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for org ${org}.`, error); @@ -411,22 +410,23 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi if (isHttpError(error, 404)) { const warning = `Organization ${org} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: org, + }, + message: "GitHub organization was not found or is inaccessible.", + }); + return []; } throw error; } }))); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { @@ -449,10 +449,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna }); logger.debug(`Found info for repository ${repo} in ${durationMs}ms`); - return { - type: 'valid' as const, - data: [result.data] - }; + return [result.data]; } catch (error) { Sentry.captureException(error); @@ -461,22 +458,23 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna if (isHttpError(error, 404)) { const warning = `Repository ${repo} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "GitHub repository was not found or is inaccessible.", + }); + return []; } throw error; } }))); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } export const shouldExcludeRepo = ({ diff --git a/packages/backend/src/githubAppAuth.test.ts b/packages/backend/src/githubAppAuth.test.ts index b9e56072f..8a28aa4f5 100644 --- a/packages/backend/src/githubAppAuth.test.ts +++ b/packages/backend/src/githubAppAuth.test.ts @@ -23,6 +23,9 @@ vi.mock('@sourcebot/shared', () => ({ FALLBACK_GITHUB_CLOUD_TOKEN: undefined, }, getTokenFromConfig: vi.fn(), + repositoryDiscoveryIssueSchema: { + parse: (issue: unknown) => issue, + }, })); vi.mock('./entitlements.js', () => ({ @@ -40,6 +43,7 @@ vi.mock('./ee/githubAppManager.js', () => ({ })); import { getOctokitWithGithubApp } from './github.js'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; describe('getOctokitWithGithubApp', () => { beforeEach(() => { @@ -89,6 +93,37 @@ describe('getOctokitWithGithubApp', () => { expect(mocks.hasEntitlement).not.toHaveBeenCalled(); }); + test('reports incomplete discovery when no matching installation exists', async () => { + const fallbackOctokit = {} as Octokit; + mocks.hasEntitlement.mockResolvedValue(true); + mocks.getInstallationToken.mockResolvedValue(null); + + await expect( + collectRepositoryDiscoveryIssues(() => + getOctokitWithGithubApp( + fallbackOctokit, + 'example', + undefined, + 'org example', + ) + ), + ).resolves.toEqual({ + value: fallbackOctokit, + issues: [ + { + code: 'AUTHENTICATION_FALLBACK', + effect: 'DISCOVERY_INCOMPLETE', + subject: { + kind: 'configuration', + value: 'GitHub App installation for example on github.com', + }, + message: + 'No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.', + }, + ], + }); + }); + test('does not fall back when GitHub App token resolution fails', async () => { const error = new Error('rate limited'); mocks.hasEntitlement.mockResolvedValue(true); diff --git a/packages/backend/src/gitlab.test.ts b/packages/backend/src/gitlab.test.ts index 2cfb4042a..31f75dcf1 100644 --- a/packages/backend/src/gitlab.test.ts +++ b/packages/backend/src/gitlab.test.ts @@ -1,6 +1,114 @@ -import { expect, test } from 'vitest'; -import { shouldExcludeProject } from './gitlab'; -import { ProjectSchema } from '@gitbeaker/rest'; +import type { ProjectSchema } from '@gitbeaker/rest'; +import type { GitlabConnectionConfig } from '@sourcebot/schemas/v3/gitlab.type'; +import { describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { + cause: { response: { status: 404 } }, + }); + + return { + groupsAllProjects: vi.fn(async () => { + throw notFound; + }), + usersAllProjects: vi.fn(async () => { + throw notFound; + }), + projectsShow: vi.fn(async () => { + throw notFound; + }), + projectsAll: vi.fn(async () => []), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@gitbeaker/rest", () => ({ + Gitlab: class { + Groups = { + allProjects: mocks.groupsAllProjects, + }; + Users = { + allProjects: mocks.usersAllProjects, + }; + Projects = { + all: mocks.projectsAll, + show: mocks.projectsShow, + }; + }, +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; +import { getGitLabReposFromConfig, shouldExcludeProject } from './gitlab'; + +describe("GitLab repository discovery", () => { + test("reports unsupported configuration and inaccessible targets", async () => { + const config = { + type: "gitlab", + all: true, + groups: ["missing-group"], + users: ["missing-user"], + projects: ["missing-group/missing-project"], + } satisfies GitlabConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getGitLabReposFromConfig(config) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "all", + }, + message: "The all option is not supported for GitLab Cloud.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "group", + value: "missing-group", + }, + message: "GitLab group was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: "missing-user", + }, + message: "GitLab user was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: "missing-group/missing-project", + }, + message: "GitLab project was not found or is inaccessible.", + }, + ], + }); + expect(mocks.projectsAll).not.toHaveBeenCalled(); + }); +}); test('shouldExcludeProject returns false when the project is not excluded.', () => { diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 182657f21..6a0c57042 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -6,6 +6,7 @@ import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; import { env } from "@sourcebot/shared"; import micromatch from "micromatch"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { fetchWithRetry, measure } from "./utils.js"; const logger = createLogger('gitlab'); @@ -50,7 +51,6 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = }); let allRepos: ProjectSchema[] = []; - let allWarnings: string[] = []; if (config.all === true) { if (hostname !== GITLAB_CLOUD_HOSTNAME) { @@ -72,7 +72,15 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = } else { const warning = `Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`; logger.warn(warning); - allWarnings = allWarnings.concat(warning); + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "all", + }, + message: "The all option is not supported for GitLab Cloud.", + }); } } @@ -88,10 +96,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return fetchWithRetry(fetchFn, `group ${group}`, logger); }); logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch projects for group ${group}.`, e); @@ -99,10 +104,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = if (e?.cause?.response?.status === 404) { const warning = `Group ${group} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "group", + value: group, + }, + message: "GitLab group was not found or is inaccessible.", + }); + return []; } throw e; @@ -110,9 +121,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = })); throwIfAnyFailed(results); - const { validItems: validRepos, warnings } = processPromiseResults(results); - allRepos = allRepos.concat(validRepos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(processPromiseResults(results)); } if (config.users) { @@ -126,10 +135,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return fetchWithRetry(fetchFn, `user ${user}`, logger); }); logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch projects for user ${user}.`, e); @@ -137,10 +143,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = if (e?.cause?.response?.status === 404) { const warning = `User ${user} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: user, + }, + message: "GitLab user was not found or is inaccessible.", + }); + return []; } throw e; @@ -148,9 +160,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = })); throwIfAnyFailed(results); - const { validItems: validRepos, warnings } = processPromiseResults(results); - allRepos = allRepos.concat(validRepos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(processPromiseResults(results)); } if (config.projects) { @@ -162,10 +172,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return fetchWithRetry(fetchFn, `project ${project}`, logger); }); logger.debug(`Found project ${project} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: [data] - }; + return [data]; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch project ${project}.`, e); @@ -173,10 +180,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = if (e?.cause?.response?.status === 404) { const warning = `Project ${project} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "GitLab project was not found or is inaccessible.", + }); + return []; } throw e; @@ -184,9 +197,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = })); throwIfAnyFailed(results); - const { validItems: validRepos, warnings } = processPromiseResults(results); - allRepos = allRepos.concat(validRepos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(processPromiseResults(results)); } let repos = allRepos @@ -204,10 +215,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; } export const shouldExcludeProject = ({ diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 1974421e6..fc2ec78fc 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -12,8 +12,9 @@ import { shutdownPosthog } from "./posthog.js"; import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { redis } from "./redis.js"; -import { createConnectionWorkload } from "./connectionWorkload.js"; -import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; +import { createConnectionSyncWorkload } from "./connectionSyncWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; @@ -50,7 +51,7 @@ logger.info('Worker started.'); const jobManager = new BullMQJobManager(redis); -const connectionWorkload = createConnectionWorkload({ +const connectionSyncWorkload = createConnectionSyncWorkload({ db: prisma, jobManager, settings, @@ -59,6 +60,10 @@ const repoIndexWorkload = createRepoIndexWorkload({ db: prisma, settings, }); +const repoCleanupWorkload = createRepoCleanupWorkload({ + db: prisma, + settings, +}); const accountPermissionSyncWorkload = createAccountPermissionSyncWorkload({ db: prisma, settings, @@ -77,8 +82,9 @@ const auditLogPruneWorkload = createAuditLogPruneWorkload({ retentionDays: env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS, }); -jobManager.register(connectionWorkload); +jobManager.register(connectionSyncWorkload); jobManager.register(repoIndexWorkload); +jobManager.register(repoCleanupWorkload); jobManager.register(accountPermissionSyncWorkload); jobManager.register(repoPermissionSyncWorkload); jobManager.register(attachmentPruneWorkload); diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index ffa874c61..161e993ef 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -1,5 +1,7 @@ import { Redis } from "ioredis"; +import type { ConnectionSyncResult } from "@sourcebot/shared"; import { beforeEach, describe, expect, test, vi } from "vitest"; +import { z, type ZodType } from "zod"; import { ProcessContext, Workload } from "./types.js"; const mocks = vi.hoisted(() => { @@ -100,11 +102,14 @@ vi.mock("bullmq", () => ({ import { BullMQJobManager } from "./jobManager.js"; const createWorkload = ( - overrides: Partial> = {}, -): Workload<"connection-sync", { repoCount: number }> => ({ + overrides: Partial> = {}, +): Workload<"connection-sync"> => ({ queueSpec: { name: "connection-sync", - dedupKey: ({ connectionId }) => `connection:${connectionId}`, + resultSchema: z.unknown() as ZodType, + deduplication: ({ connectionId }) => ({ + id: `connection:${connectionId}`, + }), jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, @@ -116,7 +121,7 @@ const createWorkload = ( }, }, concurrency: 2, - process: vi.fn(async () => ({ repoCount: 3 })), + process: vi.fn(async () => ({ outcome: "SUCCESS" as const })), ...overrides, }); @@ -256,7 +261,7 @@ describe("BullMQJobManager lifecycle", () => { }), process: vi.fn(async () => { calls.push("processed"); - return { repoCount: 3 }; + return { outcome: "SUCCESS" }; }), onCompleted: vi.fn(async () => { calls.push("completed"); @@ -282,7 +287,7 @@ describe("BullMQJobManager lifecycle", () => { jobId: "job-1", maxAttempts: 2, }), - { repoCount: 3 }, + { outcome: "SUCCESS" }, ); expect( vi.mocked(workload.onCompleted!).mock.calls[0][0], @@ -315,7 +320,7 @@ describe("BullMQJobManager lifecycle", () => { process: vi.fn(async ({ signal }) => { expect(signal).toBe(workloadSignal); calls.push("processed"); - return { repoCount: 3 }; + return { outcome: "SUCCESS" }; }), }); const manager = new BullMQJobManager({} as Redis); @@ -324,7 +329,7 @@ describe("BullMQJobManager lifecycle", () => { await expect( mocks.workers[0].processor({ ...job, attemptsMade: 0 }), - ).resolves.toEqual({ repoCount: 3 }); + ).resolves.toEqual({ outcome: "SUCCESS" }); expect(calls).toEqual([ "lock-acquired", @@ -358,7 +363,7 @@ describe("BullMQJobManager lifecycle", () => { const process = vi.fn( async (context: ProcessContext<"connection-sync">) => { expect(context).not.toHaveProperty("logger"); - return { repoCount: 3 }; + return { outcome: "SUCCESS" }; }, ); const manager = new BullMQJobManager({} as Redis); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 53beb1533..f2c33e542 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -6,6 +6,7 @@ import { DataOf, JobEnqueueOptions, QueueName, + ResultOf, Schedule, scheduleToMs, runWithJobLogContext, @@ -24,10 +25,7 @@ const STALLED_JOB_TERMINAL_ERROR = "job stalled more than allowable limit"; const logger = createLogger(LOG_TAG); export class BullMQJobManager implements JobManager { - private readonly workloads = new Map< - string, - Workload - >(); + private readonly workloads = new Map>(); private readonly workers = new Map(); private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); @@ -325,10 +323,10 @@ export class BullMQJobManager implements JobManager { ); } - private async onWorkloadJobCompleted( - workload: Workload, + private async onWorkloadJobCompleted( + workload: Workload, job: Job, - result: TResult, + result: ResultOf, ): Promise { const label = `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`; const attempt = Math.max(job.attemptsMade, 1); diff --git a/packages/backend/src/reconcileJobSchedulers.test.ts b/packages/backend/src/reconcileJobSchedulers.test.ts index 223da8506..478daaec9 100644 --- a/packages/backend/src/reconcileJobSchedulers.test.ts +++ b/packages/backend/src/reconcileJobSchedulers.test.ts @@ -138,7 +138,7 @@ describe("reconcileJobSchedulers", () => { "repo-index", "repo-index-v1-42", 3_600_000, - { repoId: 42, type: "INDEX" }, + { repoId: 42 }, { priority: 10 }, ); expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( @@ -188,7 +188,7 @@ describe("reconcileJobSchedulers", () => { "repo-index", "repo-index-v1-42", 3_600_000, - { repoId: 42, type: "INDEX" }, + { repoId: 42 }, { priority: 10 }, ); expect(mocks.upsertJobScheduler).not.toHaveBeenCalledWith( @@ -203,8 +203,8 @@ describe("reconcileJobSchedulers", () => { "repo-index-v1-84", ); expect(mocks.trigger).toHaveBeenCalledWith( - "repo-index", - { repoId: 84, type: "CLEANUP" }, + "repo-cleanup", + { repoId: 84 }, { priority: 10 }, ); expect( @@ -241,7 +241,7 @@ describe("reconcileJobSchedulers", () => { "repo-index", "repo-index-v1-84", 3_600_000, - { repoId: 84, type: "INDEX" }, + { repoId: 84 }, { priority: 10 }, ); expect(mocks.removeJobScheduler).not.toHaveBeenCalledWith( diff --git a/packages/backend/src/reconcileJobSchedulers.ts b/packages/backend/src/reconcileJobSchedulers.ts index 9f21f1d57..d4e5f43e4 100644 --- a/packages/backend/src/reconcileJobSchedulers.ts +++ b/packages/backend/src/reconcileJobSchedulers.ts @@ -118,7 +118,7 @@ export const reconcileJobSchedulers = async ({ schedulerIdPrefix: "repo-index-v1-", targets: repos.map(({ id }) => ({ schedulerId: `repo-index-v1-${id}`, - data: { repoId: id, type: "INDEX" }, + data: { repoId: id }, })), schedule: settings.reindexIntervalMs, jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, @@ -150,8 +150,8 @@ export const reconcileJobSchedulers = async ({ await Promise.all( orphanedReposToCleanup.map(({ id }) => jobManager.trigger( - "repo-index", - { repoId: id, type: "CLEANUP" }, + "repo-cleanup", + { repoId: id }, { priority: JOB_PRIORITIES.SCHEDULED }, ), ), diff --git a/packages/backend/src/repoCleanupWorkload.test.ts b/packages/backend/src/repoCleanupWorkload.test.ts new file mode 100644 index 000000000..4a1e770e5 --- /dev/null +++ b/packages/backend/src/repoCleanupWorkload.test.ts @@ -0,0 +1,185 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; + +const fsMocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + readdir: vi.fn(), + rm: vi.fn(), +})); + +const lifecycleLogger = vi.hoisted(() => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + createLogger: vi.fn(() => lifecycleLogger), +})); + +vi.mock("fs", () => ({ + existsSync: fsMocks.existsSync, +})); + +vi.mock("fs/promises", () => ({ + readdir: fsMocks.readdir, + rm: fsMocks.rm, +})); + +const repoFindUnique = vi.fn(); +const repoDeleteMany = vi.fn(); +const repoUpdate = vi.fn(); + +const db = { + repo: { + findUnique: repoFindUnique, + deleteMany: repoDeleteMany, + update: repoUpdate, + }, +} as unknown as PrismaClient; + +const workload = createRepoCleanupWorkload({ + db, + settings: { + maxRepoIndexingJobConcurrency: 2, + } as never, +}); + +const processContext = { + data: { repoId: 42 }, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +const eligibleRepo = { + id: 42, + name: "github.com/acme/repo", + cloneUrl: "https://github.com/acme/repo.git", + external_codeHostType: "github", + orgId: 1, + indexedAt: null, + isAutoCleanupDisabled: false, + connections: [], +}; + +describe("repoCleanupWorkload", () => { + beforeEach(() => { + vi.clearAllMocks(); + fsMocks.existsSync.mockReturnValue(false); + fsMocks.readdir.mockResolvedValue([]); + fsMocks.rm.mockResolvedValue(undefined); + repoFindUnique.mockResolvedValue(eligibleRepo); + repoDeleteMany.mockResolvedValue({ count: 1 }); + repoUpdate.mockResolvedValue(undefined); + }); + + test("validates eligibility without updating indexing job state", async () => { + await workload.process(processContext); + + expect(repoFindUnique).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: true, + }, + }); + expect(repoDeleteMany).toHaveBeenCalledWith({ + where: { + id: 42, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + expect(repoUpdate).not.toHaveBeenCalled(); + expect(workload.onStarted).toBeUndefined(); + expect(workload.onCompleted).toBeUndefined(); + expect(workload.onTerminalFailure).toBeUndefined(); + }); + + test("removes only shards belonging to the exact repository id", async () => { + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_420_v16.00000.zoekt", + ]); + + await workload.process(processContext); + + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_420_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test("finishes orphaned filesystem cleanup when a retry finds no repo", async () => { + repoFindUnique.mockResolvedValue(null); + fsMocks.existsSync.mockReturnValue(true); + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_99_v16.00000.zoekt", + ]); + + await workload.process(processContext); + + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringMatching(/repos\/42$/), + { recursive: true, force: true }, + ); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_99_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test.each([ + { + name: "automatic cleanup is disabled", + repo: { ...eligibleRepo, isAutoCleanupDisabled: true }, + reason: "automatic cleanup is disabled", + }, + { + name: "the repository was reattached", + repo: { ...eligibleRepo, connections: [{}] }, + reason: "repository has been reattached to a connection", + }, + ])("skips CLEANUP when $name", async ({ repo, reason }) => { + repoFindUnique.mockResolvedValue(repo); + + await workload.process(processContext); + + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.debug).toHaveBeenCalledWith( + `Skipping CLEANUP job for repo 42: ${reason}`, + ); + }); + + test("revalidates cleanup eligibility when atomically deleting the repo", async () => { + repoDeleteMany.mockResolvedValue({ count: 0 }); + + await workload.process(processContext); + + expect(repoDeleteMany).toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.debug).toHaveBeenCalledWith( + "Skipping CLEANUP job for repo 42: repository is no longer eligible for cleanup", + ); + }); +}); diff --git a/packages/backend/src/repoCleanupWorkload.ts b/packages/backend/src/repoCleanupWorkload.ts new file mode 100644 index 000000000..d0fedbe81 --- /dev/null +++ b/packages/backend/src/repoCleanupWorkload.ts @@ -0,0 +1,233 @@ +import type { PrismaClient, Repo } from "@sourcebot/db"; +import { + createLogger, + getRepoIdFromPath, + getRepoPath, + REPO_CLEANUP_QUEUE, +} from "@sourcebot/shared"; +import { existsSync } from "fs"; +import { readdir, rm } from "fs/promises"; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from "./constants.js"; +import { REPOSITORY_EXECUTION_LOCK } from "./repoLock.js"; +import type { Settings, Workload } from "./types.js"; +import { getRepoIdFromShardFileName } from "./utils.js"; + +const logger = createLogger("repo-cleanup-workload"); + +interface Props { + db: PrismaClient; + settings: Settings; +} + +export const createRepoCleanupWorkload = ({ + db, + settings, +}: Props): Workload<"repo-cleanup"> => ({ + queueSpec: REPO_CLEANUP_QUEUE, + concurrency: settings.maxRepoIndexingJobConcurrency, + // Cleanup and indexing use separate queues, but must never operate on the + // same repository path or search index concurrently. + executionLock: REPOSITORY_EXECUTION_LOCK, + process: async ({ data: { repoId }, signal }) => { + signal.throwIfAborted(); + const start = await prepareRepoCleanupJob({ db, repoId }); + + if (start.action === "skip") { + logger.debug( + `Skipping CLEANUP job for repo ${repoId}: ${start.reason}`, + ); + + if (start.repoMissing) { + signal.throwIfAborted(); + await cleanupOrphanedRepoResourcesForRepoId(repoId); + } + return; + } + + signal.throwIfAborted(); + const { repo } = start; + logger.debug(`Running CLEANUP job for repo ${repo.name} (id: ${repo.id})`); + + const { count } = await db.repo.deleteMany({ + where: { + id: repo.id, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + + if (count === 0) { + logger.debug( + `Skipping CLEANUP job for repo ${repo.id}: repository is no longer eligible for cleanup`, + ); + return; + } + + signal.throwIfAborted(); + await cleanupRepository(repo); + }, +}); + +type RepoCleanupStartDecision = + | { + action: "run"; + repo: Repo; + } + | { + action: "skip"; + reason: string; + repoMissing: boolean; + }; + +const prepareRepoCleanupJob = async ({ + db, + repoId, +}: { + db: PrismaClient; + repoId: number; +}): Promise => { + const repo = await db.repo.findUnique({ + where: { id: repoId }, + include: { + connections: true, + }, + }); + + if (!repo) { + return { + action: "skip", + reason: "repository no longer exists", + repoMissing: true, + }; + } + + if (repo.isAutoCleanupDisabled) { + return { + action: "skip", + reason: "automatic cleanup is disabled", + repoMissing: false, + }; + } + + if (repo.connections.length > 0) { + return { + action: "skip", + reason: "repository has been reattached to a connection", + repoMissing: false, + }; + } + + return { + action: "run", + repo, + }; +}; + +const cleanupRepository = async (repo: Repo) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.debug(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const files = (await readdir(INDEX_CACHE_DIR)).filter( + (file) => getRepoIdFromShardFileName(file) === repo.id, + ); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +const cleanupOrphanedRepoResourcesForRepoId = async (repoId: number) => { + const repoPath = `${REPOS_CACHE_DIR}/${repoId}`; + if (existsSync(repoPath)) { + logger.debug(`Deleting orphaned repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + if (!existsSync(INDEX_CACHE_DIR)) { + return; + } + + const shardFiles = (await readdir(INDEX_CACHE_DIR)).filter( + (file) => getRepoIdFromShardFileName(file) === repoId, + ); + for (const file of shardFiles) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting orphaned shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +// Scans the repos and index directories on disk and removes any entries +// that have no corresponding Repo record in the database. This handles +// edge cases where the DB and disk resources are out of sync. +export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { + // --- Repo directories --- + // Dirs are named by repoId: DATA_CACHE_DIR/repos// + if (existsSync(REPOS_CACHE_DIR)) { + const entries = await readdir(REPOS_CACHE_DIR); + const repoIdToPath = new Map(); + for (const entry of entries) { + const repoPath = `${REPOS_CACHE_DIR}/${entry}`; + const repoId = getRepoIdFromPath(repoPath); + if (repoId !== undefined) { + repoIdToPath.set(repoId, repoPath); + } + } + + if (repoIdToPath.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToPath.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map((repo) => repo.id)); + for (const [repoId, repoPath] of repoIdToPath) { + if (!existingIds.has(repoId)) { + logger.debug( + `Removing orphaned repo directory with no DB record: ${repoPath}`, + ); + await rm(repoPath, { recursive: true, force: true }); + } + } + } + } + + // --- Index shards --- + // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt + if (existsSync(INDEX_CACHE_DIR)) { + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdToShards = new Map(); + for (const entry of entries) { + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + const shards = repoIdToShards.get(repoId) ?? []; + shards.push(entry); + repoIdToShards.set(repoId, shards); + } + } + + if (repoIdToShards.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToShards.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map((repo) => repo.id)); + for (const [repoId, shards] of repoIdToShards) { + if (!existingIds.has(repoId)) { + for (const entry of shards) { + const shardPath = `${INDEX_CACHE_DIR}/${entry}`; + logger.debug( + `Removing orphaned index shard with no DB record: ${shardPath}`, + ); + await rm(shardPath, { force: true }); + } + } + } + } + } +}; diff --git a/packages/backend/src/repoCompileUtils.test.ts b/packages/backend/src/repoCompileUtils.test.ts index 344079f87..df0d90f82 100644 --- a/packages/backend/src/repoCompileUtils.test.ts +++ b/packages/backend/src/repoCompileUtils.test.ts @@ -25,6 +25,7 @@ vi.mock('fs/promises', () => ({ import { isPathAValidGitRepoRoot, getOriginUrl, isUrlAValidGitRepo } from './git.js'; import { glob } from 'glob'; import fs from 'fs/promises'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; const mockedGlob = vi.mocked(glob); const mockedIsPathAValidGitRepoRoot = vi.mocked(isPathAValidGitRepoRoot); @@ -43,7 +44,7 @@ describe('compileGenericGitHostConfig_file', () => { vi.resetAllMocks(); }); - test('should return warning when glob pattern matches no paths', async () => { + test('should return no repositories when glob pattern matches no paths', async () => { mockedGlob.mockResolvedValue([]); const config = { @@ -53,13 +54,34 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings).toHaveLength(1); - expect(result.warnings[0]).toContain('No paths matched the pattern'); - expect(result.warnings[0]).toContain('/path/to/nonexistent/repo'); + expect(result).toHaveLength(0); }); - test('should return warning when path is a file, not a directory', async () => { + test('reports when a glob pattern matches no paths', async () => { + mockedGlob.mockResolvedValue([]); + + const result = await collectRepositoryDiscoveryIssues(() => + compileGenericGitHostConfig_file({ + type: 'git', + url: 'file:///path/to/**/repo', + }, 1) + ); + + expect(result).toEqual({ + value: [], + issues: [{ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: "/path/to/**/repo", + }, + message: "The configured path did not match any repository sources.", + }], + }); + }); + + test('should return no repositories when path is a file, not a directory', async () => { mockedGlob.mockResolvedValue(['/path/to/a-file.txt']); mockedFsStat.mockResolvedValue({ isDirectory: () => false } as any); @@ -70,12 +92,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings.length).toBeGreaterThanOrEqual(1); - expect(result.warnings.some(w => w.includes('not a directory'))).toBe(true); + expect(result).toHaveLength(0); }); - test('should return warning when path is not a valid git repo', async () => { + test('should return no repositories when path is not a valid git repo', async () => { mockedGlob.mockResolvedValue(['/path/to/not-a-repo']); mockedIsPathAValidGitRepoRoot.mockResolvedValue(false); @@ -86,13 +106,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings.length).toBeGreaterThanOrEqual(1); - expect(result.warnings.some(w => w.includes('not a git repository'))).toBe(true); - expect(result.warnings.some(w => w.includes('No valid git repositories found'))).toBe(true); + expect(result).toHaveLength(0); }); - test('should return warning when git repo has no origin url', async () => { + test('should return no repositories when git repo has no origin url', async () => { mockedGlob.mockResolvedValue(['/path/to/repo']); mockedIsPathAValidGitRepoRoot.mockResolvedValue(true); mockedGetOriginUrl.mockResolvedValue(null); @@ -104,10 +121,7 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings.length).toBeGreaterThanOrEqual(1); - expect(result.warnings.some(w => w.includes('remote.origin.url not found'))).toBe(true); - expect(result.warnings.some(w => w.includes('No valid git repositories found'))).toBe(true); + expect(result).toHaveLength(0); }); test('should successfully compile when valid git repo is found', async () => { @@ -122,10 +136,9 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); - expect(result.repoData[0].cloneUrl).toBe('file:///path/to/valid/repo'); - expect(result.repoData[0].name).toBe('github.com/test/repo'); + expect(result).toHaveLength(1); + expect(result[0].cloneUrl).toBe('file:///path/to/valid/repo'); + expect(result[0].name).toBe('github.com/test/repo'); }); test('should include port in repo name when origin url has a port', async () => { @@ -140,14 +153,13 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); - expect(result.repoData[0].cloneUrl).toBe('file:///path/to/valid/repo'); + expect(result).toHaveLength(1); + expect(result[0].cloneUrl).toBe('file:///path/to/valid/repo'); // The name should include the port to match what zoekt derives from the origin URL - expect(result.repoData[0].name).toBe('git.kernel.org:443/pub/scm/bluetooth/bluez'); + expect(result[0].name).toBe('git.kernel.org:443/pub/scm/bluetooth/bluez'); }); - test('should return warnings for invalid repos and success for valid ones', async () => { + test('should return valid repositories and omit invalid ones', async () => { mockedGlob.mockResolvedValue(['/path/to/valid/repo', '/path/to/invalid/repo']); mockedIsPathAValidGitRepoRoot.mockImplementation(async ({ path }) => { return path === '/path/to/valid/repo'; @@ -166,10 +178,53 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(1); - expect(result.warnings[0]).toContain('/path/to/invalid/repo'); - expect(result.warnings[0]).toContain('not a git repository'); + expect(result).toHaveLength(1); + }); + + test('reports each invalid local repository source', async () => { + mockedGlob.mockResolvedValue([ + '/path/to/file', + '/path/to/not-a-repo', + '/path/to/no-origin', + ]); + mockedFsStat.mockImplementation(async (repoPath) => ({ + isDirectory: () => repoPath !== '/path/to/file', + }) as any); + mockedIsPathAValidGitRepoRoot.mockImplementation(async ({ path: repoPath }) => + repoPath !== '/path/to/not-a-repo' + ); + mockedGetOriginUrl.mockResolvedValue(null); + + const result = await collectRepositoryDiscoveryIssues(() => + compileGenericGitHostConfig_file({ + type: 'git', + url: 'file:///path/to/*', + }, 1) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { kind: "path", value: "/path/to/file" }, + message: "The configured path is not an accessible directory.", + }, + { + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { kind: "path", value: "/path/to/not-a-repo" }, + message: "The configured path is not a Git repository.", + }, + { + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { kind: "path", value: "/path/to/no-origin" }, + message: "The Git repository does not have a remote.origin.url.", + }, + ], + }); }); test('should decode URL-encoded characters in origin url pathname', async () => { @@ -185,11 +240,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); + expect(result).toHaveLength(1); // The repo name should have decoded spaces, not %20 - expect(result.repoData[0].name).toBe('github.com/test/Project Name With Spaces'); - expect(result.repoData[0].displayName).toBe('github.com/test/Project Name With Spaces'); + expect(result[0].name).toBe('github.com/test/Project Name With Spaces'); + expect(result[0].displayName).toBe('github.com/test/Project Name With Spaces'); }); }); @@ -202,7 +256,7 @@ describe('compileGenericGitHostConfig_url', () => { vi.resetAllMocks(); }); - test('should return warning when url is not a valid git repo', async () => { + test('should return no repositories when url is not a valid git repo', async () => { mockedIsUrlAValidGitRepo.mockResolvedValue(false); const config = { @@ -212,9 +266,31 @@ describe('compileGenericGitHostConfig_url', () => { const result = await compileGenericGitHostConfig_url(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings).toHaveLength(1); - expect(result.warnings[0]).toContain('not a git repository'); + expect(result).toHaveLength(0); + }); + + test('reports an invalid remote repository source', async () => { + mockedIsUrlAValidGitRepo.mockResolvedValue(false); + + const result = await collectRepositoryDiscoveryIssues(() => + compileGenericGitHostConfig_url({ + type: 'git', + url: 'https://example.com/not-a-repo', + }, 1) + ); + + expect(result).toEqual({ + value: [], + issues: [{ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "url", + value: "https://example.com/not-a-repo", + }, + message: "The configured URL is not a Git repository.", + }], + }); }); test('should successfully compile with gitConfig when valid git repo url is found', async () => { @@ -227,13 +303,12 @@ describe('compileGenericGitHostConfig_url', () => { const result = await compileGenericGitHostConfig_url(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); - expect(result.repoData[0].cloneUrl).toBe('https://git.kernel.org/pub/scm/bluetooth/bluez.git'); - expect(result.repoData[0].name).toBe('git.kernel.org/pub/scm/bluetooth/bluez'); + expect(result).toHaveLength(1); + expect(result[0].cloneUrl).toBe('https://git.kernel.org/pub/scm/bluetooth/bluez.git'); + expect(result[0].name).toBe('git.kernel.org/pub/scm/bluetooth/bluez'); // Verify gitConfig is set properly (this is the key fix for SOU-218) - const metadata = result.repoData[0].metadata as { gitConfig?: Record }; + const metadata = result[0].metadata as { gitConfig?: Record }; expect(metadata.gitConfig).toBeDefined(); expect(metadata.gitConfig!['zoekt.name']).toBe('git.kernel.org/pub/scm/bluetooth/bluez'); expect(metadata.gitConfig!['zoekt.web-url']).toBe('https://git.kernel.org/pub/scm/bluetooth/bluez.git'); @@ -253,10 +328,10 @@ describe('compileGenericGitHostConfig_url', () => { const result = await compileGenericGitHostConfig_url(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.repoData[0].name).toBe('github.com/test/repo'); + expect(result).toHaveLength(1); + expect(result[0].name).toBe('github.com/test/repo'); - const metadata = result.repoData[0].metadata as { gitConfig?: Record }; + const metadata = result[0].metadata as { gitConfig?: Record }; expect(metadata.gitConfig!['zoekt.name']).toBe('github.com/test/repo'); }); }); diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 1e65565f1..cdc78d8e0 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -22,6 +22,7 @@ import GitUrlParse from 'git-url-parse'; import { RepoMetadata } from '@sourcebot/shared'; import { SINGLE_TENANT_ORG_ID } from './constants.js'; import pLimit from 'p-limit'; +import { reportRepositoryDiscoveryIssue } from './repositoryDiscoveryIssueContext.js'; export type RepoData = WithRequired; @@ -49,18 +50,11 @@ const extractHostWithPort = (url: string): string | null => { return match ? match[1] : null; }; -type CompileResult = { - repoData: RepoData[], - warnings: string[], -} - export const compileGithubConfig = async ( config: GithubConnectionConfig, connectionId: number, - signal: AbortSignal): Promise => { - const gitHubReposResult = await getGitHubReposFromConfig(config, signal); - const gitHubRepos = gitHubReposResult.repos; - const warnings = gitHubReposResult.warnings; + signal: AbortSignal): Promise => { + const gitHubRepos = await getGitHubReposFromConfig(config, signal); const hostUrl = (config.url ?? 'https://github.com').replace(/\/+$/, ''); @@ -82,10 +76,7 @@ export const compileGithubConfig = async ( }; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const createGitHubRepoRecord = ({ @@ -160,11 +151,9 @@ export const createGitHubRepoRecord = ({ export const compileGitlabConfig = async ( config: GitlabConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { - const gitlabReposResult = await getGitLabReposFromConfig(config); - const gitlabRepos = gitlabReposResult.repos; - const warnings = gitlabReposResult.warnings; + const gitlabRepos = await getGitLabReposFromConfig(config); const hostUrl = (config.url ?? 'https://gitlab.com').replace(/\/+$/, ''); const webUrl = (config.webUrl ?? hostUrl).replace(/\/+$/, ''); @@ -240,19 +229,14 @@ export const compileGitlabConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const compileGiteaConfig = async ( config: GiteaConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { - const giteaReposResult = await getGiteaReposFromConfig(config); - const giteaRepos = giteaReposResult.repos; - const warnings = giteaReposResult.warnings; + const giteaRepos = await getGiteaReposFromConfig(config); const hostUrl = (config.url ?? 'https://gitea.com').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -310,15 +294,12 @@ export const compileGiteaConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const compileGerritConfig = async ( config: GerritConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { const gerritRepos = await getGerritReposFromConfig(config); const hostUrl = config.url.replace(/\/+$/, ''); @@ -394,19 +375,14 @@ export const compileGerritConfig = async ( return record; }) - return { - repoData: repos, - warnings: [], - }; + return repos; } export const compileBitbucketConfig = async ( config: BitbucketConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { - const bitbucketReposResult = await getBitbucketReposFromConfig(config); - const bitbucketRepos = bitbucketReposResult.repos; - const warnings = bitbucketReposResult.warnings; + const bitbucketRepos = await getBitbucketReposFromConfig(config); const hostUrl = (config.url ?? 'https://bitbucket.org').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -561,16 +537,13 @@ export const compileBitbucketConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const compileGenericGitHostConfig = async ( config: GenericGitHostConnectionConfig, connectionId: number -): Promise => { +): Promise => { const configUrl = new URL(config.url); if (configUrl.protocol === 'file:') { return compileGenericGitHostConfig_file(config, connectionId); @@ -587,7 +560,7 @@ export const compileGenericGitHostConfig = async ( export const compileGenericGitHostConfig_file = async ( config: GenericGitHostConnectionConfig, connectionId: number, -): Promise => { +): Promise => { const configUrl = new URL(config.url); assert(configUrl.protocol === 'file:', 'config.url must be a file:// URL'); @@ -597,17 +570,20 @@ export const compileGenericGitHostConfig_file = async ( }); const repos: RepoData[] = []; - const warnings: string[] = []; - // Warn if the glob pattern matched no paths at all if (repoPaths.length === 0) { const warning = `No paths matched the pattern '${configUrl.pathname}'. Please verify the path exists and is accessible.`; logger.warn(warning); - warnings.push(warning); - return { - repoData: repos, - warnings, - }; + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: configUrl.pathname, + }, + message: "The configured path did not match any repository sources.", + }); + return repos; } logger.debug(`Found ${repoPaths.length} path(s) matching pattern '${configUrl.pathname}'`); @@ -617,7 +593,15 @@ export const compileGenericGitHostConfig_file = async ( if (!stat || !stat.isDirectory()) { const warning = `Skipping ${repoPath} - path is not a directory.`; logger.warn(warning); - warnings.push(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: repoPath, + }, + message: "The configured path is not an accessible directory.", + }); return; } @@ -627,7 +611,15 @@ export const compileGenericGitHostConfig_file = async ( if (!isGitRepo) { const warning = `Skipping ${repoPath} - not a git repository.`; logger.warn(warning); - warnings.push(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: repoPath, + }, + message: "The configured path is not a Git repository.", + }); return; } @@ -635,7 +627,15 @@ export const compileGenericGitHostConfig_file = async ( if (!origin) { const warning = `Skipping ${repoPath} - remote.origin.url not found in git config.`; logger.warn(warning); - warnings.push(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: repoPath, + }, + message: "The Git repository does not have a remote.origin.url.", + }); return; } @@ -690,26 +690,20 @@ export const compileGenericGitHostConfig_file = async ( if (repos.length === 0) { const warning = `No valid git repositories found from ${repoPaths.length} matched path(s). Check the warnings for details on individual paths.`; logger.warn(warning); - warnings.push(warning); } else { logger.debug(`Successfully found ${repos.length} valid git repository(s) from ${repoPaths.length} matched path(s)`); } - return { - repoData: repos, - warnings, - } + return repos; } export const compileGenericGitHostConfig_url = async ( config: GenericGitHostConnectionConfig, connectionId: number, -): Promise => { +): Promise => { const remoteUrl = new URL(config.url); assert(remoteUrl.protocol === 'http:' || remoteUrl.protocol === 'https:', 'config.url must be a http:// or https:// URL'); - const warnings: string[] = []; - // Validate that we are dealing with a valid git repo. const isGitRepo = await isUrlAValidGitRepo({ cloneUrl: remoteUrl.toString(), @@ -717,11 +711,16 @@ export const compileGenericGitHostConfig_url = async ( if (!isGitRepo) { const warning = `Skipping ${remoteUrl.toString()} - not a git repository.`; logger.warn(warning); - warnings.push(warning); - return { - repoData: [], - warnings, - } + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "url", + value: remoteUrl.toString(), + }, + message: "The configured URL is not a Git repository.", + }); + return []; } // @note: matches the naming here: @@ -765,19 +764,14 @@ export const compileGenericGitHostConfig_url = async ( } satisfies RepoMetadata, }; - return { - repoData: [repo], - warnings, - } + return [repo]; } export const compileAzureDevOpsConfig = async ( config: AzureDevOpsConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { - const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config); - const azureDevOpsRepos = azureDevOpsReposResult.repos; - const warnings = azureDevOpsReposResult.warnings; + const azureDevOpsRepos = await getAzureDevOpsReposFromConfig(config); const hostUrl = (config.url ?? 'https://dev.azure.com').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -846,8 +840,5 @@ export const compileAzureDevOpsConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 85b0e9a63..ae3d84603 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -1,11 +1,7 @@ import type { PrismaClient } from "@sourcebot/db"; import { beforeEach, describe, expect, test, vi } from "vitest"; - -const fsMocks = vi.hoisted(() => ({ - existsSync: vi.fn(), - readdir: vi.fn(), - rm: vi.fn(), -})); +import { createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; const lifecycleLogger = vi.hoisted(() => ({ debug: vi.fn(), @@ -19,34 +15,15 @@ vi.mock("@sourcebot/shared", async (importOriginal) => ({ createLogger: vi.fn(() => lifecycleLogger), })); -vi.mock("fs", () => ({ - existsSync: fsMocks.existsSync, -})); - -vi.mock("fs/promises", () => ({ - readdir: fsMocks.readdir, - rm: fsMocks.rm, -})); - -import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; - const repoFindUnique = vi.fn(); -const repoDeleteMany = vi.fn(); -const repoIndexingJobUpsert = vi.fn(); -const repoIndexingJobUpdateMany = vi.fn(); const repoUpdate = vi.fn(); const repoUpdateMany = vi.fn(); const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ - repoIndexingJob: { - upsert: repoIndexingJobUpsert, - updateMany: repoIndexingJobUpdateMany, - }, repo: { findUnique: repoFindUnique, update: repoUpdate, - updateMany: repoUpdateMany, }, }), ); @@ -54,21 +31,20 @@ const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => const db = { $transaction: transaction, repo: { - deleteMany: repoDeleteMany, + updateMany: repoUpdateMany, }, } as unknown as PrismaClient; -const workload = createRepoIndexWorkload({ - db, - settings: { - maxRepoIndexingJobConcurrency: 2, - } as never, -}); +const settings = { + maxRepoIndexingJobConcurrency: 2, +} as never; + +const workload = createRepoIndexWorkload({ db, settings }); +const cleanupWorkload = createRepoCleanupWorkload({ db, settings }); const lifecycleContext = { data: { repoId: 42, - type: "INDEX" as const, }, jobId: "job-1", attemptsMade: 0, @@ -83,251 +59,74 @@ const processContext = { trigger: vi.fn(), }; -const eligibleRepo = { - id: 42, - name: "github.com/acme/repo", - cloneUrl: "https://github.com/acme/repo.git", - external_codeHostType: "github", - orgId: 1, - indexedAt: null, - isAutoCleanupDisabled: false, - connections: [], -}; - describe("repoIndexWorkload", () => { beforeEach(() => { vi.clearAllMocks(); - fsMocks.existsSync.mockReturnValue(false); - fsMocks.readdir.mockResolvedValue([]); - fsMocks.rm.mockResolvedValue(undefined); - repoFindUnique.mockResolvedValue(eligibleRepo); - repoDeleteMany.mockResolvedValue({ count: 1 }); - repoIndexingJobUpsert.mockResolvedValue(undefined); - repoIndexingJobUpdateMany.mockResolvedValue({ count: 1 }); + repoFindUnique.mockResolvedValue(null); repoUpdate.mockResolvedValue(undefined); repoUpdateMany.mockResolvedValue({ count: 1 }); }); - test("uses the same repository execution lock for INDEX and CLEANUP", () => { - expect(workload.executionLock).toBeDefined(); - expect( - workload.executionLock?.resource({ repoId: 42, type: "INDEX" }), - ).toBe("sourcebot:lock:repo:42"); + test("shares its repository execution lock with cleanup", () => { + expect(workload.executionLock).toBe(cleanupWorkload.executionLock); expect( - workload.executionLock?.resource({ repoId: 42, type: "CLEANUP" }), + workload.executionLock?.resource({ repoId: 42 }), ).toBe("sourcebot:lock:repo:42"); - expect(workload.executionLock?.durationMs).toBe(60_000); - expect(workload.queueSpec.dedupKey).toBeUndefined(); - expect(workload.onStarted).toBeUndefined(); - expect(workload.onCompleted).toBeTypeOf("function"); - expect(workload.onTerminalFailure).toBeTypeOf("function"); - }); - - test("validates state and marks an eligible job in progress inside process", async () => { - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(repoFindUnique).toHaveBeenCalledWith({ - where: { id: 42 }, - include: { - connections: { - include: { - connection: true, - }, - }, - }, - }); - expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - }, - create: { - id: "job-1", - repoId: 42, - type: "CLEANUP", - status: "IN_PROGRESS", - }, - }); - expect(repoUpdate).toHaveBeenCalledWith({ - where: { - id: 42, - }, - data: { - latestIndexingJobId: "job-1", - latestIndexingJobStatus: "IN_PROGRESS", - }, - }); - expect(repoDeleteMany).toHaveBeenCalledWith({ - where: { - id: 42, - isAutoCleanupDisabled: false, - connections: { - none: {}, - }, - }, - }); - }); - - test("cleanup removes only shards belonging to the exact repository id", async () => { - fsMocks.readdir.mockResolvedValue([ - "1_42_v16.00000.zoekt", - "1_420_v16.00000.zoekt", - ]); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(fsMocks.rm).toHaveBeenCalledWith( - expect.stringContaining("1_42_v16.00000.zoekt"), - { force: true }, - ); - expect(fsMocks.rm).not.toHaveBeenCalledWith( - expect.stringContaining("1_420_v16.00000.zoekt"), - expect.anything(), - ); - }); - - test("skips an INDEX job when the repository no longer exists", async () => { - repoFindUnique.mockResolvedValue(null); - - await workload.process(processContext); - - expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); - expect(repoDeleteMany).not.toHaveBeenCalled(); - expect(fsMocks.readdir).not.toHaveBeenCalled(); - expect(lifecycleLogger.debug).toHaveBeenCalledWith( - "Skipping INDEX job for repo 42: repository no longer exists", - ); - }); - - test("finishes orphaned filesystem cleanup when a CLEANUP retry finds no repo", async () => { - repoFindUnique.mockResolvedValue(null); - fsMocks.existsSync.mockReturnValue(true); - fsMocks.readdir.mockResolvedValue([ - "1_42_v16.00000.zoekt", - "1_99_v16.00000.zoekt", - ]); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); - expect(fsMocks.rm).toHaveBeenCalledWith( - expect.stringMatching(/repos\/42$/), - { recursive: true, force: true }, + expect(cleanupWorkload.executionLock?.resource({ repoId: 42 })).toBe( + "sourcebot:lock:repo:42", ); - expect(fsMocks.rm).toHaveBeenCalledWith( - expect.stringContaining("1_42_v16.00000.zoekt"), - { force: true }, - ); - expect(fsMocks.rm).not.toHaveBeenCalledWith( - expect.stringContaining("1_99_v16.00000.zoekt"), - expect.anything(), - ); - }); - - test.each([ - { - name: "automatic cleanup is disabled", - repo: { ...eligibleRepo, isAutoCleanupDisabled: true }, - reason: "automatic cleanup is disabled", - }, - { - name: "the repository was reattached", - repo: { ...eligibleRepo, connections: [{}] }, - reason: "repository has been reattached to a connection", - }, - ])("skips CLEANUP when $name", async ({ repo, reason }) => { - repoFindUnique.mockResolvedValue(repo); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, + expect(workload.executionLock?.durationMs).toBe(60_000); + expect(workload.queueSpec.name).toBe("repo-index"); + expect(cleanupWorkload.queueSpec.name).toBe("repo-cleanup"); + expect(workload.queueSpec.deduplication?.({ repoId: 42 })).toEqual({ + id: "repo:42", + keepLastIfActive: true, }); - - expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); - expect(repoDeleteMany).not.toHaveBeenCalled(); - expect(fsMocks.readdir).not.toHaveBeenCalled(); - expect(lifecycleLogger.debug).toHaveBeenCalledWith( - `Skipping CLEANUP job for repo 42: ${reason}`, - ); - }); - - test("revalidates cleanup eligibility when atomically deleting the repo", async () => { - repoDeleteMany.mockResolvedValue({ count: 0 }); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, + expect( + cleanupWorkload.queueSpec.deduplication?.({ repoId: 42 }), + ).toEqual({ + id: "repo:42", + keepLastIfActive: true, }); - - expect(repoIndexingJobUpsert).toHaveBeenCalled(); - expect(repoDeleteMany).toHaveBeenCalled(); - expect(fsMocks.readdir).not.toHaveBeenCalled(); - expect(lifecycleLogger.debug).toHaveBeenCalledWith( - "Skipping CLEANUP job for repo 42: repository is no longer eligible for cleanup", - ); }); - test("marks a completed job and fences the repository summary by job id", async () => { + test("records the first successful indexing job terminal state", async () => { await workload.onCompleted?.(lifecycleContext, undefined); - expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); expect(repoUpdateMany).toHaveBeenCalledWith({ where: { id: 42, - latestIndexingJobId: "job-1", + firstIndexingJobFinishedAt: null, }, data: { - latestIndexingJobStatus: "COMPLETED", + firstIndexingJobFinishedAt: expect.any(Date), }, }); }); - test("marks a terminal failure and fences the repository summary by job id", async () => { + test("records the first failed indexing job terminal state", async () => { await workload.onTerminalFailure?.( lifecycleContext, - new Error("Unable to clone repository"), + new Error("indexing failed"), ); - expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Unable to clone repository", - }, - }); expect(repoUpdateMany).toHaveBeenCalledWith({ where: { id: 42, - latestIndexingJobId: "job-1", + firstIndexingJobFinishedAt: null, }, data: { - latestIndexingJobStatus: "FAILED", + firstIndexingJobFinishedAt: expect.any(Date), }, }); }); + + test("skips an INDEX job when the repository no longer exists", async () => { + await workload.process(processContext); + + expect(repoUpdate).not.toHaveBeenCalled(); + expect(lifecycleLogger.debug).toHaveBeenCalledWith( + "Skipping INDEX job for repo 42: repository no longer exists", + ); + }); }); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 7633e5a4c..2a15d90aa 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -1,18 +1,17 @@ -import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; -import { createLogger, getRepoPath, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { PrismaClient } from "@sourcebot/db"; +import { createLogger, getRepoPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; +import { rm } from 'fs/promises'; import micromatch from 'micromatch'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; import { captureEvent } from './posthog.js'; +import { REPOSITORY_EXECUTION_LOCK } from "./repoLock.js"; import { RepoWithConnections, Settings, Workload } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, measure } from './utils.js'; +import { getAuthCredentialsForRepo, measure } from './utils.js'; import { cleanupTempShards, indexGitRepository } from './zoekt.js'; const LOG_TAG = 'repo-index-workload'; const logger = createLogger(LOG_TAG); -const REPO_INDEX_LOCK_DURATION_MS = 60_000; interface Props { db: PrismaClient; @@ -25,146 +24,88 @@ export const createRepoIndexWorkload = ({ }: Props): Workload<'repo-index'> => ({ queueSpec: REPO_INDEX_QUEUE, concurrency: settings.maxRepoIndexingJobConcurrency, - // Indexing and cleanup share this lock because both operate on the same - // repository path and search index. - executionLock: { - resource: ({ repoId }) => `sourcebot:lock:repo:${repoId}`, - durationMs: REPO_INDEX_LOCK_DURATION_MS, - }, + executionLock: REPOSITORY_EXECUTION_LOCK, process: async ({ data, jobId, signal }) => { signal.throwIfAborted(); + const start = await prepareRepoIndexJob({ db, repoId: data.repoId, - type: data.type, jobId, }); if (start.action === "skip") { logger.debug( - `Skipping ${data.type} job for repo ${data.repoId}: ${start.reason}`, + `Skipping INDEX job for repo ${data.repoId}: ${start.reason}`, ); - - if (data.type === "CLEANUP" && start.repoMissing) { - signal.throwIfAborted(); - await cleanupOrphanedRepoResourcesForRepoId(data.repoId); - } return; } signal.throwIfAborted(); const { repo } = start; - logger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); - - if (data.type === "CLEANUP") { - signal.throwIfAborted(); - const { count } = await db.repo.deleteMany({ - where: { - id: repo.id, - isAutoCleanupDisabled: false, - connections: { - none: {}, - }, - }, - }); + logger.debug(`Running INDEX job for repo ${repo.name} (id: ${repo.id})`); - if (count === 0) { - logger.debug( - `Skipping CLEANUP job for repo ${repo.id}: repository is no longer eligible for cleanup`, - ); - return; - } - - signal.throwIfAborted(); - await cleanupRepository(repo); - } else { - const isFirstIndex = repo.indexedAt === null; - const revisions = await indexRepository(db, settings, repo, signal); - signal.throwIfAborted(); - const { path: repoPath } = getRepoPath(repo); - const isEmpty = await isRepoEmpty({ path: repoPath }); - const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ - path: repoPath, - refName: 'HEAD', - }); - const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); - const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); - const currentRepo = await db.repo.findUniqueOrThrow({ - where: { id: repo.id }, - select: { metadata: true }, - }); + const isFirstIndex = repo.indexedAt === null; + const revisions = await indexRepository(db, settings, repo, signal); + signal.throwIfAborted(); + const { path: repoPath } = getRepoPath(repo); + const isEmpty = await isRepoEmpty({ path: repoPath }); + const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ + path: repoPath, + refName: 'HEAD', + }); + const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); + const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); + const currentRepo = await db.repo.findUniqueOrThrow({ + where: { id: repo.id }, + select: { metadata: true }, + }); - signal.throwIfAborted(); - await db.repo.update({ - where: { id: repo.id }, - data: { - indexedAt: new Date(), - indexedCommitHash: commitHash, - pushedAt, - metadata: { - ...(currentRepo.metadata as RepoMetadata), - indexedRevisions: revisions, - } satisfies RepoMetadata, - defaultBranch, - }, - }); + signal.throwIfAborted(); + await db.repo.update({ + where: { id: repo.id }, + data: { + indexedAt: new Date(), + indexedCommitHash: commitHash, + pushedAt, + metadata: { + ...(currentRepo.metadata as RepoMetadata), + indexedRevisions: revisions, + } satisfies RepoMetadata, + defaultBranch, + }, + }); - if (isFirstIndex) { - captureEvent('backend_repo_first_indexed', { - repoId: repo.id, - type: repo.external_codeHostType, - }); - } + if (isFirstIndex) { + captureEvent('backend_repo_first_indexed', { + repoId: repo.id, + type: repo.external_codeHostType, + }); } }, - onCompleted: async ({ data: { repoId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.repoIndexingJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); - await tx.repo.updateMany({ - where: { - id: repoId, - latestIndexingJobId: jobId, - }, - data: { - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - }, - }); - }); + onCompleted: async ({ data }) => { + await markFirstIndexingJobFinished(db, data); }, - onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { - await db.$transaction(async (tx) => { - await tx.repoIndexingJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); - await tx.repo.updateMany({ - where: { - id: repoId, - latestIndexingJobId: jobId, - }, - data: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - }, - }); - }); + onTerminalFailure: async ({ data }) => { + await markFirstIndexingJobFinished(db, data); }, }); +const markFirstIndexingJobFinished = async ( + db: PrismaClient, + data: { repoId: number }, +) => { + await db.repo.updateMany({ + where: { + id: data.repoId, + firstIndexingJobFinishedAt: null, + }, + data: { + firstIndexingJobFinishedAt: new Date(), + }, + }); +}; + type RepoIndexStartDecision = | { action: "run"; @@ -173,18 +114,15 @@ type RepoIndexStartDecision = | { action: "skip"; reason: string; - repoMissing: boolean; }; const prepareRepoIndexJob = async ({ db, repoId, - type, jobId, }: { db: PrismaClient; repoId: number; - type: "INDEX" | "CLEANUP"; jobId: string; }): Promise => db.$transaction(async (tx) => { @@ -203,49 +141,15 @@ const prepareRepoIndexJob = async ({ return { action: "skip", reason: "repository no longer exists", - repoMissing: true, }; } - if (type === "CLEANUP" && repo.isAutoCleanupDisabled) { - return { - action: "skip", - reason: "automatic cleanup is disabled", - repoMissing: false, - }; - } - - if (type === "CLEANUP" && repo.connections.length > 0) { - return { - action: "skip", - reason: "repository has been reattached to a connection", - repoMissing: false, - }; - } - - await tx.repoIndexingJob.upsert({ - where: { - id: jobId, - }, - update: { - status: RepoIndexingJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - repoId, - type: RepoIndexingJobType[type], - status: RepoIndexingJobStatus.IN_PROGRESS, - }, - }); await tx.repo.update({ where: { id: repoId, }, data: { latestIndexingJobId: jobId, - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, }, }); @@ -445,106 +349,3 @@ const indexRepository = async ( return revisions; }; - -const cleanupRepository = async (repo: Repo) => { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.debug(`Deleting repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => getRepoIdFromShardFileName(file) === repo.id); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting shard file ${filePath}`); - await rm(filePath, { force: true }); - } -}; - -const cleanupOrphanedRepoResourcesForRepoId = async ( - repoId: number, -) => { - const repoPath = `${REPOS_CACHE_DIR}/${repoId}`; - if (existsSync(repoPath)) { - logger.debug(`Deleting orphaned repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - if (!existsSync(INDEX_CACHE_DIR)) { - return; - } - - const shardFiles = (await readdir(INDEX_CACHE_DIR)).filter( - (file) => getRepoIdFromShardFileName(file) === repoId, - ); - for (const file of shardFiles) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting orphaned shard file ${filePath}`); - await rm(filePath, { force: true }); - } -}; - -// Scans the repos and index directories on disk and removes any entries -// that have no corresponding Repo record in the database. This handles -// edge cases where the DB and disk resources are out of sync. -export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { - // --- Repo directories --- - // Dirs are named by repoId: DATA_CACHE_DIR/repos// - if (existsSync(REPOS_CACHE_DIR)) { - const entries = await readdir(REPOS_CACHE_DIR); - const repoIdToPath = new Map(); - for (const entry of entries) { - const repoPath = `${REPOS_CACHE_DIR}/${entry}`; - const repoId = getRepoIdFromPath(repoPath); - if (repoId !== undefined) { - repoIdToPath.set(repoId, repoPath); - } - } - - if (repoIdToPath.size > 0) { - const existingRepos = await db.repo.findMany({ - where: { id: { in: [...repoIdToPath.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, repoPath] of repoIdToPath) { - if (!existingIds.has(repoId)) { - logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - } - } - } - - // --- Index shards --- - // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt - if (existsSync(INDEX_CACHE_DIR)) { - const entries = await readdir(INDEX_CACHE_DIR); - const repoIdToShards = new Map(); - for (const entry of entries) { - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - const shards = repoIdToShards.get(repoId) ?? []; - shards.push(entry); - repoIdToShards.set(repoId, shards); - } - } - - if (repoIdToShards.size > 0) { - const existingRepos = await db.repo.findMany({ - where: { id: { in: [...repoIdToShards.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, shards] of repoIdToShards) { - if (!existingIds.has(repoId)) { - for (const entry of shards) { - const shardPath = `${INDEX_CACHE_DIR}/${entry}`; - logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); - await rm(shardPath, { force: true }); - } - } - } - } - } -}; diff --git a/packages/backend/src/repoLock.ts b/packages/backend/src/repoLock.ts new file mode 100644 index 000000000..c55c30585 --- /dev/null +++ b/packages/backend/src/repoLock.ts @@ -0,0 +1,5 @@ +export const REPOSITORY_EXECUTION_LOCK = { + resource: ({ repoId }: { repoId: number }) => + `sourcebot:lock:repo:${repoId}`, + durationMs: 60_000, +}; diff --git a/packages/backend/src/repositoryDiscoveryIssueContext.test.ts b/packages/backend/src/repositoryDiscoveryIssueContext.test.ts new file mode 100644 index 000000000..6db8ff351 --- /dev/null +++ b/packages/backend/src/repositoryDiscoveryIssueContext.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from "vitest"; +import { + collectRepositoryDiscoveryIssues, + reportRepositoryDiscoveryIssue, +} from "./repositoryDiscoveryIssueContext.js"; + +const repositoryIssue = (repository: string) => ({ + code: "NOT_FOUND_OR_INACCESSIBLE" as const, + effect: "TARGET_SKIPPED" as const, + subject: { + kind: "repository" as const, + value: repository, + }, + message: `Repository ${repository} was not found or is inaccessible.`, +}); + +describe("repository discovery issue context", () => { + test("returns the routine value and collected issues", async () => { + const issue = repositoryIssue("sourcebot-dev/legacy"); + + await expect( + collectRepositoryDiscoveryIssues(async () => { + reportRepositoryDiscoveryIssue(issue); + return ["repository"]; + }), + ).resolves.toEqual({ + value: ["repository"], + issues: [issue], + }); + }); + + test("does nothing when reporting outside a collection context", () => { + expect(() => + reportRepositoryDiscoveryIssue( + repositoryIssue("sourcebot-dev/legacy"), + ) + ).not.toThrow(); + }); + + test("deduplicates identical issues", async () => { + const issue = repositoryIssue("sourcebot-dev/legacy"); + + await expect( + collectRepositoryDiscoveryIssues(() => { + reportRepositoryDiscoveryIssue(issue); + reportRepositoryDiscoveryIssue(issue); + }), + ).resolves.toEqual({ + value: undefined, + issues: [issue], + }); + }); + + test("keeps concurrent collection contexts isolated", async () => { + let releaseFirst: () => void = () => {}; + const firstCanFinish = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const first = collectRepositoryDiscoveryIssues(async () => { + reportRepositoryDiscoveryIssue(repositoryIssue("org/first")); + await firstCanFinish; + reportRepositoryDiscoveryIssue( + repositoryIssue("org/first-after-await"), + ); + }); + const second = collectRepositoryDiscoveryIssues(async () => { + reportRepositoryDiscoveryIssue(repositoryIssue("org/second")); + releaseFirst(); + }); + + await expect(Promise.all([first, second])).resolves.toEqual([ + { + value: undefined, + issues: [ + repositoryIssue("org/first"), + repositoryIssue("org/first-after-await"), + ], + }, + { + value: undefined, + issues: [repositoryIssue("org/second")], + }, + ]); + }); + + test("does not retain a context after the routine rejects", async () => { + await expect( + collectRepositoryDiscoveryIssues(async () => { + throw new Error("Discovery failed"); + }), + ).rejects.toThrow("Discovery failed"); + + reportRepositoryDiscoveryIssue( + repositoryIssue("sourcebot-dev/legacy"), + ); + await expect( + collectRepositoryDiscoveryIssues(async () => {}), + ).resolves.toEqual({ value: undefined, issues: [] }); + }); +}); diff --git a/packages/backend/src/repositoryDiscoveryIssueContext.ts b/packages/backend/src/repositoryDiscoveryIssueContext.ts new file mode 100644 index 000000000..983cd4082 --- /dev/null +++ b/packages/backend/src/repositoryDiscoveryIssueContext.ts @@ -0,0 +1,53 @@ +import { + repositoryDiscoveryIssueSchema, + type RepositoryDiscoveryIssue, +} from "@sourcebot/shared"; +import { AsyncLocalStorage } from "node:async_hooks"; + +interface RepositoryDiscoveryIssueContext { + issuesByKey: Map; +} + +export interface RepositoryDiscoveryIssueCollection { + value: T; + issues: RepositoryDiscoveryIssue[]; +} + +const repositoryDiscoveryIssueStorage = + new AsyncLocalStorage(); + +const getIssueKey = (issue: RepositoryDiscoveryIssue): string => + JSON.stringify([ + issue.code, + issue.effect, + issue.subject?.kind ?? null, + issue.subject?.value ?? null, + issue.message, + ]); + +export const collectRepositoryDiscoveryIssues = async ( + routine: () => T | Promise, +): Promise> => { + const context: RepositoryDiscoveryIssueContext = { + issuesByKey: new Map(), + }; + + const value = await repositoryDiscoveryIssueStorage.run(context, routine); + + return { + value, + issues: [...context.issuesByKey.values()], + }; +}; + +export const reportRepositoryDiscoveryIssue = ( + issue: RepositoryDiscoveryIssue, +): void => { + const context = repositoryDiscoveryIssueStorage.getStore(); + if (!context) { + return; + } + + const parsedIssue = repositoryDiscoveryIssueSchema.parse(issue); + context.issuesByKey.set(getIssueKey(parsedIssue), parsedIssue); +}; diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 52ba10096..895a858b3 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -11,6 +11,7 @@ import { JobEnqueueOptions, QueueName, QueueSpec, + ResultOf, Schedule, } from "@sourcebot/shared"; import type { Queue } from "bullmq"; @@ -69,7 +70,7 @@ export interface WorkloadExecutionLock { durationMs: number; } -export interface Workload { +export interface Workload { queueSpec: QueueSpec; concurrency: number; executionLock?: WorkloadExecutionLock; @@ -79,11 +80,11 @@ export interface Workload { options?: JobEnqueueOptions; }; rateLimit?: { max: number; per: string }; - process(ctx: ProcessContext): Promise; + process(ctx: ProcessContext): Promise>; onStarted?(ctx: JobLifecycleContext): Promise; onCompleted?( ctx: JobLifecycleContext, - result: TResult, + result: ResultOf, ): Promise; onTerminalFailure?( ctx: JobLifecycleContext, diff --git a/packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql b/packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql new file mode 100644 index 000000000..f735104ea --- /dev/null +++ b/packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "latestIndexingJobStatus"; + +-- DropTable +DROP TABLE "RepoIndexingJob"; + +-- DropEnum +DROP TYPE "RepoIndexingJobStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingJobType"; diff --git a/packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql b/packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql new file mode 100644 index 000000000..3424e7740 --- /dev/null +++ b/packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql @@ -0,0 +1,5 @@ +-- DropTable +DROP TABLE "RepoPermissionSyncJob"; + +-- DropEnum +DROP TYPE "RepoPermissionSyncJobStatus"; diff --git a/packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql b/packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql new file mode 100644 index 000000000..debcecf60 --- /dev/null +++ b/packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql @@ -0,0 +1,5 @@ +-- DropTable +DROP TABLE "AccountPermissionSyncJob"; + +-- DropEnum +DROP TYPE "AccountPermissionSyncJobStatus"; diff --git a/packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql b/packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql new file mode 100644 index 000000000..c89ae953a --- /dev/null +++ b/packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "firstIndexingJobFinishedAt" TIMESTAMP(3); + +-- Existing successful indexes have already reached a terminal state. +UPDATE "Repo" +SET "firstIndexingJobFinishedAt" = "indexedAt" +WHERE "indexedAt" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "Repo_orgId_firstIndexingJobFinishedAt_idx" +ON "Repo"("orgId", "firstIndexingJobFinishedAt"); diff --git a/packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql b/packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql new file mode 100644 index 000000000..ef37c453c --- /dev/null +++ b/packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql @@ -0,0 +1,5 @@ +-- DropTable +DROP TABLE "ConnectionSyncJob"; + +-- DropEnum +DROP TYPE "ConnectionSyncJobStatus"; diff --git a/packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql b/packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql new file mode 100644 index 000000000..637c978a5 --- /dev/null +++ b/packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "firstSyncJobFinishedAt" TIMESTAMP(3); + +-- Existing successful syncs have already reached a terminal state. +UPDATE "Connection" +SET "firstSyncJobFinishedAt" = "syncedAt" +WHERE "syncedAt" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "Connection_orgId_firstSyncJobFinishedAt_idx" +ON "Connection"("orgId", "firstSyncJobFinishedAt"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 7ebf93f14..5632d10ed 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -72,16 +72,14 @@ model Repo { permittedAccounts AccountToRepoPermission[] scopedAccessTokens ScopedAccessTokenToRepo[] - permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest repo state. - jobs RepoIndexingJob[] - indexedAt DateTime? /// When the repo was last indexed successfully. - indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). - latestIndexingJobId String? /// The job allowed to publish the latest indexing status. - latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. - pushedAt DateTime? /// The timestamp of the most recent commit across all branches. + indexedAt DateTime? /// When the repo was last indexed successfully. + indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). + latestIndexingJobId String? /// The latest indexing job in the workload queue. + firstIndexingJobFinishedAt DateTime? /// When the first indexing job reached a terminal state, whether successful or failed. + pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service external_codeHostType CodeHostType /// The type of the external service (e.g., github, gitlab, etc.) @@ -95,58 +93,10 @@ model Repo { @@unique([external_id, external_codeHostUrl, orgId]) @@index([orgId]) + @@index([orgId, firstIndexingJobFinishedAt]) @@index([indexedAt]) } -enum RepoIndexingJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -enum RepoIndexingJobType { - INDEX - CLEANUP -} - -model RepoIndexingJob { - id String @id @default(cuid()) - type RepoIndexingJobType - status RepoIndexingJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts - - errorMessage String? - - repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) - repoId Int - - @@index([repoId, type, status]) -} - -enum RepoPermissionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model RepoPermissionSyncJob { - id String @id @default(cuid()) - status RepoPermissionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - errorMessage String? - - repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) - repoId Int -} - model SearchContext { id Int @id @default(autoincrement()) @@ -184,10 +134,10 @@ model Connection { // The type of connection (e.g., github, gitlab, etc.) connectionType ConnectionType - syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? latestSyncJobId String? /// The most recently started connection sync job. + firstSyncJobFinishedAt DateTime? /// When the first sync job reached a terminal state, whether successful or failed. /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -210,27 +160,7 @@ model Connection { orgId Int @@unique([name, orgId]) -} - -enum ConnectionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model ConnectionSyncJob { - id String @id @default(cuid()) - status ConnectionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - warningMessages String[] - errorMessage String? - - connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) - connectionId Int + @@index([orgId, firstSyncJobFinishedAt]) } model RepoToConnection { @@ -574,26 +504,6 @@ model User { } -enum AccountPermissionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model AccountPermissionSyncJob { - id String @id @default(cuid()) - status AccountPermissionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - errorMessage String? - - account Account @relation(fields: [accountId], references: [id], onDelete: Cascade) - accountId String -} - enum PermissionSyncSource { ACCOUNT_DRIVEN REPO_DRIVEN @@ -646,7 +556,6 @@ model Account { /// List of repos that this account has access to. accessibleRepos AccountToRepoPermission[] - permissionSyncJobs AccountPermissionSyncJob[] permissionSyncedAt DateTime? latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest account state. diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 609bcdcc7..a5109ebe1 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,8 +2,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; -const NUM_INDEXING_JOBS_PER_REPO = 10000; -const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { run: async (prisma: PrismaClient) => { @@ -36,11 +34,8 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); - const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; - const indexingJobTypes = ['INDEX', 'CLEANUP'] as const; - for (let i = 0; i < NUM_REPOS; i++) { - const repo = await prisma.repo.create({ + await prisma.repo.create({ data: { name: `test-repo-${i}`, isFork: false, @@ -60,34 +55,8 @@ export const injectRepoData: Script = { } }); - for (let j = 0; j < NUM_PERMISSION_JOBS_PER_REPO; j++) { - const status = statuses[Math.floor(Math.random() * statuses.length)]; - await prisma.repoPermissionSyncJob.create({ - data: { - repoId: repo.id, - status, - completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, - errorMessage: status === 'FAILED' ? 'Mock error message' : null - } - }); - } - - for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) { - const status = statuses[Math.floor(Math.random() * statuses.length)]; - const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)]; - await prisma.repoIndexingJob.create({ - data: { - repoId: repo.id, - type, - status, - completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, - errorMessage: status === 'FAILED' ? 'Mock indexing error' : null, - metadata: {} - } - }); - } } - console.log(`Created ${NUM_REPOS} repos with associated jobs.`); + console.log(`Created ${NUM_REPOS} repos.`); } -}; \ No newline at end of file +}; diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index 8cf08309f..72ed1918e 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -3,6 +3,8 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ add: vi.fn(async () => ({ id: "job-1" })), + getJob: vi.fn(), + listJobs: vi.fn(), upsertJobScheduler: vi.fn(async () => ({ id: "scheduled-job" })), getJobScheduler: vi.fn(), getJobSchedulers: vi.fn(async () => [ @@ -15,6 +17,8 @@ const mocks = vi.hoisted(() => ({ vi.mock("bullmq", () => ({ Queue: class { add = mocks.add; + getJob = mocks.getJob; + getJobs = mocks.listJobs; upsertJobScheduler = mocks.upsertJobScheduler; getJobScheduler = mocks.getJobScheduler; getJobSchedulers = mocks.getJobSchedulers; @@ -28,16 +32,124 @@ vi.mock("./jobLogger.js", () => ({ })); import { BullMQClient } from "./bullmqClient.js"; -import { CONNECTION_QUEUE } from "./queue.js"; +import { CONNECTION_QUEUE, type QueueSpec } from "./queue.js"; describe("BullMQClient", () => { beforeEach(() => { vi.restoreAllMocks(); vi.clearAllMocks(); vi.spyOn(Date, "now").mockReturnValue(1_000_000); + mocks.getJob.mockResolvedValue(undefined); + mocks.listJobs.mockResolvedValue([]); mocks.getJobScheduler.mockResolvedValue(undefined); }); + test("gets jobs by id and preserves missing jobs", async () => { + mocks.getJob.mockImplementation(async (jobId: string) => { + if (jobId === "job-1") { + return { + id: jobId, + data: { connectionId: 1 }, + failedReason: "", + returnvalue: null, + getState: vi.fn(async () => "active"), + }; + } + if (jobId === "job-2") { + return { + id: jobId, + data: { connectionId: 2 }, + failedReason: "", + returnvalue: { outcome: "SUCCESS" }, + getState: vi.fn(async () => "completed"), + }; + } + return undefined; + }); + const client = new BullMQClient({} as Redis); + + await expect( + client.getJobs(CONNECTION_QUEUE, ["job-1", "missing", "job-2"]), + ).resolves.toEqual(new Map([ + ["job-1", { + id: "job-1", + data: { connectionId: 1 }, + status: "IN_PROGRESS", + errorMessage: null, + result: null, + }], + ["missing", null], + ["job-2", { + id: "job-2", + data: { connectionId: 2 }, + status: "COMPLETED", + errorMessage: null, + result: { outcome: "SUCCESS" }, + }], + ])); + }); + + test("returns null for an unrecognized legacy connection result", async () => { + mocks.getJob.mockResolvedValue({ + id: "job-1", + data: { connectionId: 1 }, + failedReason: "", + returnvalue: { + reposToCleanup: [], + reposToIndex: [], + }, + getState: vi.fn(async () => "completed"), + }); + const client = new BullMQClient({} as Redis); + + await expect( + client.getJob(CONNECTION_QUEUE, "job-1"), + ).resolves.toEqual({ + id: "job-1", + data: { connectionId: 1 }, + status: "COMPLETED", + errorMessage: null, + result: null, + }); + }); + + test("deduplicates job ids when getting jobs", async () => { + mocks.getJob.mockResolvedValue({ + id: "job-1", + data: { connectionId: 1 }, + failedReason: "", + returnvalue: null, + getState: vi.fn(async () => "waiting"), + }); + const client = new BullMQClient({} as Redis); + + const jobs = await client.getJobs(CONNECTION_QUEUE, [ + "job-1", + "job-1", + ]); + + expect(jobs).toHaveLength(1); + expect(mocks.getJob).toHaveBeenCalledTimes(1); + }); + + test("lists failed job ids", async () => { + mocks.listJobs.mockResolvedValue([ + { id: "failed-1" }, + { id: "failed-2" }, + ]); + const client = new BullMQClient({} as Redis); + + await expect( + client.getFailedJobIds(CONNECTION_QUEUE), + ).resolves.toEqual(["failed-1", "failed-2"]); + expect(mocks.listJobs).toHaveBeenCalledWith( + ["failed"], + 0, + -1, + true, + ); + }); + test("includes workload data in scheduled jobs", async () => { const client = new BullMQClient({} as Redis); const data = { connectionId: 42 }; @@ -56,7 +168,7 @@ describe("BullMQClient", () => { name: "connection-sync", data, opts: { - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, @@ -84,7 +196,7 @@ describe("BullMQClient", () => { { connectionId: 42 }, expect.objectContaining({ priority: 1, - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, @@ -94,6 +206,44 @@ describe("BullMQClient", () => { ); }); + test("adds simple deduplication to immediate jobs", async () => { + const client = new BullMQClient({} as Redis); + + await client.enqueue(CONNECTION_QUEUE, { connectionId: 42 }); + + expect(mocks.add).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + expect.objectContaining({ + deduplication: { id: "connection:42" }, + }), + ); + }); + + test("passes keepLastIfActive to BullMQ deduplication", async () => { + const queueSpec = { + ...CONNECTION_QUEUE, + deduplication: ({ connectionId }) => ({ + id: `connection:${connectionId}`, + keepLastIfActive: true, + }), + } satisfies QueueSpec<"connection-sync">; + const client = new BullMQClient({} as Redis); + + await client.enqueue(queueSpec, { connectionId: 42 }); + + expect(mocks.add).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + expect.objectContaining({ + deduplication: { + id: "connection:42", + keepLastIfActive: true, + }, + }), + ); + }); + test("adds enqueue priority to scheduled jobs", async () => { const client = new BullMQClient({} as Redis); @@ -125,7 +275,7 @@ describe("BullMQClient", () => { data: { connectionId: 42 }, opts: { priority: 10, - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, @@ -162,7 +312,7 @@ describe("BullMQClient", () => { template: { data: { connectionId: 42 }, opts: { - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index b92f006bb..0562543ad 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -7,6 +7,7 @@ import type { JobEnqueueOptions, QueueName, QueueSpec, + ResultOf, } from "./queue.js"; import { scheduleToMs } from "./schedule.js"; import type { Schedule } from "./schedule.js"; @@ -24,14 +25,15 @@ export interface WorkloadJob { data: DataOf; status: WorkloadJobStatus; errorMessage: string | null; + result: ResultOf | null; } type WorkloadQueue = Queue< DataOf, - unknown, + ResultOf, string, DataOf, - unknown, + ResultOf, string >; @@ -85,14 +87,51 @@ export class BullMQClient { return null; } + const result = (() => { + if (status !== "COMPLETED" || !spec.resultSchema) { + return null; + } + + const parsed = spec.resultSchema.safeParse(job.returnvalue); + return parsed.success ? parsed.data : null; + })(); + return { id: job.id ?? jobId, data: job.data as DataOf, status, errorMessage: status === "FAILED" ? job.failedReason || null : null, + result, }; } + async getJobs( + spec: QueueSpec, + jobIds: readonly string[], + ): Promise | null>> { + const uniqueJobIds = [...new Set(jobIds)]; + const jobs = await Promise.all( + uniqueJobIds.map((jobId) => this.getJob(spec, jobId)), + ); + + return new Map( + uniqueJobIds.map((jobId, index) => [jobId, jobs[index] ?? null]), + ); + } + + async getFailedJobIds( + spec: QueueSpec, + ): Promise { + const jobs = await this.getQueue(spec).getJobs( + ["failed"], + 0, + -1, + true, + ); + + return jobs.flatMap((job) => job.id ? [job.id] : []); + } + async getJobLogs( spec: QueueSpec, jobId: string, @@ -106,13 +145,21 @@ export class BullMQClient { data: DataOf, options: JobEnqueueOptions = {}, ): Promise { - const dedupKey = spec.dedupKey?.(data); + // QueueSpec is distributive so queue names, data, and result schemas stay + // correlated when TName is a union. Re-establish the shared generic here + // before invoking the optional method. + const { deduplication: getDeduplication }: { + deduplication?( + data: DataOf, + ): { id: string; keepLastIfActive?: boolean }; + } = spec; + const deduplication = getDeduplication?.(data); const queue = this.getQueue(spec); const requestedJobId = randomUUID(); const job = await queue.add(spec.name, data, { jobId: requestedJobId, - ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + ...(deduplication ? { deduplication } : {}), ...(options.priority !== undefined ? { priority: options.priority } : {}), diff --git a/packages/shared/src/connectionSync.test.ts b/packages/shared/src/connectionSync.test.ts new file mode 100644 index 000000000..9076d3be8 --- /dev/null +++ b/packages/shared/src/connectionSync.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { connectionSyncResultSchema } from "./connectionSync.js"; + +describe("connectionSyncResultSchema", () => { + test("accepts a successful result", () => { + expect( + connectionSyncResultSchema.parse({ outcome: "SUCCESS" }), + ).toEqual({ outcome: "SUCCESS" }); + }); + + test("accepts a partial success with structured reasons", () => { + const reason = { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "sourcebot-dev/missing-repo", + }, + message: + "Repository sourcebot-dev/missing-repo was not found or is inaccessible.", + }; + + expect( + connectionSyncResultSchema.parse({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }), + ).toEqual({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }); + }); + + test("requires at least one reason for a partial success", () => { + expect( + connectionSyncResultSchema.safeParse({ + outcome: "PARTIAL_SUCCESS", + reasons: [], + }).success, + ).toBe(false); + }); +}); diff --git a/packages/shared/src/connectionSync.ts b/packages/shared/src/connectionSync.ts new file mode 100644 index 000000000..3df66e5bb --- /dev/null +++ b/packages/shared/src/connectionSync.ts @@ -0,0 +1,16 @@ +import { z } from "zod"; +import { repositoryDiscoveryIssueSchema } from "./repositoryDiscovery.js"; + +export const connectionSyncResultSchema = z.discriminatedUnion("outcome", [ + z.object({ + outcome: z.literal("SUCCESS"), + }), + z.object({ + outcome: z.literal("PARTIAL_SUCCESS"), + reasons: z.array(repositoryDiscoveryIssueSchema).min(1), + }), +]); + +export type ConnectionSyncResult = z.infer< + typeof connectionSyncResultSchema +>; diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 2199df140..618e1a857 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -348,8 +348,6 @@ const options = { DEBUG_ENABLE_REACT_SCAN: booleanSchema.default('false'), DEBUG_ENABLE_REACT_GRAB: booleanSchema.default('false'), - SOURCEBOT_DEMO_EXAMPLES_PATH: z.string().optional(), - DISABLE_API_KEY_USAGE_FOR_NON_OWNER_USERS: booleanSchema.default('false'), DISABLE_API_KEY_CREATION_FOR_NON_OWNER_USERS: booleanSchema diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index ff92ac127..73b08c5e0 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -94,9 +94,26 @@ export { compareVersions, } from "./versionUtils.js"; export type { Version } from "./versionUtils.js"; +export { + connectionSyncResultSchema, +} from "./connectionSync.js"; +export type { ConnectionSyncResult } from "./connectionSync.js"; +export { + repositoryDiscoveryIssueCodeSchema, + repositoryDiscoveryIssueEffectSchema, + repositoryDiscoveryIssueSchema, + repositoryDiscoveryIssueSubjectSchema, +} from "./repositoryDiscovery.js"; +export type { + RepositoryDiscoveryIssue, + RepositoryDiscoveryIssueCode, + RepositoryDiscoveryIssueEffect, + RepositoryDiscoveryIssueSubject, +} from "./repositoryDiscovery.js"; export type { QueueName, DataOf, + ResultOf, JobEnqueueOptions, QueueSpec, JobOptions, @@ -108,6 +125,8 @@ export { CONNECTION_QUEUE, DEFAULT_JOB_OPTIONS, JOB_PRIORITIES, + QUEUE_SPECS, + REPO_CLEANUP_QUEUE, REPO_INDEX_QUEUE, REPO_PERMISSION_SYNC_QUEUE, } from "./queue.js"; diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 993fc9b6c..a80dddcac 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,12 +1,29 @@ -import type { KeepJobs } from "bullmq"; +import type { DeduplicationOptions, KeepJobs } from "bullmq"; +import { z, type ZodType } from "zod"; +import { + connectionSyncResultSchema, + type ConnectionSyncResult, +} from "./connectionSync.js"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; -export interface QueueSpec { +interface BaseQueueSpec { name: TName; - dedupKey?(data: DataOf): string; + deduplication?( + data: DataOf, + ): Pick; jobOptions: JobOptions; } +type ResultSchemaSpec = TName extends QueueName + ? [ResultOf] extends [void] + ? { resultSchema?: never } + : { resultSchema: ZodType> } + : never; + +export type QueueSpec = TName extends QueueName + ? BaseQueueSpec & ResultSchemaSpec + : never; + export type JobOptions = { attempts: number; backoff: { @@ -31,10 +48,13 @@ export const JOB_PRIORITIES = { SCHEDULED: 10, } as const; +// BullMQ evaluates age-based cleanup only when another job reaches the same +// terminal state. A lone completed or failed job therefore remains available +// past this age; once a newer same-state job finishes, it replaces the old one. const TWO_WEEKS_IN_SECONDS = 14 * 24 * 60 * 60; export const DEFAULT_JOB_OPTIONS: JobOptions = { - attempts: 4, + attempts: 2, backoff: { type: "exponential", delayMs: 30_000, @@ -48,58 +68,122 @@ export const DEFAULT_JOB_OPTIONS: JobOptions = { }; export type QueueName = keyof QueueRegistry; -export type DataOf = QueueRegistry[TName]; +export type DataOf = QueueRegistry[TName]["data"]; +export type ResultOf = QueueRegistry[TName]["result"]; + +const attachmentPruneResultSchema = z.object({ + pendingClaimed: z.number().int().nonnegative(), + committedClaimed: z.number().int().nonnegative(), + reclaimed: z.number().int().nonnegative(), +}); + +const auditLogPruneResultSchema = z.object({ + deleted: z.number().int().nonnegative(), +}); + +const repoPermissionSyncResultSchema = z.object({ + repoName: z.string().min(1), +}); interface QueueRegistry { - "attachment-prune": Record; - "audit-log-prune": Record; + "attachment-prune": { + data: Record; + result: z.infer; + }; + "audit-log-prune": { + data: Record; + result: z.infer; + }; "connection-sync": { - connectionId: number; + data: { + connectionId: number; + }; + result: ConnectionSyncResult; }; "repo-index": { - repoId: number; - type: "INDEX" | "CLEANUP"; + data: { + repoId: number; + }; + result: void; + }; + "repo-cleanup": { + data: { + repoId: number; + }; + result: void; }; "account-permission-sync": { - accountId: string; + data: { + accountId: string; + }; + result: void; }; "repo-permission-sync": { - repoId: number; + data: { + repoId: number; + }; + result: z.infer; }; } export const ATTACHMENT_PRUNE_QUEUE: QueueSpec<"attachment-prune"> = { name: "attachment-prune", + resultSchema: attachmentPruneResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: () => "global", + deduplication: () => ({ id: "global" }), }; export const AUDIT_LOG_PRUNE_QUEUE: QueueSpec<"audit-log-prune"> = { name: "audit-log-prune", + resultSchema: auditLogPruneResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: () => "global", + deduplication: () => ({ id: "global" }), }; export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { name: "connection-sync", + resultSchema: connectionSyncResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `connection:${data.connectionId}`, + deduplication: (data) => ({ id: `connection:${data.connectionId}` }), }; export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { name: "repo-index", jobOptions: DEFAULT_JOB_OPTIONS, + deduplication: ({ repoId }) => ({ + id: `repo:${repoId}`, + keepLastIfActive: true, + }), }; -export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = - { - name: "account-permission-sync", - jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `account:${data.accountId}`, - }; +export const REPO_CLEANUP_QUEUE: QueueSpec<"repo-cleanup"> = { + name: "repo-cleanup", + jobOptions: DEFAULT_JOB_OPTIONS, + deduplication: ({ repoId }) => ({ + id: `repo:${repoId}`, + keepLastIfActive: true, + }), +}; + +export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = { + name: "account-permission-sync", + jobOptions: DEFAULT_JOB_OPTIONS, + deduplication: (data) => ({ id: `account:${data.accountId}` }), +}; export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { name: "repo-permission-sync", + resultSchema: repoPermissionSyncResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `repo:${data.repoId}`, + deduplication: (data) => ({ id: `repo:${data.repoId}` }), }; + +export const QUEUE_SPECS = { + [ATTACHMENT_PRUNE_QUEUE.name]: ATTACHMENT_PRUNE_QUEUE, + [AUDIT_LOG_PRUNE_QUEUE.name]: AUDIT_LOG_PRUNE_QUEUE, + [CONNECTION_QUEUE.name]: CONNECTION_QUEUE, + [REPO_INDEX_QUEUE.name]: REPO_INDEX_QUEUE, + [REPO_CLEANUP_QUEUE.name]: REPO_CLEANUP_QUEUE, + [ACCOUNT_PERMISSION_SYNC_QUEUE.name]: ACCOUNT_PERMISSION_SYNC_QUEUE, + [REPO_PERMISSION_SYNC_QUEUE.name]: REPO_PERMISSION_SYNC_QUEUE, +} as const satisfies { [TName in QueueName]: QueueSpec }; diff --git a/packages/shared/src/repositoryDiscovery.test.ts b/packages/shared/src/repositoryDiscovery.test.ts new file mode 100644 index 000000000..b0364516d --- /dev/null +++ b/packages/shared/src/repositoryDiscovery.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { repositoryDiscoveryIssueSchema } from "./repositoryDiscovery.js"; + +describe("repositoryDiscoveryIssueSchema", () => { + test("allows an issue without a subject", () => { + expect( + repositoryDiscoveryIssueSchema.parse({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "The provider returned an invalid repository.", + }), + ).toEqual({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "The provider returned an invalid repository.", + }); + }); + + test("accepts an authentication fallback issue", () => { + expect( + repositoryDiscoveryIssueSchema.parse({ + code: "AUTHENTICATION_FALLBACK", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "configuration", + value: "GitHub App installation for sourcebot on github.com", + }, + message: + "No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.", + }), + ).toEqual({ + code: "AUTHENTICATION_FALLBACK", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "configuration", + value: "GitHub App installation for sourcebot on github.com", + }, + message: + "No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.", + }); + }); +}); diff --git a/packages/shared/src/repositoryDiscovery.ts b/packages/shared/src/repositoryDiscovery.ts new file mode 100644 index 000000000..8cded6ddc --- /dev/null +++ b/packages/shared/src/repositoryDiscovery.ts @@ -0,0 +1,55 @@ +import { z } from "zod"; + +export const repositoryDiscoveryIssueCodeSchema = z.enum([ + "NOT_FOUND_OR_INACCESSIBLE", + "INVALID_TARGET", + "UNSUPPORTED_CONFIGURATION", + "INVALID_REPOSITORY_SOURCE", + "ENUMERATION_FAILED", + "INVALID_PROVIDER_RESPONSE", + "AUTHENTICATION_FALLBACK", +]); + +export type RepositoryDiscoveryIssueCode = z.infer< + typeof repositoryDiscoveryIssueCodeSchema +>; + +export const repositoryDiscoveryIssueEffectSchema = z.enum([ + "TARGET_SKIPPED", + "CONFIGURATION_IGNORED", + "DISCOVERY_INCOMPLETE", +]); + +export type RepositoryDiscoveryIssueEffect = z.infer< + typeof repositoryDiscoveryIssueEffectSchema +>; + +export const repositoryDiscoveryIssueSubjectSchema = z.object({ + kind: z.enum([ + "organization", + "group", + "user", + "workspace", + "project", + "repository", + "path", + "url", + "configuration", + ]), + value: z.string().min(1), +}); + +export type RepositoryDiscoveryIssueSubject = z.infer< + typeof repositoryDiscoveryIssueSubjectSchema +>; + +export const repositoryDiscoveryIssueSchema = z.object({ + code: repositoryDiscoveryIssueCodeSchema, + effect: repositoryDiscoveryIssueEffectSchema, + subject: repositoryDiscoveryIssueSubjectSchema.optional(), + message: z.string().min(1), +}); + +export type RepositoryDiscoveryIssue = z.infer< + typeof repositoryDiscoveryIssueSchema +>; diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index c64979173..0483e75f0 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { OrgRole, Prisma } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -220,96 +220,6 @@ export const getRepos = async ({ } satisfies RepositoryQuery)) })); -/** - * Returns a set of aggregated stats about the repos in the org - */ -export const getReposStats = async () => sew(() => - withOptionalAuth(async ({ org, prisma }) => { - const [ - // Total number of repos. - numberOfRepos, - // Number of repos with their first time indexing jobs either - // pending or in progress. - numberOfReposWithFirstTimeIndexingJobsInProgress, - // Number of repos that have been indexed at least once. - numberOfReposWithIndex, - ] = await Promise.all([ - prisma.repo.count({ - where: { - orgId: org.id, - } - }), - prisma.repo.count({ - where: { - orgId: org.id, - indexedAt: null, - jobs: { - some: { - type: RepoIndexingJobType.INDEX, - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - } - }, - }, - } - }), - prisma.repo.count({ - where: { - orgId: org.id, - NOT: { - indexedAt: null, - } - } - }) - ]); - - return { - numberOfRepos, - numberOfReposWithFirstTimeIndexingJobsInProgress, - numberOfReposWithIndex, - }; - }) -) - -export const getConnectionStats = async () => sew(() => - withAuth(async ({ org, prisma }) => { - const [ - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - ] = await Promise.all([ - prisma.connection.count({ - where: { - orgId: org.id, - } - }), - prisma.connection.count({ - where: { - orgId: org.id, - syncedAt: null, - syncJobs: { - some: { - status: { - in: [ - ConnectionSyncJobStatus.PENDING, - ConnectionSyncJobStatus.IN_PROGRESS, - ] - } - } - } - } - }) - ]); - - return { - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - }; - }) -); - export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuth(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx index a0ccd6269..85880f908 100644 --- a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx +++ b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx @@ -2,7 +2,6 @@ import { cookies } from "next/headers"; import { auth } from "@/auth"; import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants"; import { HomeView } from "@/hooks/useHomeView"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; @@ -46,11 +45,9 @@ export async function DefaultSidebar() { if (!isOwner) { return false; } - const connectionStats = await getConnectionStats(); const joinRequests = await getOrgAccountRequests(); - const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0; const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0; - return hasConnectionNotification || hasJoinRequestNotification; + return hasJoinRequestNotification; })(); return ( @@ -132,4 +129,4 @@ const getUserChatHistory = async () => sew(() => visibility: chat.visibility, })) }) -); \ No newline at end of file +); diff --git a/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts b/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts index 64d37f2e6..647f63911 100644 --- a/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts +++ b/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts @@ -9,14 +9,6 @@ export const getRepoInfo = async (repoId: number): Promise { const repo = await prisma.repo.findUnique({ where: { id: repoId }, - include: { - jobs: { - orderBy: { - createdAt: 'desc', - }, - take: 1, - }, - }, }); if (!repo) { @@ -31,4 +23,4 @@ export const getRepoInfo = async (repoId: number): Promise { - try { - return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; - } catch (error) { - console.error('Failed to load demo examples:', error); - return undefined; - } - })() : undefined; - return (
@@ -80,26 +50,11 @@ export async function ChatLandingPage() { isLoginWallEnabled={env.EXPERIMENT_ASK_GH_ENABLED === 'true'} maxImageBytes={env.SOURCEBOT_CHAT_ATTACHMENT_MAX_IMAGE_BYTES} /> - - -
- -
- - {demoExamples && ( - <> -
- -
- - - - )} +
) diff --git a/packages/web/src/app/(app)/chat/components/demoCards.tsx b/packages/web/src/app/(app)/chat/components/demoCards.tsx deleted file mode 100644 index 7c4fe316b..000000000 --- a/packages/web/src/app/(app)/chat/components/demoCards.tsx +++ /dev/null @@ -1,157 +0,0 @@ -'use client'; - -import { useState } from "react"; -import Image from "next/image"; -import { Search, LibraryBigIcon, Code, Info } from "lucide-react"; -import { Badge } from "@/components/ui/badge"; -import { Card } from "@/components/ui/card"; -import { CardContent } from "@/components/ui/card"; -import { DemoExamples, DemoSearchExample, DemoSearchScope } from "@/types"; -import { cn, getCodeHostIcon } from "@/lib/utils"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard"; -import { CodeHostType } from "@sourcebot/db"; - -interface DemoCards { - demoExamples: DemoExamples; -} - -export const DemoCards = ({ - demoExamples, -}: DemoCards) => { - const captureEvent = useCaptureEvent(); - const [selectedFilterSearchScope, setSelectedFilterSearchScope] = useState(null); - - const handleExampleClick = (example: DemoSearchExample) => { - captureEvent('wa_demo_search_example_card_pressed', { - exampleTitle: example.title, - exampleUrl: example.url || '', - }); - - if (example.url) { - window.open(example.url, '_blank'); - } - } - - const getSearchScopeIcon = (searchScope: DemoSearchScope, size: number = 20, isSelected: boolean = false) => { - const sizeClass = size === 12 ? "h-3 w-3" : "h-5 w-5"; - const colorClass = isSelected ? "text-primary-foreground" : "text-muted-foreground"; - - if (searchScope.type === "reposet") { - return ; - } - - if (searchScope.codeHostType) { - const codeHostIcon = getCodeHostIcon(searchScope.codeHostType as CodeHostType); - // When selected, icons need to match the inverted badge colors - // In light mode selected: light icon on dark bg (invert) - // In dark mode selected: dark icon on light bg (no invert, override dark:invert) - const selectedIconClass = isSelected - ? "invert dark:invert-0" - : codeHostIcon.className; - - return ( - {`${searchScope.codeHostType} - ); - } - - return ; - } - - return ( -
- {/* Example Searches Row */} -
-
-
- -

Community Ask Results

-
-
- - {/* Search Scope Filter */} -
-
-
- -
- -
-
-
- Search Scope: -
- { - setSelectedFilterSearchScope(null); - }} - > - All - - {demoExamples.searchScopes.map((searchScope) => ( - { - setSelectedFilterSearchScope(searchScope.id); - }} - > - {getSearchScopeIcon(searchScope, 12, selectedFilterSearchScope === searchScope.id)} - {searchScope.displayName} - - ))} -
- -
- {demoExamples.searchExamples - .filter((example) => { - if (selectedFilterSearchScope === null) return true; - return example.searchScopes.includes(selectedFilterSearchScope); - }) - .map((example) => { - const searchScopes = demoExamples.searchScopes.filter((searchScope) => example.searchScopes.includes(searchScope.id)) - return ( - handleExampleClick(example)} - > - -
-
- {searchScopes.map((searchScope) => ( - - {getSearchScopeIcon(searchScope, 12)} - {searchScope.displayName} - - ))} -
-
-

- {example.title} -

-

- {example.description} -

-
-
- - - ) - })} -
-
-
- ); -}; \ No newline at end of file diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx new file mode 100644 index 000000000..dc0a0ee7c --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx @@ -0,0 +1,78 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { useEffect } from "react"; +import { Node } from "slate"; +import { ReactEditor, useSlate } from "slate-react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; +import type { CustomEditor } from "@/features/chat/types"; +import { ExampleQuestionBadges } from "./exampleQuestionBadges"; +import type { ExampleQuestion } from "./exampleQuestions"; + +const questions = [ + { + label: "Entry points", + question: "Find the main entry points across the indexed repositories.", + icon: "search", + }, + { + label: "Configuration", + question: "Where is configuration defined and loaded?", + icon: "settings", + }, + { + label: "Testing", + question: "Find examples of how this code is tested.", + icon: "flask", + }, +] as const satisfies readonly ExampleQuestion[]; + +const EditorValue = ({ + onEditor, +}: { + onEditor: (editor: CustomEditor) => void; +}) => { + const editor = useSlate(); + + useEffect(() => { + onEditor(editor); + }, [editor, onEditor]); + + return null; +}; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +describe("ExampleQuestionBadges", () => { + test("inserts the selected question and replaces the previous prompt", () => { + vi.spyOn(ReactEditor, "focus").mockImplementation(() => undefined); + const onEditor = vi.fn<(editor: CustomEditor) => void>(); + render( + + + + , + ); + const editor = onEditor.mock.calls[0]?.[0]; + expect(editor).toBeDefined(); + + expect(screen.getByText("Ask questions about:")).toBeTruthy(); + expect(screen.getByRole("button", { name: "Ask about entry points" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Ask about configuration" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Ask about testing" })).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { + name: "Ask about entry points", + })); + expect(Node.string(editor!)).toBe( + "Find the main entry points across the indexed repositories.", + ); + + fireEvent.click(screen.getByRole("button", { + name: "Ask about testing", + })); + expect(Node.string(editor!)).toBe("Find examples of how this code is tested."); + }); +}); diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx new file mode 100644 index 000000000..f3236d0fe --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx @@ -0,0 +1,76 @@ +'use client'; + +import { Button } from "@/components/ui/button"; +import { + BookOpen, + CircleAlert, + Database, + FlaskConical, + ListTodo, + Logs, + Package, + Search, + Settings, + Variable, + type LucideIcon, +} from "lucide-react"; +import { Editor, Transforms } from "slate"; +import { ReactEditor, useSlate } from "slate-react"; +import type { ExampleQuestion, ExampleQuestionIcon } from "./exampleQuestions"; + +const icons: Record = { + book: BookOpen, + database: Database, + error: CircleAlert, + flask: FlaskConical, + list: ListTodo, + logs: Logs, + package: Package, + search: Search, + settings: Settings, + variable: Variable, +}; + +interface ExampleQuestionBadgesProps { + questions: readonly ExampleQuestion[]; + disabled?: boolean; +} + +export function ExampleQuestionBadges({ + questions, + disabled = false, +}: ExampleQuestionBadgesProps) { + const editor = useSlate(); + + const insertQuestion = (question: string) => { + Transforms.select(editor, Editor.range(editor, [])); + Transforms.insertText(editor, question); + ReactEditor.focus(editor); + }; + + return ( +
+ + Ask questions about: + + {questions.map(({ label, question, icon }) => { + const Icon = icons[icon]; + return ( + + ); + })} +
+ ); +} diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts b/packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts new file mode 100644 index 000000000..9ce7a03d8 --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from "vitest"; +import { + exampleQuestions, + selectRandomExampleQuestions, +} from "./exampleQuestions"; + +describe("selectRandomExampleQuestions", () => { + test("selects three unique questions from the larger pool", () => { + const selectedQuestions = selectRandomExampleQuestions(3, () => 0); + + expect(exampleQuestions.length).toBeGreaterThan(3); + expect(selectedQuestions).toHaveLength(3); + expect(new Set(selectedQuestions.map(({ label }) => label)).size).toBe(3); + expect(selectedQuestions.every((question) => + exampleQuestions.some(({ label }) => label === question.label) + )).toBe(true); + }); +}); diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestions.ts b/packages/web/src/app/(app)/chat/components/exampleQuestions.ts new file mode 100644 index 000000000..3cd4a17ad --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestions.ts @@ -0,0 +1,86 @@ +export type ExampleQuestionIcon = + | "book" + | "database" + | "error" + | "flask" + | "list" + | "logs" + | "package" + | "search" + | "settings" + | "variable"; + +export interface ExampleQuestion { + label: string; + question: string; + icon: ExampleQuestionIcon; +} + +export const exampleQuestions = [ + { + label: "Entry points", + question: "Find the main entry points across all repositories.", + icon: "search", + }, + { + label: "Configuration", + question: "Where is configuration defined and loaded?", + icon: "settings", + }, + { + label: "Testing", + question: "Find examples of how this code is tested.", + icon: "flask", + }, + { + label: "Error handling", + question: "Find common error-handling patterns across the codebase.", + icon: "error", + }, + { + label: "Data models", + question: "Where are the main data models defined?", + icon: "database", + }, + { + label: "TODOs", + question: "Find TODOs and summarize the unfinished work.", + icon: "list", + }, + { + label: "Dependencies", + question: "Where are project dependencies declared and configured?", + icon: "package", + }, + { + label: "Environment", + question: "Where are environment variables read and used?", + icon: "variable", + }, + { + label: "Logging", + question: "Where and how is logging used?", + icon: "logs", + }, + { + label: "Documentation", + question: "Find the most useful developer documentation.", + icon: "book", + }, +] as const satisfies readonly ExampleQuestion[]; + +export const selectRandomExampleQuestions = ( + count: number, + random: () => number = Math.random, +): ExampleQuestion[] => { + const shuffled: ExampleQuestion[] = [...exampleQuestions]; + for (let index = shuffled.length - 1; index > 0; index -= 1) { + const swapIndex = Math.floor(random() * (index + 1)); + [shuffled[index], shuffled[swapIndex]] = [ + shuffled[swapIndex]!, + shuffled[index]!, + ]; + } + + return shuffled.slice(0, Math.max(0, count)); +}; diff --git a/packages/web/src/app/(app)/chats/chatsPage.tsx b/packages/web/src/app/(app)/chats/chatsPage.tsx index 421cbce19..d522cb1b5 100644 --- a/packages/web/src/app/(app)/chats/chatsPage.tsx +++ b/packages/web/src/app/(app)/chats/chatsPage.tsx @@ -322,7 +322,7 @@ export function ChatsPage() { return (
-

Chats

+

Chats

diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts index 2cc92230d..135bb5831 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts @@ -21,6 +21,10 @@ vi.mock('./invoicePastDueBanner', () => ({ InvoicePastDueBanner: () => null })); vi.mock('./servicePingFailedBanner', () => ({ ServicePingFailedBanner: () => null })); vi.mock('./trialBanner', () => ({ TrialBanner: () => null })); vi.mock('./upgradeAvailableBanner', () => ({ UpgradeAvailableBanner: () => null })); +vi.mock('./repositorySyncIssuesBanner', () => ({ RepositorySyncIssuesBanner: () => null })); +vi.mock('./repositoryFirstSyncBanner', () => ({ RepositoryFirstSyncBanner: () => null })); +vi.mock('./connectionSyncIssuesBanner', () => ({ ConnectionSyncIssuesBanner: () => null })); +vi.mock('./connectionFirstSyncBanner', () => ({ ConnectionFirstSyncBanner: () => null })); import { resolveActiveBanner, type BannerContext } from './bannerResolver'; @@ -79,6 +83,8 @@ const makeContext = (overrides: Partial = {}): BannerContext => ( hasPermissionSyncEntitlement: false, hasPendingFirstSync: false, permissionSyncIssues: [], + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }, dismissals: {}, today: TODAY, now: NOW, @@ -134,6 +140,184 @@ describe('resolveActiveBanner', () => { })); expect(result?.id).toBe('permissionSync'); }); + + test('permission sync outranks repository sync failures', () => { + const result = resolveActiveBanner(makeContext({ + hasPermissionSyncEntitlement: true, + hasPendingFirstSync: true, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, + })); + expect(result?.id).toBe('permissionSync'); + }); + + test('connection sync failures outrank repository sync failures', () => { + const result = resolveActiveBanner(makeContext({ + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, + })); + expect(result?.id).toBe('connectionSyncFailed'); + }); + + test('repository sync failures outrank trial notices', () => { + const result = resolveActiveBanner(makeContext({ + license: makeLicense({ + status: 'trialing', + trialEnd: daysFromNow(7), + }), + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, + })); + expect(result?.id).toBe('repositorySyncFailed'); + }); + + test('trial notices outrank repository sync warnings', () => { + const result = resolveActiveBanner(makeContext({ + license: makeLicense({ + status: 'trialing', + trialEnd: daysFromNow(7), + }), + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 1 }, + })); + expect(result?.id).toBe('trial'); + }); + }); + + describe('connection sync issues', () => { + test('shows failures and warnings as separate banners', () => { + const failed = resolveActiveBanner(makeContext({ + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 2, warningCount: 3 }, + })); + expect(failed?.id).toBe('connectionSyncFailed'); + expect(failed?.dismissible).toBe(true); + expect(failed?.audience).toBe('owner'); + + const warning = resolveActiveBanner(makeContext({ + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 2, warningCount: 3 }, + dismissals: { connectionSyncFailed: TODAY }, + })); + expect(warning?.id).toBe('connectionSyncWarning'); + expect(warning?.dismissible).toBe(true); + }); + + test('hides connection sync issues from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, + })); + expect(result).toBeNull(); + }); + }); + + describe('connection first sync', () => { + test('shows first-time syncing connections to owners', () => { + const result = resolveActiveBanner(makeContext({ + connectionSyncCounts: { + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result?.id).toBe('connectionFirstSync'); + expect(result?.dismissible).toBe(true); + expect(result?.audience).toBe('owner'); + }); + + test('hides first-time syncing connections from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + connectionSyncCounts: { + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result).toBeNull(); + }); + }); + + describe('repository sync issues', () => { + test('shows failures to owners', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 2, warningCount: 1 }, + })); + expect(result?.id).toBe('repositorySyncFailed'); + expect(result?.dismissible).toBe(true); + expect(result?.audience).toBe('owner'); + }); + + test('shows warnings when there are no failures', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 2 }, + })); + expect(result?.id).toBe('repositorySyncWarning'); + expect(result?.dismissible).toBe(true); + }); + + test('hides issues from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, + })); + expect(result).toBeNull(); + }); + + test('does not let a warning dismissal suppress a later failure', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, + dismissals: { repositorySyncWarning: TODAY }, + })); + expect(result?.id).toBe('repositorySyncFailed'); + }); + + test('hides failures dismissed today', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, + dismissals: { repositorySyncFailed: TODAY }, + })); + expect(result).toBeNull(); + }); + }); + + describe('repository first sync', () => { + test('shows first-time syncing repositories to owners', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result?.id).toBe('repositoryFirstSync'); + expect(result?.dismissible).toBe(true); + expect(result?.audience).toBe('owner'); + }); + + test('hides first-time syncing repositories from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + repositorySyncCounts: { + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result).toBeNull(); + }); + + test('repository warnings take priority over first-time syncing', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 1, + }, + })); + + expect(result?.id).toBe('repositorySyncWarning'); + }); }); describe('audience filtering', () => { diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx index 879abe251..cdad2a1c4 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx @@ -15,6 +15,10 @@ import { InvoicePastDueBanner } from "./invoicePastDueBanner"; import { ServicePingFailedBanner } from "./servicePingFailedBanner"; import { TrialBanner } from "./trialBanner"; import { UpgradeAvailableBanner } from "./upgradeAvailableBanner"; +import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; +import { RepositoryFirstSyncBanner } from "./repositoryFirstSyncBanner"; +import { ConnectionSyncIssuesBanner } from "./connectionSyncIssuesBanner"; +import { ConnectionFirstSyncBanner } from "./connectionFirstSyncBanner"; import type { PermissionSyncStatusResponse } from "@/app/api/(server)/ee/permissionSyncStatus/api"; // Mirrors the value in `lighthouse: lambda/serviceError.ts` and the gating @@ -30,6 +34,16 @@ export interface BannerContext { hasPermissionSyncEntitlement: boolean; hasPendingFirstSync: boolean; permissionSyncIssues: PermissionSyncStatusResponse['issues']; + connectionSyncCounts: { + firstTimeSyncingCount: number; + failedCount: number; + warningCount: number; + }; + repositorySyncCounts: { + firstTimeSyncingCount: number; + failedCount: number; + warningCount: number; + }; dismissals: Partial>; today: string; now: Date; @@ -178,6 +192,100 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { }); } + if (ctx.connectionSyncCounts.failedCount > 0) { + banners.push({ + id: 'connectionSyncFailed', + priority: BannerPriority.CONNECTION_SYNC_FAILED, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + if (ctx.connectionSyncCounts.warningCount > 0) { + banners.push({ + id: 'connectionSyncWarning', + priority: BannerPriority.CONNECTION_SYNC_WARNING, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + if (ctx.connectionSyncCounts.firstTimeSyncingCount > 0) { + banners.push({ + id: 'connectionFirstSync', + priority: BannerPriority.CONNECTION_FIRST_SYNC, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + + const { + firstTimeSyncingCount: repositoryFirstTimeSyncingCount, + failedCount: repositorySyncFailedCount, + warningCount: repositorySyncWarningCount, + } = ctx.repositorySyncCounts; + if (repositorySyncFailedCount > 0) { + banners.push({ + id: 'repositorySyncFailed', + priority: BannerPriority.REPOSITORY_SYNC_FAILED, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } else if (repositorySyncWarningCount > 0) { + banners.push({ + id: 'repositorySyncWarning', + priority: BannerPriority.REPOSITORY_SYNC_WARNING, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + if (repositoryFirstTimeSyncingCount > 0) { + banners.push({ + id: 'repositoryFirstSync', + priority: BannerPriority.REPOSITORY_FIRST_SYNC, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + const upgrade = getUpgradeAvailability(ctx); if (upgrade) { banners.push({ diff --git a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx index 3c2b93d5d..83ffe4199 100644 --- a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx @@ -9,6 +9,12 @@ const KNOWN_BANNER_IDS: BannerId[] = [ 'licenseReboundElsewhere', 'invoicePastDue', 'permissionSync', + 'connectionSyncFailed', + 'connectionSyncWarning', + 'connectionFirstSync', + 'repositorySyncFailed', + 'repositorySyncWarning', + 'repositoryFirstSync', 'licenseExpiryHeadsUp', 'trial', 'servicePingFailed', diff --git a/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx new file mode 100644 index 000000000..884e3b931 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx @@ -0,0 +1,116 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ refresh: vi.fn() })); +const api = vi.hoisted(() => ({ getConnectionSyncCounts: vi.fn() })); + +vi.mock("next/navigation", () => ({ + useRouter: () => navigation, +})); + +vi.mock("@/app/api/(client)/client", () => ({ + getConnectionSyncCounts: api.getConnectionSyncCounts, +})); + +vi.mock("@/lib/utils", () => ({ + cn: (...classes: Array) => + classes.filter(Boolean).join(" "), + unwrapServiceError: (value: unknown) => value, +})); + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ title, description, action }: { + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+
{title}
+
{description}
+
{action}
+
+ ), +})); + +const { ConnectionFirstSyncBanner } = await import( + "./connectionFirstSyncBanner" +); + +const renderBanner = (firstTimeSyncingCount: number) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + , + ); + return queryClient; +}; + +beforeEach(() => { + api.getConnectionSyncCounts.mockResolvedValue({ + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 0, + }); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("ConnectionFirstSyncBanner", () => { + test("updates the displayed count from the latest status", async () => { + api.getConnectionSyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }); + renderBanner(3); + + expect(screen.getByText(/3 code host connections are syncing/)).toBeTruthy(); + await waitFor(() => { + expect(screen.getByText(/2 code host connections are syncing/)).toBeTruthy(); + }); + expect( + screen.getByRole("link", { name: "View connections" }).getAttribute("href"), + ).toBe("/settings/connections?sortBy=syncedAt&sortOrder=desc"); + }); + + test("hides and refreshes when the first sync count reaches zero", async () => { + const queryClient = renderBanner(3); + await waitFor(() => { + expect(api.getConnectionSyncCounts).toHaveBeenCalled(); + }); + api.getConnectionSyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 0, + failedCount: 1, + warningCount: 0, + }); + + await act(async () => { + await queryClient.refetchQueries({ + queryKey: ["connection-sync-counts"], + }); + }); + + await waitFor(() => { + expect(screen.queryByText(/syncing for the first time/)).toBeNull(); + expect(navigation.refresh).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx new file mode 100644 index 000000000..561fe0d64 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx @@ -0,0 +1,72 @@ +"use client"; + +import { getConnectionSyncCounts } from "@/app/api/(client)/client"; +import { Button } from "@/components/ui/button"; +import type { ConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; +import { unwrapServiceError } from "@/lib/utils"; +import { useQuery } from "@tanstack/react-query"; +import { usePrevious } from "@uidotdev/usehooks"; +import { Loader2 } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect } from "react"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +const POLL_INTERVAL_MS = 5_000; + +interface ConnectionFirstSyncBannerProps extends BannerProps { + initialCounts: ConnectionSyncCounts; +} + +export function ConnectionFirstSyncBanner({ + id, + dismissible, + initialCounts, +}: ConnectionFirstSyncBannerProps) { + const router = useRouter(); + const { data: counts, isError, isPending } = useQuery({ + queryKey: ["connection-sync-counts"], + queryFn: () => unwrapServiceError(getConnectionSyncCounts()), + refetchInterval: (query) => + query.state.data?.firstTimeSyncingCount + ? POLL_INTERVAL_MS + : false, + initialData: initialCounts, + }); + const previousCount = usePrevious(counts.firstTimeSyncingCount); + + useEffect(() => { + if ( + previousCount !== undefined + && previousCount > 0 + && counts.firstTimeSyncingCount === 0 + ) { + router.refresh(); + } + }, [counts.firstTimeSyncingCount, previousCount, router]); + + if (isError || isPending || counts.firstTimeSyncingCount === 0) { + return null; + } + + const firstTimeSyncingCount = counts.firstTimeSyncingCount; + const isSingular = firstTimeSyncingCount === 1; + + return ( + } + title={`${firstTimeSyncingCount} code host ${isSingular ? "connection is" : "connections are"} syncing for the first time`} + description={`Repositories from ${isSingular ? "this connection are" : "these connections are"} unavailable until syncing completes.`} + action={( + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx new file mode 100644 index 000000000..195379c74 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx @@ -0,0 +1,73 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { ConnectionSyncIssuesBanner } from "./connectionSyncIssuesBanner"; + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ + icon, + title, + description, + action, + }: { + icon?: ReactNode; + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+ {icon} +
{title}
+
{description}
+
{action}
+
+ ), +})); + +afterEach(cleanup); + +describe("ConnectionSyncIssuesBanner", () => { + test("renders a never-synced failure independently", () => { + render( + , + ); + + expect(screen.getByText("1 code host connection needs attention")).toBeTruthy(); + expect(screen.getByText( + "This connection failed to sync. Repositories are unavailable.", + )).toBeTruthy(); + expect( + screen.getByRole("link", { name: "View failed" }).getAttribute("href"), + ).toBe("/settings/connections?status=failed"); + expect(screen.queryByRole("link", { name: "View warnings" })).toBeNull(); + }); + + test("renders warnings independently", () => { + render( + , + ); + + expect(screen.getByText("2 code host connections need attention")).toBeTruthy(); + expect(screen.getByText( + "These connections have warnings. Repository discovery may be incomplete or out of date.", + )).toBeTruthy(); + expect( + screen.getByRole("link", { name: "View warnings" }).getAttribute("href"), + ).toBe("/settings/connections?status=warning"); + expect(screen.queryByRole("link", { name: "View failed" })).toBeNull(); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx new file mode 100644 index 000000000..6b01de68c --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx @@ -0,0 +1,50 @@ +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; +import { AlertTriangle } from "lucide-react"; +import Link from "next/link"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +interface ConnectionSyncIssuesBannerProps extends BannerProps { + count: number; + status: "failed" | "warning"; +} + +const pluralizeConnection = (count: number) => + count === 1 ? "connection" : "connections"; + +export function ConnectionSyncIssuesBanner({ + id, + dismissible, + count, + status, +}: ConnectionSyncIssuesBannerProps) { + const isFailed = status === "failed"; + const isSingular = count === 1; + + return ( + + )} + title={`${count} code host ${pluralizeConnection(count)} ${isSingular ? "needs" : "need"} attention`} + description={isFailed + ? `${isSingular ? "This connection" : "These connections"} failed to sync. Repositories are unavailable.` + : `${isSingular ? "This connection has a warning" : "These connections have warnings"}. Repository discovery may be incomplete or out of date.`} + action={( + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx new file mode 100644 index 000000000..caf639415 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx @@ -0,0 +1,113 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ refresh: vi.fn() })); +const api = vi.hoisted(() => ({ getRepositorySyncCounts: vi.fn() })); + +vi.mock("next/navigation", () => ({ + useRouter: () => navigation, +})); + +vi.mock("@/app/api/(client)/client", () => ({ + getRepositorySyncCounts: api.getRepositorySyncCounts, +})); + +vi.mock("@/lib/utils", () => ({ + cn: (...classes: Array) => + classes.filter(Boolean).join(" "), + unwrapServiceError: (value: unknown) => value, +})); + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ title, description, action }: { + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+
{title}
+
{description}
+
{action}
+
+ ), +})); + +const { RepositoryFirstSyncBanner } = await import( + "./repositoryFirstSyncBanner" +); + +const renderBanner = (firstTimeSyncingCount: number) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + , + ); + return queryClient; +}; + +beforeEach(() => { + api.getRepositorySyncCounts.mockResolvedValue({ + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 0, + }); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("RepositoryFirstSyncBanner", () => { + test("updates the displayed count from the latest status", async () => { + api.getRepositorySyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }); + renderBanner(3); + + expect(screen.getByText(/3 repositories are syncing/)).toBeTruthy(); + await waitFor(() => { + expect(screen.getByText(/2 repositories are syncing/)).toBeTruthy(); + }); + }); + + test("hides and refreshes when the first sync count reaches zero", async () => { + const queryClient = renderBanner(3); + await waitFor(() => { + expect(api.getRepositorySyncCounts).toHaveBeenCalled(); + }); + api.getRepositorySyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 0, + failedCount: 1, + warningCount: 0, + }); + + await act(async () => { + await queryClient.refetchQueries({ + queryKey: ["repository-sync-counts"], + }); + }); + + await waitFor(() => { + expect(screen.queryByText(/syncing for the first time/)).toBeNull(); + expect(navigation.refresh).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx new file mode 100644 index 000000000..34970448e --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx @@ -0,0 +1,72 @@ +"use client"; + +import { getRepositorySyncCounts } from "@/app/api/(client)/client"; +import { Button } from "@/components/ui/button"; +import type { RepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import { unwrapServiceError } from "@/lib/utils"; +import { useQuery } from "@tanstack/react-query"; +import { usePrevious } from "@uidotdev/usehooks"; +import { Loader2 } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect } from "react"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +const POLL_INTERVAL_MS = 5_000; + +interface RepositoryFirstSyncBannerProps extends BannerProps { + initialCounts: RepositorySyncCounts; +} + +export function RepositoryFirstSyncBanner({ + id, + dismissible, + initialCounts, +}: RepositoryFirstSyncBannerProps) { + const router = useRouter(); + const { data: counts, isError, isPending } = useQuery({ + queryKey: ["repository-sync-counts"], + queryFn: () => unwrapServiceError(getRepositorySyncCounts()), + refetchInterval: (query) => + query.state.data?.firstTimeSyncingCount + ? POLL_INTERVAL_MS + : false, + initialData: initialCounts, + }); + const previousCount = usePrevious(counts.firstTimeSyncingCount); + + useEffect(() => { + if ( + previousCount !== undefined + && previousCount > 0 + && counts.firstTimeSyncingCount === 0 + ) { + router.refresh(); + } + }, [counts.firstTimeSyncingCount, previousCount, router]); + + if (isError || isPending || counts.firstTimeSyncingCount === 0) { + return null; + } + + const firstTimeSyncingCount = counts.firstTimeSyncingCount; + const isSingular = firstTimeSyncingCount === 1; + + return ( + } + title={`${firstTimeSyncingCount} ${isSingular ? "repository is" : "repositories are"} syncing for the first time`} + description={`${isSingular ? "It" : "They"} won't be available until syncing completes.`} + action={( + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx new file mode 100644 index 000000000..5a8ada7e4 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx @@ -0,0 +1,73 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ + icon, + title, + description, + action, + }: { + icon?: ReactNode; + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+ {icon} +
{title}
+
{description}
+
{action}
+
+ ), +})); + +afterEach(cleanup); + +describe("RepositorySyncIssuesBanner", () => { + test("summarizes failures and warnings with links to both filters", () => { + render( + , + ); + + expect(screen.getByText("3 repositories need attention")).toBeTruthy(); + expect(screen.getByText( + "1 repository failed to sync and is unavailable. 2 repositories have warnings and may contain stale results.", + )).toBeTruthy(); + expect( + screen.getByRole("link", { name: "View failed" }).getAttribute("href"), + ).toBe("/repos?status=failed"); + expect( + screen.getByRole("link", { name: "View warnings" }).getAttribute("href"), + ).toBe("/repos?status=warning"); + }); + + test("uses singular copy and only renders the relevant action", () => { + render( + , + ); + + expect(screen.getByText("1 repository needs attention")).toBeTruthy(); + expect(screen.getByText( + "1 repository has a warning and may contain stale results.", + )).toBeTruthy(); + expect(screen.queryByRole("link", { name: "View failed" })).toBeNull(); + expect(screen.getByRole("link", { name: "View warnings" })).toBeTruthy(); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx new file mode 100644 index 000000000..d8a270c89 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx @@ -0,0 +1,62 @@ +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; +import { AlertTriangle } from "lucide-react"; +import Link from "next/link"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +interface RepositorySyncIssuesBannerProps extends BannerProps { + failedCount: number; + warningCount: number; +} + +const pluralizeRepository = (count: number) => + count === 1 ? "repository" : "repositories"; + +export function RepositorySyncIssuesBanner({ + id, + dismissible, + failedCount, + warningCount, +}: RepositorySyncIssuesBannerProps) { + const totalCount = failedCount + warningCount; + const descriptions = [ + failedCount > 0 + ? `${failedCount} ${pluralizeRepository(failedCount)} failed to sync and ${failedCount === 1 ? "is" : "are"} unavailable.` + : null, + warningCount > 0 + ? `${warningCount} ${pluralizeRepository(warningCount)} ${warningCount === 1 ? "has a warning" : "have warnings"} and may contain stale results.` + : null, + ].filter((description): description is string => description !== null); + + return ( + 0 && "text-destructive", + )} + /> + )} + title={`${totalCount} ${pluralizeRepository(totalCount)} ${totalCount === 1 ? "needs" : "need"} attention`} + description={descriptions.join(" ")} + action={( + <> + {failedCount > 0 && ( + + )} + {warningCount > 0 && ( + + )} + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/types.ts b/packages/web/src/app/(app)/components/banners/types.ts index 1edbcd186..38d6a11dd 100644 --- a/packages/web/src/app/(app)/components/banners/types.ts +++ b/packages/web/src/app/(app)/components/banners/types.ts @@ -7,8 +7,14 @@ export const BannerPriority = { SERVICE_PING_ENFORCED: 95, INVOICE_PAST_DUE: 90, PERMISSION_SYNC: 50, + CONNECTION_SYNC_FAILED: 48, + REPOSITORY_SYNC_FAILED: 45, TRIAL: 25, LICENSE_EXPIRY_HEADS_UP: 20, + CONNECTION_SYNC_WARNING: 18, + REPOSITORY_SYNC_WARNING: 15, + CONNECTION_FIRST_SYNC: 14, + REPOSITORY_FIRST_SYNC: 12, SERVICE_PING_FAILED: 10, UPGRADE_AVAILABLE: 5, } as const; @@ -18,6 +24,12 @@ export type BannerId = | 'licenseReboundElsewhere' | 'invoicePastDue' | 'permissionSync' + | 'connectionSyncFailed' + | 'connectionSyncWarning' + | 'connectionFirstSync' + | 'repositorySyncFailed' + | 'repositorySyncWarning' + | 'repositoryFirstSync' | 'licenseExpiryHeadsUp' | 'trial' | 'servicePingFailed' diff --git a/packages/web/src/app/(app)/components/jobLogsDialog.tsx b/packages/web/src/app/(app)/components/jobLogsDialog.tsx new file mode 100644 index 000000000..421ae7150 --- /dev/null +++ b/packages/web/src/app/(app)/components/jobLogsDialog.tsx @@ -0,0 +1,280 @@ +"use client"; + +import { CopyIconButton } from "@/app/(app)/components/copyIconButton"; +import { + getJobLogs, +} from "@/app/api/(client)/client"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import type { JobLogEntry, JobLogLevel, QueueName } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { Check, Copy, Loader2 } from "lucide-react"; +import { useRef, useState } from "react"; + +type JobLogsDialogProps = { + queue: QueueName; + subject: string; + jobId: string; + open: boolean; + onOpenChange: (open: boolean) => void; +}; + +const timestampFormatter = new Intl.DateTimeFormat("en-CA", { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + fractionalSecondDigits: 3, + timeZoneName: "short", + hourCycle: "h23", +}); + +const formatTimestamp = (timestamp: string | null) => { + if (!timestamp) { + return "Unknown time"; + } + + const date = new Date(timestamp); + return Number.isNaN(date.getTime()) + ? timestamp + : timestampFormatter.format(date).replace(",", ""); +}; + +const formatLogLine = (entry: JobLogEntry) => { + const fields = entry.fields + ? ` ${JSON.stringify(entry.fields)}` + : ""; + return `${formatTimestamp(entry.timestamp)} ${entry.level.toUpperCase()} ${entry.message}${fields}`; +}; + +const levelStyles: Record< + JobLogLevel, + { dot: string; text: string } +> = { + debug: { + dot: "bg-muted-foreground", + text: "text-muted-foreground", + }, + info: { + dot: "bg-blue-500", + text: "text-blue-600 dark:text-blue-400", + }, + warn: { + dot: "bg-amber-500", + text: "text-amber-700 dark:text-amber-400", + }, + error: { + dot: "bg-destructive", + text: "text-destructive", + }, +}; + +const LogLevel = ({ level }: { level: JobLogLevel }) => ( + + + {level} + +); + +export const JobLogsDialog = ({ + queue, + subject, + jobId, + open, + onOpenChange, +}: JobLogsDialogProps) => { + const titleRef = useRef(null); + const [copiedAll, setCopiedAll] = useState(false); + const { data, isPending, isError } = useQuery({ + queryKey: ["job-logs", queue, jobId], + queryFn: ({ signal }) => + getJobLogs(queue, jobId, signal), + enabled: open, + }); + const latestAttempt = data?.logs.reduce( + (latest, entry) => entry.attempt === null + ? latest + : Math.max(latest ?? entry.attempt, entry.attempt), + null, + ) ?? null; + const displayedLogs = latestAttempt === null + ? data?.logs ?? [] + : data?.logs.filter((entry) => entry.attempt === latestAttempt) ?? []; + + const copyAllLogs = async () => { + try { + await navigator.clipboard.writeText( + displayedLogs.map(formatLogLine).join("\n"), + ); + setCopiedAll(true); + setTimeout(() => setCopiedAll(false), 2_000); + } catch { + setCopiedAll(false); + } + }; + + return ( + + { + event.preventDefault(); + titleRef.current?.focus(); + }} + > + +
+
+ + Job logs + + + {subject} · {jobId} + +
+ +
+ +
+
+ + + + + + + + + {[ + "Time", + "Level", + "Message", + ].map((heading) => ( + + ))} + + + + {isPending + ? ( + + + + ) + : isError + ? ( + + + + ) + : displayedLogs.length > 0 + ? displayedLogs.map((entry, index) => ( + + + + + + )) + : ( + + + + )} + +
+ {heading} +
+
+ + + Loading job logs + +
+
+ Failed to load job logs. +
+ + + + +
+ {entry.message} + {entry.fields && ( + + {` ${JSON.stringify(entry.fields)}`} + + )} +
+ { + try { + void navigator.clipboard.writeText( + formatLogLine( + entry, + ), + ); + return true; + } catch { + return false; + } + }} + /> +
+ No logs were recorded for this job. +
+
+
+
+ +
+ ); +}; diff --git a/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx b/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx index a88a3911f..cbbe416d6 100644 --- a/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx +++ b/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx @@ -15,6 +15,7 @@ interface LightweightCodeHighlighter { /* 1-based line number offset */ lineNumbersOffset?: number; renderWhitespace?: boolean; + wrapLines?: boolean; isCopyButtonVisible?: boolean; } @@ -37,6 +38,7 @@ export const LightweightCodeHighlighter = memo((prop lineNumbers = false, lineNumbersOffset = 1, renderWhitespace = false, + wrapLines = true, isCopyButtonVisible = false, } = props; @@ -135,8 +137,10 @@ export const LightweightCodeHighlighter = memo((prop style={{ fontFamily: tailwind.theme.fontFamily.editor, fontSize: tailwind.theme.fontSize.editor, - whiteSpace: renderWhitespace ? 'pre-wrap' : 'none', - wordBreak: 'break-all', + whiteSpace: wrapLines + ? (renderWhitespace ? 'pre-wrap' : 'normal') + : 'pre', + wordBreak: wrapLines ? 'break-all' : 'normal', }} > {(highlightedLines ?? unhighlightedLines).map((line, index) => ( diff --git a/packages/web/src/app/(app)/components/repositoryCarousel.tsx b/packages/web/src/app/(app)/components/repositoryCarousel.tsx deleted file mode 100644 index 26d92b5b0..000000000 --- a/packages/web/src/app/(app)/components/repositoryCarousel.tsx +++ /dev/null @@ -1,124 +0,0 @@ -'use client'; - -import { - Carousel, - CarouselContent, - CarouselItem, -} from "@/components/ui/carousel"; -import { RepositoryQuery } from "@/lib/types"; -import { getCodeHostInfoForRepo } from "@/lib/utils"; -import clsx from "clsx"; -import Autoscroll from "embla-carousel-auto-scroll"; -import Image from "next/image"; -import Link from "next/link"; - -interface RepositoryCarouselProps { - displayRepos: RepositoryQuery[]; - numberOfReposWithIndex: number; -} - -export function RepositoryCarousel({ - displayRepos, - numberOfReposWithIndex, -}: RepositoryCarouselProps) { - if (numberOfReposWithIndex === 0) { - return ( -
- No repositories found - -
-
- - <> - Create a{" "} - - connection - {" "} - to start indexing repositories - - -
-
-
- ) - } - - return ( -
- - {`${numberOfReposWithIndex} `} - - {numberOfReposWithIndex > 1 ? 'repositories' : 'repository'} - - {` indexed`} - - - - {displayRepos.map((repo, index) => ( - - - - ))} - - -
- ) -} - -interface RepositoryBadgeProps { - repo: RepositoryQuery; -} - -const RepositoryBadge = ({ - repo -}: RepositoryBadgeProps) => { - const { repoIcon, displayName } = (() => { - const info = getCodeHostInfoForRepo({ - codeHostType: repo.codeHostType, - name: repo.repoName, - displayName: repo.repoDisplayName, - externalWebUrl: repo.externalWebUrl, - }); - - return { - repoIcon: {info.codeHostName}, - displayName: info.displayName, - } - })(); - - return ( - - {repoIcon} - - {displayName} - - - ) -} diff --git a/packages/web/src/app/(app)/layout.tsx b/packages/web/src/app/(app)/layout.tsx index 7f5975c5d..435678ef5 100644 --- a/packages/web/src/app/(app)/layout.tsx +++ b/packages/web/src/app/(app)/layout.tsx @@ -36,6 +36,8 @@ import { tryGetLatestSourcebotTag } from "./components/banners/actions"; import { LanguageModelProvider } from "@/features/chat/languageModelContext"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { NavigationGuardProvider } from "next-navigation-guard"; +import { getRepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import { getConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; interface LayoutProps { children: React.ReactNode; @@ -169,6 +171,34 @@ export default async function Layout(props: LayoutProps) { permissionSyncStatus !== null && !isServiceError(permissionSyncStatus) ? permissionSyncStatus.issues : []; + const repositorySyncCountsResult = role === OrgRole.OWNER + ? await getRepositorySyncCounts().catch((error) => { + console.error("Failed to load repository sync counts", error); + return { + firstTimeSyncingCount: 0, + failedCount: 0, + warningCount: 0, + }; + }) + : { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }; + if (isServiceError(repositorySyncCountsResult)) { + throw new ServiceErrorException(repositorySyncCountsResult); + } + const repositorySyncCounts = repositorySyncCountsResult; + const connectionSyncCountsResult = role === OrgRole.OWNER + ? await getConnectionSyncCounts().catch((error) => { + console.error("Failed to load connection sync counts", error); + return { + firstTimeSyncingCount: 0, + failedCount: 0, + warningCount: 0, + }; + }) + : { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }; + if (isServiceError(connectionSyncCountsResult)) { + throw new ServiceErrorException(connectionSyncCountsResult); + } + const connectionSyncCounts = connectionSyncCountsResult; const offlineLicense = getOfflineLicenseMetadata(); const license = offlineLicense @@ -203,6 +233,8 @@ export default async function Layout(props: LayoutProps) { hasPermissionSyncEntitlement={hasPermissionSyncEntitlement} hasPendingFirstSync={hasPendingFirstSync} permissionSyncIssues={permissionSyncIssues} + connectionSyncCounts={connectionSyncCounts} + repositorySyncCounts={repositorySyncCounts} currentVersion={SOURCEBOT_VERSION} latestVersion={latestVersion} /> diff --git a/packages/web/src/app/(app)/repos/[id]/page.tsx b/packages/web/src/app/(app)/repos/[id]/page.tsx deleted file mode 100644 index 8145a535a..000000000 --- a/packages/web/src/app/(app)/repos/[id]/page.tsx +++ /dev/null @@ -1,198 +0,0 @@ -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" -import { Skeleton } from "@/components/ui/skeleton" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { env } from "@sourcebot/shared" -import { cn, getCodeHostInfoForRepo } from "@/lib/utils" -import { authenticatedPage, type OptionalAuthOptions } from "@/middleware/authenticatedPage" -import { getConfigSettings, repoMetadataSchema } from "@sourcebot/shared" -import { ExternalLink, Info } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { notFound } from "next/navigation" -import { Suspense } from "react" -import { BackButton } from "../../components/backButton" -import { DisplayDate } from "../../components/DisplayDate" -import { RepoBranchesTable } from "../components/repoBranchesTable" -import { RepoJobsTable } from "../components/repoJobsTable" -import { OrgRole } from "@sourcebot/db" - -type RepoDetailPageProps = { - params: Promise<{ id: string }> -} - -export default authenticatedPage(async ({ org, role, prisma }, props) => { - const { id } = await props.params; - const repo = await prisma.repo.findUnique({ - where: { - id: Number.parseInt(id), - orgId: org.id, - }, - include: { - jobs: { - orderBy: { - createdAt: 'desc', - }, - }, - }, - }); - if (!repo) { - notFound(); - } - - const codeHostInfo = getCodeHostInfoForRepo({ - codeHostType: repo.external_codeHostType, - name: repo.name, - displayName: repo.displayName ?? undefined, - externalWebUrl: repo.webUrl ?? undefined, - }); - - const configSettings = await getConfigSettings(env.CONFIG_PATH); - - const nextIndexAttempt = (() => { - const latestJob = repo.jobs.length > 0 ? repo.jobs[0] : null; - if (!latestJob) { - return undefined; - } - - if (latestJob.completedAt) { - return new Date(latestJob.completedAt.getTime() + configSettings.reindexIntervalMs); - } - - return undefined; - })(); - - const repoMetadata = repoMetadataSchema.parse(repo.metadata); - - return ( - <> -
- - -
-
-

{repo.displayName || repo.name}

-
- {codeHostInfo.externalWebUrl && ( - - )} -
- -
- {repo.isArchived && Archived} - {repo.isPublic && Public} -
-
- -
- - - - Created - - - - - -

When this repository was first added to Sourcebot

- - - - - - - - - - - - - Last indexed - - - - - -

The last time this repository was successfully indexed

- - - - - - {repo.indexedAt ? : "Never"} - - - - - - - Scheduled - - - - - -

When the next indexing job is scheduled to run

- - - - - - {nextIndexAttempt ? : "-"} - - -
- - {repoMetadata.indexedRevisions && ( - - -
- Indexed Branches -
- Branches that have been indexed for this repository. Docs - - - }> - - - - - )} - - - - Indexing History - History of all indexing and cleanup jobs for this repository. - - - }> - - - - - - ); -}, { allowAnonymous: true }); \ No newline at end of file diff --git a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx b/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx deleted file mode 100644 index 878ba3f2b..000000000 --- a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx +++ /dev/null @@ -1,82 +0,0 @@ -"use client" - -import { Button } from "@/components/ui/button" -import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger } from "@/components/ui/dropdown-menu" -import { getCodeHostInfoForRepo, isServiceError } from "@/lib/utils" -import { ExternalLink, MoreHorizontal } from "lucide-react" -import Link from "next/link" -import { useState } from "react" -import { indexRepo } from "@/features/repos/actions" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import type { Repo } from "./reposTable" - -interface RepoActionsDropdownProps { - repo: Repo -} - -export const RepoActionsDropdown = ({ repo }: RepoActionsDropdownProps) => { - const [isSyncing, setIsSyncing] = useState(false) - const router = useRouter() - const { toast } = useToast() - - const codeHostInfo = getCodeHostInfoForRepo({ - codeHostType: repo.codeHostType, - name: repo.name, - displayName: repo.displayName ?? undefined, - externalWebUrl: repo.webUrl ?? undefined, - }) - - const handleTriggerSync = async () => { - setIsSyncing(true) - const response = await indexRepo(repo.id) - - if (!isServiceError(response)) { - const { jobId } = response - toast({ - description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, - }) - router.refresh() - } else { - toast({ - description: `❌ Failed to sync repository. ${response.message}`, - }) - } - - setIsSyncing(false) - } - - return ( - - - - - - Actions - - View details - - - Trigger sync - - {repo.webUrl && ( - <> - - - - Open in {codeHostInfo.codeHostName} - - - - - )} - - - ) -} diff --git a/packages/web/src/app/(app)/repos/components/repoActionsMenu.tsx b/packages/web/src/app/(app)/repos/components/repoActionsMenu.tsx new file mode 100644 index 000000000..1cfdf2d74 --- /dev/null +++ b/packages/web/src/app/(app)/repos/components/repoActionsMenu.tsx @@ -0,0 +1,112 @@ +"use client"; + +import { useToast } from "@/components/hooks/use-toast"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { indexRepo } from "@/features/repos/actions"; +import { getCodeHostInfoForRepo, isServiceError } from "@/lib/utils"; +import { EllipsisVertical, ExternalLink, Loader2, RefreshCw } from "lucide-react"; +import { useState } from "react"; +import type { Repo } from "./reposTable"; + +type RepoActionsMenuProps = { + repo: Repo; + canSync: boolean; + isSyncing: boolean; + onSyncScheduled: (repoId: number, jobId: string) => void; +}; + +export const RepoActionsMenu = ({ + repo, + canSync, + isSyncing, + onSyncScheduled, +}: RepoActionsMenuProps) => { + const [isScheduling, setIsScheduling] = useState(false); + const { toast } = useToast(); + const displayName = repo.displayName ?? repo.name; + const codeHostInfo = getCodeHostInfoForRepo({ + codeHostType: repo.codeHostType, + name: repo.name, + displayName, + externalWebUrl: repo.webUrl ?? undefined, + }); + + const syncRepo = async () => { + setIsScheduling(true); + + try { + const response = await indexRepo(repo.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to sync repository", + description: response.message, + }); + return; + } + + onSyncScheduled(repo.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${displayName} was queued for indexing.`, + }); + } catch { + toast({ + variant: "destructive", + title: "Failed to sync repository", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsScheduling(false); + } + }; + + return ( + + + + + + {canSync && ( + void syncRepo()} + > + {isScheduling ? ( + + ) : ( + + )} + Sync + + )} + {codeHostInfo.externalWebUrl && ( + + + + Open in {codeHostInfo.codeHostName} + + + )} + + + ); +}; diff --git a/packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx b/packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx deleted file mode 100644 index dcd043a55..000000000 --- a/packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx +++ /dev/null @@ -1,143 +0,0 @@ -"use client" - -import * as React from "react" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { getCodeHostBrowseAtBranchUrl } from "@/lib/utils" -import Link from "next/link" -import { CodeHostType } from "@sourcebot/db"; - -type RepoBranchesTableProps = { - indexRevisions: string[]; - repoWebUrl: string | null; - repoCodeHostType: CodeHostType; -} - -export const RepoBranchesTable = ({ indexRevisions, repoWebUrl, repoCodeHostType }: RepoBranchesTableProps) => { - const [sorting, setSorting] = React.useState([]) - const [columnFilters, setColumnFilters] = React.useState([]) - - const columns = React.useMemo[]>(() => { - return [ - { - id: "refName", - accessorFn: (row) => row, - header: "Revision", - cell: ({ row }) => { - const refName = row.original; - const shortRefName = refName.replace(/^refs\/(heads|tags)\//, ""); - - const branchUrl = getCodeHostBrowseAtBranchUrl({ - webUrl: repoWebUrl, - codeHostType: repoCodeHostType, - branchName: refName, - }); - - return branchUrl ? ( - - {shortRefName} - - ) : ( - - {shortRefName} - - ) - }, - } - ] - }, [repoCodeHostType, repoWebUrl]); - - const table = useReactTable({ - data: indexRevisions, - columns, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - state: { - sorting, - columnFilters, - }, - initialState: { - pagination: { - pageSize: 5, - }, - }, - }) - - return ( -
-
- table.getColumn("refName")?.setFilterValue(event.target.value)} - className="max-w-sm" - /> -
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ))} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No branches found. - - - )} - -
-
- -
- - -
-
- ) -} diff --git a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx b/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx deleted file mode 100644 index 7f82ff796..000000000 --- a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx +++ /dev/null @@ -1,363 +0,0 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { AlertCircle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" -import * as React from "react" -import { CopyIconButton } from "../../components/copyIconButton" -import { useMemo } from "react" -import { LightweightCodeHighlighter } from "../../components/lightweightCodeHighlighter" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import { DisplayDate } from "../../components/DisplayDate" -import { LoadingButton } from "@/components/ui/loading-button" -import { indexRepo } from "@/features/repos/actions" -import { isServiceError } from "@/lib/utils" - -// @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS - -export type RepoIndexingJob = { - id: string - type: "INDEX" | "CLEANUP" - status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" - createdAt: Date - updatedAt: Date - completedAt: Date | null - errorMessage: string | null -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: RepoIndexingJob["status"]) => { - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -const getTypeBadge = (type: RepoIndexingJob["type"]) => { - return ( - - {type} - - ) -} - -const getDuration = (start: Date, end: Date | null) => { - if (!end) return "-" - const diff = end.getTime() - start.getTime() - const minutes = Math.floor(diff / 60000) - const seconds = Math.floor((diff % 60000) / 1000) - return `${minutes}m ${seconds}s` -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "type", - header: "Type", - cell: ({ row }) => getTypeBadge(row.getValue("type")), - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "status", - header: "Status", - cell: ({ row }) => { - const job = row.original - return ( -
- {getStatusBadge(row.getValue("status"))} - {job.errorMessage && ( - - - - - - - - {job.errorMessage} - - - - - )} -
- ) - }, - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "createdAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => , - }, - { - accessorKey: "completedAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const completedAt = row.getValue("completedAt") as Date | null; - if (!completedAt) { - return "-"; - } - - return - }, - }, - { - id: "duration", - header: "Duration", - cell: ({ row }) => { - const job = row.original - return getDuration(job.createdAt, job.completedAt) - }, - }, - { - accessorKey: "id", - header: "Job ID", - cell: ({ row }) => { - const id = row.getValue("id") as string - return ( -
- {id} - { - navigator.clipboard.writeText(id); - return true; - }} /> -
- ) - }, - }, -] - -export const RepoJobsTable = ({ - data, - repoId, - isIndexButtonVisible, -}: { - data: RepoIndexingJob[], - repoId: number, - isIndexButtonVisible: boolean, -}) => { - const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) - const [columnFilters, setColumnFilters] = React.useState([]) - const [columnVisibility, setColumnVisibility] = React.useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const [isIndexSubmitting, setIsIndexSubmitting] = React.useState(false); - const onIndexButtonClick = React.useCallback(async () => { - setIsIndexSubmitting(true); - const response = await indexRepo(repoId); - - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, - }) - router.refresh(); - } else { - toast({ - description: `❌ Failed to index repository. ${response.message}`, - }); - } - - setIsIndexSubmitting(false); - }, [repoId, router, toast]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - state: { - sorting, - columnFilters, - columnVisibility, - }, - }) - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - } = useMemo(() => { - return { - numCompleted: data.filter((job) => job.status === "COMPLETED").length, - numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, - numPending: data.filter((job) => job.status === "PENDING").length, - numFailed: data.filter((job) => job.status === "FAILED").length, - }; - }, [data]); - - return ( -
-
- - - - -
- - - {isIndexButtonVisible && ( - - - Trigger sync - - )} -
-
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No indexing jobs found. - - - )} - -
-
- -
-
- {table.getFilteredRowModel().rows.length} job(s) total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx new file mode 100644 index 000000000..5ed70c23c --- /dev/null +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -0,0 +1,628 @@ +import { TooltipProvider } from "@/components/ui/tooltip"; +import type { CodeHostType } from "@sourcebot/db"; +import type { JobLogs } from "@sourcebot/shared"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import type { RepoIndexingStatusesResponse } from "../types"; +import type { Repo } from "./reposTable"; + +const navigation = vi.hoisted(() => ({ + push: vi.fn(), + replace: vi.fn(), + refresh: vi.fn(), + searchParams: "", +})); +const reposActions = vi.hoisted(() => ({ + indexRepo: vi.fn(), + retryReposWithSyncIssues: vi.fn(), +})); + +vi.mock("@/features/repos/actions", () => ({ + indexRepo: reposActions.indexRepo, + retryReposWithSyncIssues: reposActions.retryReposWithSyncIssues, +})); + +vi.mock("next/navigation", () => ({ + usePathname: () => "/reposv2", + useRouter: () => navigation, + useSearchParams: () => new URLSearchParams(navigation.searchParams), +})); + +const { ReposTable } = await import("./reposTable"); + +const repos: Repo[] = [ + { + id: 1, + name: "github.com/acme/first", + displayName: "acme/first", + indexedAt: null, + indexedCommitHash: null, + latestJob: { + id: "job-1", + data: { repoId: 1 }, + status: "IN_PROGRESS", + errorMessage: null, + result: null, + }, + imageUrl: null, + webUrl: "https://github.com/acme/first", + codeHostType: "github" as CodeHostType, + }, + { + id: 2, + name: "github.com/acme/second", + displayName: "acme/second", + indexedAt: new Date("2026-08-16T12:00:00.000Z"), + indexedCommitHash: "2222222222222222222222222222222222222222", + latestJob: null, + imageUrl: null, + webUrl: "https://github.com/acme/second", + codeHostType: "github" as CodeHostType, + }, +]; + +const renderTable = ( + data: Repo[] = repos, + canRetry = true, + retryableCount = 0, +) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + + return render( + + + + + , + ); +}; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); + vi.unstubAllGlobals(); + navigation.searchParams = ""; +}); + +describe("ReposTable", () => { + test("retries all repository sync issues without refreshing", async () => { + reposActions.retryReposWithSyncIssues.mockResolvedValue({ + jobs: [{ repoId: 999, jobId: "retry-999" }], + failedCount: 0, + }); + renderTable([repos[1]], true, 2); + + fireEvent.click(screen.getByRole("button", { name: "Retry all (2)" })); + + await waitFor(() => { + expect(reposActions.retryReposWithSyncIssues).toHaveBeenCalledOnce(); + }); + await waitFor(() => { + expect( + screen.queryByRole("button", { name: /Retry all/ }), + ).toBeNull(); + }); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("hides retry all from non-owners", () => { + renderTable([repos[1]], false, 2); + + expect(screen.queryByRole("button", { name: /Retry all/ })).toBeNull(); + }); + + test("reflects the status filter from the URL", () => { + navigation.searchParams = "status=failed"; + + renderTable([repos[1]]); + + expect( + screen + .getByRole("combobox", { + name: "Filter repositories by status", + }) + .textContent, + ).toContain("Failed"); + }); + + test("centers the empty state across the table and hides pagination", () => { + navigation.searchParams = "status=warning"; + + renderTable([]); + + const emptyState = screen.getByText("No repositories with warnings."); + expect(emptyState.closest("td")?.getAttribute("colspan")).toBe("4"); + expect(screen.queryByText("Page 1 of 1")).toBeNull(); + expect(screen.queryByRole("button", { name: "Previous" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Next" })).toBeNull(); + }); + + test("clears search and status filters from the empty state", () => { + navigation.searchParams = "search=missing&status=failed&page=2&sortBy=indexedAt"; + + renderTable([]); + + fireEvent.click(screen.getByRole("button", { name: "Clear filters" })); + + expect( + (screen.getByPlaceholderText("Search repositories...") as HTMLInputElement) + .value, + ).toBe(""); + expect(navigation.replace).toHaveBeenCalledWith( + "/reposv2?sortBy=indexedAt", + { scroll: false }, + ); + }); + + test.each(["search=first", "status=warning"])( + "shows clear filters in the toolbar for %s", + (searchParams) => { + navigation.searchParams = searchParams; + + renderTable([repos[1]]); + + expect( + screen.getByRole("button", { name: "Clear filters" }), + ).toBeTruthy(); + }, + ); + + test("does not offer to clear filters in the unfiltered empty state", () => { + renderTable([]); + + expect( + screen.queryByRole("button", { name: "Clear filters" }), + ).toBeNull(); + }); + + test("does not reset pagination when the search value is unchanged", () => { + navigation.searchParams = "page=2"; + + renderTable([repos[1]]); + + expect(navigation.replace).not.toHaveBeenCalled(); + }); + + test("focuses repository search when slash is pressed", () => { + renderTable([repos[1]]); + + const searchInput = screen.getByPlaceholderText("Search repositories..."); + fireEvent.keyDown(document, { key: "/" }); + + expect(document.activeElement).toBe(searchInput); + }); + + test("shows a loading indicator while search is being debounced", () => { + renderTable([repos[1]]); + + fireEvent.change( + screen.getByPlaceholderText("Search repositories..."), + { target: { value: "sourcebot" } }, + ); + + expect(screen.getByLabelText("Searching repositories")).toBeTruthy(); + }); + + test("does not show completed for a repository already indexed on page load", () => { + renderTable([{ + ...repos[1], + latestJob: { + id: "completed-job", + data: { repoId: 2 }, + status: "COMPLETED", + errorMessage: null, + result: null, + }, + }]); + + expect(screen.queryByText("Completed")).toBeNull(); + }); + + test("links only synced repository names to their root browse page", () => { + renderTable(); + + expect( + screen.queryByRole("link", { name: "acme/first" }), + ).toBeNull(); + expect( + screen + .getByRole("link", { name: "acme/second" }) + .getAttribute("href"), + ).toBe("/browse/github.com/acme/second/-/tree"); + }); + + test("does not link a repository name without a synced commit", () => { + renderTable([{ + ...repos[1], + indexedCommitHash: null, + }]); + + expect( + screen.queryByRole("link", { name: "acme/second" }), + ).toBeNull(); + expect(screen.getByText("acme/second")).toBeTruthy(); + }); + + test("shows repository actions to everyone and sync only to owners", () => { + const { unmount } = renderTable([repos[1]], false); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect(screen.queryByRole("menuitem", { name: "Sync" })).toBeNull(); + expect( + screen + .getByRole("menuitem", { name: "Open in GitHub" }) + .getAttribute("href"), + ).toBe("https://github.com/acme/second"); + + unmount(); + renderTable([repos[1]]); + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect(screen.getByRole("menuitem", { name: "Sync" })).toBeTruthy(); + }); + + test("shows client-only syncing after an owner explicitly schedules a sync", async () => { + reposActions.indexRepo.mockResolvedValue({ jobId: "interactive-job" }); + vi.stubGlobal("fetch", vi.fn(() => new Promise(() => {}))); + renderTable([repos[1]]); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => { + expect(reposActions.indexRepo).toHaveBeenCalledWith(2); + expect(screen.getByText("Syncing")).toBeTruthy(); + expect(fetch).toHaveBeenCalledOnce(); + }); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect( + screen + .getByRole("menuitem", { name: "Sync" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("shows syncing for an indexed repository with an active latest job", async () => { + vi.stubGlobal("fetch", vi.fn(() => new Promise(() => {}))); + renderTable([{ + ...repos[1], + latestJob: { + id: "active-reindex-job", + data: { repoId: repos[1].id }, + status: "IN_PROGRESS", + errorMessage: null, + result: null, + }, + }]); + + expect(screen.getByText("Syncing")).toBeTruthy(); + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect( + screen + .getByRole("menuitem", { name: "Sync" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + }); + + test("preserves a completed sync timestamp when another sync starts", async () => { + const firstRepo: Repo = { + ...repos[0], + indexedAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000), + indexedCommitHash: "1111111111111111111111111111111111111111", + latestJob: null, + }; + reposActions.indexRepo + .mockResolvedValueOnce({ jobId: "first-interactive-job" }) + .mockResolvedValueOnce({ jobId: "second-interactive-job" }); + const firstCompletedResponse: RepoIndexingStatusesResponse = { + repositories: [{ + repoId: 1, + indexedAt: new Date().toISOString(), + indexedCommitHash: "3333333333333333333333333333333333333333", + latestJob: { + id: "first-interactive-job", + data: { repoId: 1 }, + status: "COMPLETED", + errorMessage: null, + result: null, + }, + }], + }; + vi.stubGlobal( + "fetch", + vi.fn() + .mockResolvedValueOnce(Response.json(firstCompletedResponse)) + .mockImplementation(() => new Promise(() => {})), + ); + renderTable([firstRepo, repos[1]]); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/first", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => { + const firstRow = screen + .getByRole("link", { name: "acme/first" }) + .closest("tr"); + expect(firstRow && within(firstRow).getByText("just now")).toBeTruthy(); + }); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + const firstRow = screen + .getByRole("link", { name: "acme/first" }) + .closest("tr"); + expect(firstRow && within(firstRow).getByText("just now")).toBeTruthy(); + }); + + test("shows the impact and worker error for a warning", () => { + renderTable([{ + ...repos[1], + latestJob: { + id: "warning-job", + data: { repoId: 2 }, + status: "FAILED", + errorMessage: "The remote repository could not be reached", + result: null, + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View warning details for acme/second", + })); + + expect(screen.getByText("Latest sync failed")).toBeTruthy(); + expect(screen.getByText(/results may be stale/)).toBeTruthy(); + expect( + screen.getByText("The remote repository could not be reached"), + ).toBeTruthy(); + expect(screen.getByText("warning-job")).toBeTruthy(); + }); + + test("shows the impact and worker error for a failed repository", () => { + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + + expect(screen.getByText("Repository sync failed")).toBeTruthy(); + expect(screen.getByText(/not available in search/)).toBeTruthy(); + expect( + screen.getByText("Authentication failed while cloning"), + ).toBeTruthy(); + expect(screen.getByText("job-1")).toBeTruthy(); + }); + + test("opens retained repository indexing logs", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const jobLogs: JobLogs = { + count: 1, + logs: [{ + version: 1, + timestamp: "2026-08-18T23:00:00.000Z", + level: "error", + message: "Repository indexing failed", + attempt: 2, + }], + }; + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValueOnce(Response.json(jobLogs)), + ); + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + fireEvent.click(screen.getByRole("button", { name: "View logs" })); + + const dialog = await screen.findByRole("dialog"); + await waitFor(() => { + expect(dialog.textContent).toContain("Repository indexing failed"); + }); + fireEvent.click(within(dialog).getByRole("button", { + name: "Copy all", + })); + await waitFor(() => expect(writeText).toHaveBeenCalledOnce()); + expect(writeText.mock.calls[0]?.[0]).toContain( + "Repository indexing failed", + ); + expect(fetch).toHaveBeenCalledWith( + "/api/job-logs", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + queue: "repo-index", + jobId: "job-1", + }), + }), + ); + }); + + test("hides job log access from non-owners", () => { + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }], false); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + + expect( + screen.queryByRole("button", { name: "View logs" }), + ).toBeNull(); + }); + + test("schedules a retry and transitions an unindexed repository to syncing", async () => { + reposActions.indexRepo.mockResolvedValue({ jobId: "retry-job" }); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ + repositories: [{ + repoId: 1, + indexedAt: null, + indexedCommitHash: null, + latestJob: { + id: "job-1", + data: { repoId: 1 }, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }], + } satisfies RepoIndexingStatusesResponse))); + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + fireEvent.click(screen.getByRole("button", { name: "Retry sync" })); + + await waitFor(() => { + expect(reposActions.indexRepo).toHaveBeenCalledWith(1); + expect(screen.getByText("Syncing")).toBeTruthy(); + expect(fetch).toHaveBeenCalledOnce(); + }); + expect(screen.queryByText("Repository sync failed")).toBeNull(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("updates a completed repository in place without refreshing or reordering", async () => { + let resolveRequest: ((response: Response) => void) | undefined; + vi.stubGlobal("fetch", vi.fn(() => new Promise((resolve) => { + resolveRequest = resolve; + }))); + + renderTable(); + + expect(screen.getByText("Syncing")).toBeTruthy(); + const initialRows = within(screen.getByRole("table")).getAllByRole("row"); + expect(within(initialRows[1]).getByText("acme/first")).toBeTruthy(); + expect(within(initialRows[2]).getByText("acme/second")).toBeTruthy(); + + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + const response: RepoIndexingStatusesResponse = { + repositories: [{ + repoId: 1, + indexedAt: "2026-08-17T12:00:00.000Z", + indexedCommitHash: "1111111111111111111111111111111111111111", + latestJob: { + id: "job-1", + data: { repoId: 1 }, + status: "COMPLETED", + errorMessage: null, + result: null, + }, + }], + }; + await act(async () => { + resolveRequest?.(Response.json(response)); + }); + + await waitFor(() => expect(screen.getByText("Completed")).toBeTruthy()); + expect(screen.queryByText("Syncing")).toBeNull(); + const updatedRows = within(screen.getByRole("table")).getAllByRole("row"); + expect(within(updatedRows[1]).getByText("acme/first")).toBeTruthy(); + expect( + within(updatedRows[1]) + .getByRole("link", { name: "acme/first" }) + .getAttribute("href"), + ).toBe("/browse/github.com/acme/first/-/tree"); + expect( + within(updatedRows[1]) + .getByRole("link", { name: "1111111" }) + .getAttribute("href"), + ).toBe( + "/browse/github.com/acme/first/-/commit/1111111111111111111111111111111111111111", + ); + expect(within(updatedRows[1]).queryByText("-")).toBeNull(); + expect(within(updatedRows[2]).getByText("acme/second")).toBeTruthy(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("polls a syncing repository whose latest job is missing", async () => { + const response: RepoIndexingStatusesResponse = { + repositories: [{ + repoId: 1, + indexedAt: null, + indexedCommitHash: null, + latestJob: { + id: "job-1", + data: { repoId: 1 }, + status: "FAILED", + errorMessage: "Indexing failed", + result: null, + }, + }], + }; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json(response))); + + renderTable([{ ...repos[0], latestJob: null }]); + + expect(screen.getByText("Syncing")).toBeTruthy(); + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + await waitFor(() => expect(screen.getByText("Failed")).toBeTruthy()); + expect(screen.queryByText("Syncing")).toBeNull(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index 865f19402..3a0cfb358 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -1,511 +1,1046 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { cn, getCodeHostCommitUrl, getCodeHostIcon, getRepoImageSrc, isServiceError } from "@/lib/utils" +"use client"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { useToast } from "@/components/hooks/use-toast"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, +} from "@/components/ui/input-group"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { + retryReposWithSyncIssues, + type ScheduledRepoIndexJob, +} from "@/features/repos/actions"; +import { cn, getCodeHostIcon, getRepoImageSrc, isServiceError } from "@/lib/utils"; +import type { CodeHostType } from "@sourcebot/db"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { useDebounce } from "@uidotdev/usehooks"; import { type ColumnDef, - type VisibilityState, flexRender, getCoreRowModel, useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { ArrowDown, ArrowUp, ArrowUpDown, Loader2, RefreshCwIcon } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { useEffect, useRef, useState } from "react" -import { getBrowsePath } from "../../browse/hooks/utils" -import { useRouter, useSearchParams, usePathname } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast"; -import { DisplayDate } from "../../components/DisplayDate" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { NotificationDot } from "../../components/notificationDot" -import { CodeHostType } from "@sourcebot/db" -import { useHotkeys } from "react-hotkeys-hook" -import { indexRepo } from "@/features/repos/actions" -import { RepoActionsDropdown } from "./repoActionsDropdown" +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, Check, CircleX, Loader2, RotateCw, Search } from "lucide-react"; +import Image from "next/image"; +import Link from "next/link"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import { DisplayDate } from "../../components/DisplayDate"; +import { getBrowsePath } from "../../browse/hooks/utils"; +import type { RepoIndexingStatusesResponse } from "../types"; +import { RepoActionsMenu } from "./repoActionsMenu"; +import { SyncIssuePopover } from "./syncIssuePopover"; -// @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS +const POLL_INTERVAL_MS = 5_000; +const COMPLETED_BADGE_VISIBLE_MS = 5_000; export type Repo = { - id: number - name: string - displayName: string | null - isArchived: boolean - isPublic: boolean - indexedAt: Date | null - createdAt: Date - webUrl: string | null - codeHostType: CodeHostType - imageUrl: string | null - indexedCommitHash: string | null - latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null - isFirstTimeIndex: boolean -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + id: number; + name: string; + displayName: string | null; + indexedAt: Date | null; + indexedCommitHash: string | null; + latestJob: WorkloadJob<"repo-index"> | null; + imageUrl: string | null; + webUrl: string | null; + codeHostType: CodeHostType; +}; + +type DisplayedRepo = Repo & { + showCompleted: boolean; + showExplicitSyncing: boolean; +}; + +type SortOrder = "asc" | "desc"; +type SortBy = "name" | "indexedAt"; +type StatusFilter = "all" | "failed" | "warning"; +type SyncAnnotation = "SYNCING" | "WARNING" | "FAILED" | null; + +const getStatusFilter = (value: string | null): StatusFilter => { + if (value === "failed" || value === "warning") { + return value; + } + + return "all"; +}; + +const getRepoName = (repo: Repo) => repo.displayName ?? repo.name; + +const getRepoIndexingStatuses = async ( + repoIds: number[], + signal: AbortSignal, +): Promise => { + const response = await fetch("/api/repo-index-status", { + method: "POST", + headers: { + "Content-Type": "application/json", }, - }, -}) + body: JSON.stringify({ repoIds }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load repository indexing statuses"); + } + + return response.json() as Promise; +}; + +const getSyncAnnotation = (repo: Repo): SyncAnnotation => { + const latestJob = repo.latestJob; + const isLatestIndexJob = latestJob?.data.repoId === repo.id; -const getStatusBadge = (status: Repo["latestJobStatus"]) => { - if (!status) { - return "-"; + if (isLatestIndexJob && latestJob.status === "FAILED") { + return repo.indexedAt ? "WARNING" : "FAILED"; } - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", + if ( + isLatestIndexJob + && ( + latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ) + ) { + return "SYNCING"; } - return {labels[status]} -} - -interface ColumnsContext { - onSortChange: (sortBy: string) => void; - currentSortBy?: string; - currentSortOrder: string; - onTriggerSync: (repoId: number) => void; -} - -export const getColumns = (context: ColumnsContext): ColumnDef[] => [ - { - accessorKey: "displayName", - size: 400, - header: () => { - const isActive = context.currentSortBy === 'displayName'; - const Icon = isActive - ? (context.currentSortOrder === 'asc' ? ArrowUp : ArrowDown) - : ArrowUpDown; - - return ( - - ) - }, - cell: ({ row }) => { - const repo = row.original; - const codeHostIcon = getCodeHostIcon(repo.codeHostType); - const repoImageSrc = repo.imageUrl ? getRepoImageSrc(repo.imageUrl, repo.id) : undefined; - // Internal API routes require authentication headers (cookies/API keys) to be passed through. - // Next.js Image Optimization doesn't forward these headers, so we use unoptimized=true - // to bypass the optimization and make direct requests that include auth headers. - const isInternalApiImage = repoImageSrc?.startsWith('/api/'); - - return ( -
- { - repoImageSrc ? ( + {badge} + + )} + + ); +}; + +const SortableHeader = ({ + label, + column, + sortBy, + sortOrder, + onSortChange, + tooltip, +}: { + label: string; + column: SortBy; + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + tooltip?: string; +}) => { + const isActive = sortBy === column; + const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; + + const button = ( + + ); + + if (!tooltip) { + return button; + } + + return ( + + {button} + +

{tooltip}

+ + + ); +}; + +const getColumns = ({ + sortBy, + sortOrder, + onSortChange, + canRetry, + onRetryScheduled, +}: { + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + canRetry: boolean; + onRetryScheduled: (repoId: number, jobId: string) => void; +}): ColumnDef[] => [ + { + id: "name", + accessorFn: getRepoName, + header: () => ( + + ), + cell: ({ row }) => { + const repo = row.original; + const displayName = getRepoName(repo); + const codeHostIcon = getCodeHostIcon(repo.codeHostType); + const repoImageSrc = repo.imageUrl + ? getRepoImageSrc(repo.imageUrl, repo.id) + : undefined; + const isInternalApiImage = repoImageSrc?.startsWith("/api/"); + const repoBrowseUrl = repo.indexedCommitHash + ? getBrowsePath({ + repoName: repo.name, + path: "", + pathType: "tree", + }) + : null; + + return ( +
+ {repoImageSrc ? ( {`${repo.displayName} - ) : {`${repo.displayName} + )} + {repoBrowseUrl ? ( + + {displayName} + + ) : ( + + {displayName} + + )} + - } +
+ ); + }, + }, + { + accessorKey: "indexedAt", + header: () => ( + + ), + cell: ({ row }) => { + const indexedAt = row.original.indexedAt; + return indexedAt ? : "-"; + }, + }, + { + accessorKey: "indexedCommitHash", + header: "Synced commit", + cell: ({ row }) => { + const hash = row.original.indexedCommitHash; + if (!hash) { + return "-"; + } - {/* Link to the details page (instead of browse) when the repo is indexing - as the code will not be available yet */} + const repo = row.original; + const shortHash = hash.slice(0, 7); + const commitUrl = getBrowsePath({ + repoName: repo.name, + path: "", + pathType: "commit", + commitSha: hash, + }); + const hashElement = ( - {repo.displayName || repo.name} + {shortHash} - {repo.isFirstTimeIndex && ( - - - - - - - - This is the first time Sourcebot is indexing this repository. It may take a few minutes to complete. - - - )} -
- ) - }, - }, - { - accessorKey: "latestJobStatus", - size: 150, - header: "Lastest status", - cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), - }, - { - accessorKey: "indexedAt", - size: 200, - header: () => { - const isActive = context.currentSortBy === 'indexedAt'; - const Icon = isActive - ? (context.currentSortOrder === 'asc' ? ArrowUp : ArrowDown) - : ArrowUpDown; - - return ( - - ) - }, - cell: ({ row }) => { - const indexedAt = row.getValue("indexedAt") as Date | null; - if (!indexedAt) { - return "-"; - } - - return ( - - ) - } - }, - { - accessorKey: "indexedCommitHash", - size: 150, - header: "Synced commit", - cell: ({ row }) => { - const hash = row.getValue("indexedCommitHash") as string | null; - if (!hash) { - return "-"; - } - - const smallHash = hash.slice(0, 7); - const repo = row.original; - const codeHostType = repo.codeHostType; - const webUrl = repo.webUrl; - - const commitUrl = getCodeHostCommitUrl({ - webUrl, - codeHostType, - commitHash: hash, - }); + ); - const HashComponent = commitUrl ? ( - - {smallHash} - - ) : ( - - {smallHash} - - ) - - return ( - - - {HashComponent} - - - {hash} - - - ); + return ( + + {hashElement} + + {hash} + + + ); + }, }, - }, - { - id: "actions", - size: 80, - enableHiding: false, - cell: ({ row }) => { - const repo = row.original - return + { + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ +
+ ), }, - }, -] + ]; -interface ReposTableProps { +type ReposTableProps = { data: Repo[]; currentPage: number; pageSize: number; totalCount: number; - initialSearch: string; - initialStatus: string; - initialSortBy?: string; - initialSortOrder: string; - stats: { - numCompleted: number - numFailed: number - numPending: number - numInProgress: number - numNoJobs: number - } -} - -export const ReposTable = ({ - data, - currentPage, - pageSize, - totalCount, - initialSearch, - initialStatus, - initialSortBy, - initialSortOrder, - stats, + canRetry: boolean; + retryableCount: number; + sortBy: SortBy; + sortOrder: SortOrder; +}; + +export const ReposTable = ({ + data, + currentPage, + pageSize, + totalCount, + canRetry, + retryableCount, + sortBy, + sortOrder, }: ReposTableProps) => { - const [columnVisibility, setColumnVisibility] = useState({}) - const [rowSelection, setRowSelection] = useState({}) - const [searchValue, setSearchValue] = useState(initialSearch) - const [isPendingSearch, setIsPendingSearch] = useState(false) - const searchInputRef = useRef(null) + const pathname = usePathname(); const router = useRouter(); const searchParams = useSearchParams(); - const pathname = usePathname(); + const searchParamsString = searchParams.toString(); + const urlSearchValue = searchParams.get("search") ?? ""; + const statusFilter = getStatusFilter(searchParams.get("status")); + const [searchValue, setSearchValue] = useState(urlSearchValue); + const [scheduledRetryJobs, setScheduledRetryJobs] = useState< + Map> + >(() => new Map()); + const [displayedRetryableCount, setDisplayedRetryableCount] = useState( + retryableCount, + ); + const [isRetryingAll, setIsRetryingAll] = useState(false); const { toast } = useToast(); + const debouncedSearchValue = useDebounce(searchValue, 300); + const [isSearchNavigationPending, startSearchTransition] = useTransition(); + const pendingSearchValuesRef = useRef>(new Set()); + const searchInputRef = useRef(null); + const isSearchPending = searchValue !== debouncedSearchValue + || isSearchNavigationPending; - // Focus search box when '/' is pressed - useHotkeys('/', (event) => { + useHotkeys("/", (event) => { event.preventDefault(); searchInputRef.current?.focus(); }); - // Debounced search effect - only runs when searchValue changes useEffect(() => { - setIsPendingSearch(true); - const timer = setTimeout(() => { - const params = new URLSearchParams(searchParams.toString()); - if (searchValue) { - params.set('search', searchValue); - } else { - params.delete('search'); - } - params.set('page', '1'); // Reset to page 1 on search - router.replace(`${pathname}?${params.toString()}`); - setIsPendingSearch(false); - }, 300); - - return () => { - clearTimeout(timer); - setIsPendingSearch(false); - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchValue]); - - const updateStatusFilter = (value: string) => { - const params = new URLSearchParams(searchParams.toString()); - if (value === 'all') { - params.delete('status'); - } else { - params.set('status', value); + if (pendingSearchValuesRef.current.delete(urlSearchValue)) { + return; } - params.set('page', '1'); // Reset to page 1 on filter change - router.replace(`${pathname}?${params.toString()}`); - }; - const handleSortChange = (sortBy: string) => { - const params = new URLSearchParams(searchParams.toString()); - - // Toggle sort order if clicking the same column - if (initialSortBy === sortBy) { - const newOrder = initialSortOrder === 'asc' ? 'desc' : 'asc'; - params.set('sortOrder', newOrder); + setSearchValue(urlSearchValue); + }, [urlSearchValue]); + + useEffect(() => { + setDisplayedRetryableCount(retryableCount); + }, [retryableCount]); + + useEffect(() => { + if (debouncedSearchValue !== searchValue) { + return; + } + + const nextSearchValue = debouncedSearchValue.trim(); + if (nextSearchValue === urlSearchValue) { + return; + } + + const params = new URLSearchParams(searchParamsString); + if (nextSearchValue) { + params.set("search", nextSearchValue); } else { - // Default to ascending when changing columns - params.set('sortBy', sortBy); - params.set('sortOrder', 'asc'); + params.delete("search"); } - - params.set('page', '1'); // Reset to page 1 on sort change - router.replace(`${pathname}?${params.toString()}`); - }; + params.delete("page"); - const handleTriggerSync = async (repoId: number) => { - const response = await indexRepo(repoId); + const nextSearchParamsString = params.toString(); + if (nextSearchParamsString === searchParamsString) { + return; + } - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, + pendingSearchValuesRef.current.add(nextSearchValue); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }, [ + debouncedSearchValue, + pathname, + router, + searchParamsString, + searchValue, + startSearchTransition, + urlSearchValue, + ]); + const onRetryScheduled = useCallback((repoId: number, jobId: string) => { + setScheduledRetryJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + nextJobs.set(repoId, { + id: jobId, + data: { repoId }, + status: "PENDING", + errorMessage: null, + result: null, }); - router.refresh(); - } else { + return nextJobs; + }); + }, []); + const onRetriesScheduled = useCallback((jobs: ScheduledRepoIndexJob[]) => { + setScheduledRetryJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + for (const { repoId, jobId } of jobs) { + nextJobs.set(repoId, { + id: jobId, + data: { repoId }, + status: "PENDING", + errorMessage: null, + result: null, + }); + } + return nextJobs; + }); + }, []); + const retryAll = async () => { + setIsRetryingAll(true); + + try { + const response = await retryReposWithSyncIssues(); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to retry repository syncs", + description: response.message, + }); + return; + } + + onRetriesScheduled(response.jobs); + setDisplayedRetryableCount(response.failedCount); + + if (response.failedCount > 0) { + toast({ + variant: "destructive", + title: "Some repository syncs could not be retried", + description: `${response.jobs.length} scheduled; ${response.failedCount} failed to schedule.`, + }); + } else if (response.jobs.length > 0) { + toast({ + title: "Syncs scheduled", + description: `${response.jobs.length} ${response.jobs.length === 1 ? "repository was" : "repositories were"} queued for indexing.`, + }); + } else { + toast({ + title: "No retries needed", + description: "No repositories currently have failed syncs.", + }); + } + } catch { toast({ - description: `❌ Failed to sync repository. ${response.message}`, + variant: "destructive", + title: "Failed to retry repository syncs", + description: "An unexpected error occurred while scheduling the syncs.", }); + } finally { + setIsRetryingAll(false); } }; + const retryAwareData = useMemo( + () => data.map((repo) => { + const scheduledJob = scheduledRetryJobs.get(repo.id); + return scheduledJob + ? { ...repo, latestJob: scheduledJob } + : repo; + }), + [data, scheduledRetryJobs], + ); + const pollingTargets = useMemo( + () => retryAwareData.flatMap((repo) => { + const scheduledJob = scheduledRetryJobs.get(repo.id); + return scheduledJob || getSyncAnnotation(repo) === "SYNCING" + ? [{ repoId: repo.id, jobId: repo.latestJob?.id ?? null }] + : []; + }), + [retryAwareData, scheduledRetryJobs], + ); + const pollingRepoIds = useMemo( + () => pollingTargets.map(({ repoId }) => repoId), + [pollingTargets], + ); + const pollingKey = useMemo( + () => pollingTargets.map(({ repoId, jobId }) => `${repoId}:${jobId}`), + [pollingTargets], + ); + const { data: polledStatuses } = useQuery({ + queryKey: ["reposv2-indexing-status", pollingRepoIds, pollingKey], + queryFn: ({ signal }) => getRepoIndexingStatuses(pollingRepoIds, signal), + enabled: pollingRepoIds.length > 0, + placeholderData: (previousData) => previousData, + refetchInterval: (query) => { + const statuses = query.state.data?.repositories; + if (!statuses) { + return POLL_INTERVAL_MS; + } + + return pollingTargets.some((target) => { + const status = statuses.find( + ({ repoId }) => repoId === target.repoId, + ); + if (!status) { + return true; + } + + const latestJob = status.latestJob; + const isLatestIndexJob = latestJob?.data.repoId === status.repoId; + const expectedJobId = target.jobId; + + if (expectedJobId && latestJob?.id !== expectedJobId) { + return true; + } - const totalPages = Math.ceil(totalCount / pageSize); + if (expectedJobId) { + return latestJob?.status === "PENDING" + || latestJob?.status === "IN_PROGRESS"; + } - const columns = getColumns({ - onSortChange: handleSortChange, - currentSortBy: initialSortBy, - currentSortOrder: initialSortOrder, - onTriggerSync: handleTriggerSync + return !status.indexedAt + && ( + !isLatestIndexJob + || latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ); + }) + ? POLL_INTERVAL_MS + : false; + }, }); + const completedDuringPollingRepoIds = useMemo(() => { + const pollingTargetsByRepoId = new Map( + pollingTargets.map((target) => [target.repoId, target]), + ); + return new Set( + polledStatuses?.repositories.flatMap((status) => { + const target = pollingTargetsByRepoId.get(status.repoId); + const isExpectedJob = !target?.jobId + || status.latestJob?.id === target.jobId; + return target + && isExpectedJob + && status.latestJob?.status === "COMPLETED" + && status.indexedAt + ? [status.repoId] + : []; + }) ?? [], + ); + }, [polledStatuses, pollingTargets]); + const displayedData = useMemo(() => { + const statusesByRepoId = new Map( + polledStatuses?.repositories.map((status) => [ + status.repoId, + status, + ]) ?? [], + ); + return retryAwareData.map((repo): DisplayedRepo => { + const showCompleted = completedDuringPollingRepoIds.has(repo.id); + const status = statusesByRepoId.get(repo.id); + const scheduledJob = scheduledRetryJobs.get(repo.id); + const showExplicitSyncing = Boolean( + scheduledJob + && ( + !status + || status.latestJob?.id !== scheduledJob.id + || status.latestJob.status === "PENDING" + || status.latestJob.status === "IN_PROGRESS" + ), + ); + if (!status) { + return { ...repo, showCompleted, showExplicitSyncing }; + } + if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { + return { ...repo, showCompleted, showExplicitSyncing }; + } + + return { + ...repo, + indexedAt: status.indexedAt + ? new Date(status.indexedAt) + : null, + indexedCommitHash: status.indexedCommitHash, + latestJob: status.latestJob, + showCompleted, + showExplicitSyncing, + }; + }); + }, [ + completedDuringPollingRepoIds, + polledStatuses, + retryAwareData, + scheduledRetryJobs, + ]); + const onSortChange = useCallback((column: SortBy) => { + const params = new URLSearchParams(searchParams.toString()); + const nextSortOrder = sortBy === column && sortOrder === "asc" + ? "desc" + : "asc"; + params.delete("page"); + if (column === "name") { + params.delete("sortBy"); + } else { + params.set("sortBy", column); + } + if (nextSortOrder === "asc") { + params.delete("sortOrder"); + } else { + params.set("sortOrder", nextSortOrder); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }, [pathname, router, searchParams, sortBy, sortOrder]); + const columns = useMemo( + () => getColumns({ + sortBy, + sortOrder, + onSortChange, + canRetry, + onRetryScheduled, + }), + [ + canRetry, + onRetryScheduled, + onSortChange, + sortBy, + sortOrder, + ], + ); const table = useReactTable({ - data, + data: displayedData, columns, getCoreRowModel: getCoreRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - columnResizeMode: 'onChange', - enableColumnResizing: false, + manualPagination: true, + manualSorting: true, + rowCount: totalCount, state: { - columnVisibility, - rowSelection, + pagination: { + pageIndex: currentPage - 1, + pageSize, + }, + sorting: [{ id: sortBy, desc: sortOrder === "desc" }], }, - }) + }); + const totalPages = Math.max(1, table.getPageCount()); + const firstVisibleRepo = totalCount === 0 + ? 0 + : (currentPage - 1) * pageSize + 1; + const lastVisibleRepo = Math.min(currentPage * pageSize, totalCount); + + const goToPage = (page: number) => { + const params = new URLSearchParams(searchParams.toString()); + if (page === 1) { + params.delete("page"); + } else { + params.set("page", page.toString()); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }; + + const onStatusFilterChange = (status: StatusFilter) => { + const params = new URLSearchParams(searchParams.toString()); + if (status === "all") { + params.delete("status"); + } else { + params.set("status", status); + } + params.delete("page"); + + const query = params.toString(); + router.replace(query ? `${pathname}?${query}` : pathname, { + scroll: false, + }); + }; + + const hasActiveFilters = statusFilter !== "all" + || urlSearchValue.trim().length > 0; + const clearFilters = () => { + const params = new URLSearchParams(searchParamsString); + params.delete("search"); + params.delete("status"); + params.delete("page"); + + const nextSearchParamsString = params.toString(); + setSearchValue(""); + pendingSearchValuesRef.current.add(""); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }; + + const emptyMessage = statusFilter === "failed" + ? "No failed repositories." + : statusFilter === "warning" + ? "No repositories with warnings." + : "No repositories found."; return ( -
-
- +
+
+ + + + setSearchValue(event.target.value)} - className="ring-0" + placeholder="Search repositories..." /> - {isPendingSearch && ( + {isSearchPending && ( - + )} - + {hasActiveFilters && ( + + )} + {canRetry && displayedRetryableCount > 0 && ( + + + + + + Retry all repositories whose latest sync failed. + + + )}
- +
{table.getHeaderGroups().map((headerGroup) => ( - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} ))} - {table.getRowModel().rows?.length ? ( + {table.getRowModel().rows.length > 0 ? ( table.getRowModel().rows.map((row) => ( - + {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} ))} )) ) : ( - - No results. + +

{emptyMessage}

 )}
-
-
- {totalCount} {totalCount !== 1 ? 'repositories' : 'repository'} total - {totalPages > 1 && ` • Page ${currentPage} of ${totalPages}`} + {totalCount > 0 && ( +
+

+ Showing {firstVisibleRepo}-{lastVisibleRepo} of {totalCount} +

+
+

+ Page {currentPage} of {totalPages} +

+
+ + +
+
-
- - -
-
+ )}
- ) -} + ); +}; diff --git a/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx new file mode 100644 index 000000000..dbe846b10 --- /dev/null +++ b/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx @@ -0,0 +1,213 @@ +"use client"; + +import { JobLogsDialog } from "@/app/(app)/components/jobLogsDialog"; +import { badgeVariants } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { useToast } from "@/components/hooks/use-toast"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { indexRepo } from "@/features/repos/actions"; +import { cn, isServiceError } from "@/lib/utils"; +import { + ChevronDown, + CircleX, + Loader2, + RotateCw, + ScrollText, + TriangleAlert, +} from "lucide-react"; +import { useState } from "react"; +import { DisplayDate } from "../../components/DisplayDate"; +import { LightweightCodeHighlighter } from "../../components/lightweightCodeHighlighter"; +import type { Repo } from "./reposTable"; + +type SyncIssuePopoverProps = { + repo: Repo; + annotation: "WARNING" | "FAILED"; + canRetry: boolean; + onRetryScheduled: (repoId: number, jobId: string) => void; +}; + +export const SyncIssuePopover = ({ + repo, + annotation, + canRetry, + onRetryScheduled, +}: SyncIssuePopoverProps) => { + const [isOpen, setIsOpen] = useState(false); + const [isLogsOpen, setIsLogsOpen] = useState(false); + const [isRetrying, setIsRetrying] = useState(false); + const { toast } = useToast(); + const latestJob = repo.latestJob; + if (!latestJob || latestJob.status !== "FAILED") { + return null; + } + + const isWarning = annotation === "WARNING"; + const label = isWarning ? "Warning" : "Failed"; + const Icon = isWarning ? TriangleAlert : CircleX; + const repoDisplayName = repo.displayName ?? repo.name; + + const retrySync = async () => { + setIsRetrying(true); + + try { + const response = await indexRepo(repo.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: response.message, + }); + return; + } + + onRetryScheduled(repo.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${repoDisplayName} was queued for indexing.`, + }); + setIsOpen(false); + } catch { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsRetrying(false); + } + }; + + return ( + <> + + + + + + + + +

View failure details

+ + + +
+ +
+
+

+ {isWarning + ? "Latest sync failed" + : "Repository sync failed"} +

+

+ {isWarning + ? "Search remains available using the last successful sync, but results may be stale." + : "This repository has not synced successfully, so its contents are not available in search."} +

+
+
+

+ Error +

+
+ + {latestJob.errorMessage + ?? "No error details were reported."} + +
+
+
+ {isWarning && repo.indexedAt && ( + <> +
+ Last successful sync +
+
+ +
+ + )} +
Job ID
+
+ {latestJob.id} +
+
+ {canRetry && ( +
+ + +
+ )} +
+
+ + + + + ); +}; diff --git a/packages/web/src/app/(app)/repos/layout.tsx b/packages/web/src/app/(app)/repos/layout.tsx deleted file mode 100644 index 88738d22e..000000000 --- a/packages/web/src/app/(app)/repos/layout.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { getReposStats } from "@/actions"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; - -interface LayoutProps { - children: React.ReactNode; -} - -export default async function Layout( - props: LayoutProps -) { - const { children } = props; - - const repoStats = await getReposStats(); - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - - return ( -
-
-
-
- {children} -
-
-
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/(app)/repos/page.tsx b/packages/web/src/app/(app)/repos/page.tsx index 5e123b735..4b23e012e 100644 --- a/packages/web/src/app/(app)/repos/page.tsx +++ b/packages/web/src/app/(app)/repos/page.tsx @@ -1,185 +1,141 @@ -import { sew } from "@/middleware/sew"; -import { ServiceErrorException } from "@/lib/serviceError"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + authenticatedPage, + type OptionalAuthOptions, +} from "@/middleware/authenticatedPage"; +import { + REPO_INDEX_QUEUE, + type WorkloadJob, +} from "@sourcebot/shared"; +import { getRepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; import { isServiceError } from "@/lib/utils"; -import { withOptionalAuth } from "@/middleware/withAuth"; +import { OrgRole, type Prisma } from "@sourcebot/db"; +import { z } from "zod"; import { ReposTable } from "./components/reposTable"; -import { RepoIndexingJobStatus, Prisma } from "@sourcebot/db"; -import z from "zod"; - -const numberSchema = z.coerce.number().int().positive(); const DEFAULT_PAGE_SIZE = 20; +const pageSchema = z.coerce.number().int().positive(); +const sortBySchema = z.enum(["name", "indexedAt"]); +const sortOrderSchema = z.enum(["asc", "desc"]); +const statusSchema = z.enum(["failed", "warning"]); -interface ReposPageProps { +type ReposPageProps = { searchParams: Promise<{ page?: string; - pageSize?: string; search?: string; status?: string; sortBy?: string; sortOrder?: string; }>; -} - -export default async function ReposPage(props: ReposPageProps) { - const params = await props.searchParams; - - // Parse pagination parameters with defaults - const page = numberSchema.safeParse(params.page).data ?? 1; - const pageSize = numberSchema.safeParse(params.pageSize).data ?? DEFAULT_PAGE_SIZE; - - // Parse filter parameters - const search = z.string().optional().safeParse(params.search).data ?? ''; - const status = z.enum(['all', 'none', 'COMPLETED', 'IN_PROGRESS', 'PENDING', 'FAILED']).safeParse(params.status).data ?? 'all'; - const sortBy = z.enum(['displayName', 'indexedAt']).safeParse(params.sortBy).data ?? undefined; - const sortOrder = z.enum(['asc', 'desc']).safeParse(params.sortOrder).data ?? 'asc'; - - // Calculate skip for pagination - const skip = (page - 1) * pageSize; - - const _result = await getRepos({ - skip, - take: pageSize, - search, - status, - sortBy, - sortOrder, - }); - if (isServiceError(_result)) { - throw new ServiceErrorException(_result); +}; + +export default authenticatedPage< + ReposPageProps, + OptionalAuthOptions +>(async ({ org, prisma, role }, { searchParams }) => { + const params = await searchParams; + const page = pageSchema.safeParse(params.page).data ?? 1; + const search = z.string().optional().safeParse(params.search).data?.trim() ?? ""; + const status = statusSchema.safeParse(params.status).data ?? "all"; + const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; + const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const canRetry = role === OrgRole.OWNER; + const skip = (page - 1) * DEFAULT_PAGE_SIZE; + const orderBy = sortBy === "indexedAt" + ? [{ indexedAt: sortOrder }, { id: "asc" as const }] + : [{ displayName: sortOrder }, { id: "asc" as const }]; + const failedJobIds = status === "all" + ? [] + : await getBullMQClient().getFailedJobIds(REPO_INDEX_QUEUE); + const repositorySyncCounts = canRetry + ? await getRepositorySyncCounts() + : null; + const retryableCount = repositorySyncCounts + && !isServiceError(repositorySyncCounts) + ? repositorySyncCounts.failedCount + repositorySyncCounts.warningCount + : 0; + const where: Prisma.RepoWhereInput = { + orgId: org.id, + ...(search + ? { + displayName: { + contains: search, + mode: "insensitive" as const, + }, + } + : {}), + ...(status === "all" + ? {} + : { + latestIndexingJobId: { in: failedJobIds }, + indexedAt: status === "failed" ? null : { not: null }, + }), + }; + + const [repos, totalCount] = await Promise.all([ + prisma.repo.findMany({ + where, + orderBy, + skip, + take: DEFAULT_PAGE_SIZE, + select: { + id: true, + name: true, + displayName: true, + indexedAt: true, + indexedCommitHash: true, + latestIndexingJobId: true, + imageUrl: true, + webUrl: true, + external_codeHostType: true, + }, + }), + prisma.repo.count({ + where, + }), + ]); + const latestJobIds = repos.flatMap((repo) => + repo.latestIndexingJobId ? [repo.latestIndexingJobId] : [], + ); + let latestJobs = new Map | null>(); + try { + latestJobs = await getBullMQClient().getJobs( + REPO_INDEX_QUEUE, + latestJobIds, + ); + } catch (error) { + console.error("Failed to load latest repository indexing jobs", error); } - const { repos, totalCount, stats } = _result; - return ( - <> -
-

Repositories

-

View and manage your code repositories and their indexing status.

+
+
+

Repositories

- ({ - id: repo.id, - name: repo.name, - displayName: repo.displayName ?? repo.name, - isArchived: repo.isArchived, - isPublic: repo.isPublic, - indexedAt: repo.indexedAt, - createdAt: repo.createdAt, - webUrl: repo.webUrl, - imageUrl: repo.imageUrl, - latestJobStatus: repo.latestIndexingJobStatus, - isFirstTimeIndex: repo.indexedAt === null, - codeHostType: repo.external_codeHostType, - indexedCommitHash: repo.indexedCommitHash, - }))} - currentPage={page} - pageSize={pageSize} - totalCount={totalCount} - initialSearch={search} - initialStatus={status} - initialSortBy={sortBy} - initialSortOrder={sortOrder} - stats={stats} - /> - - ) -} - -interface GetReposParams { - skip: number; - take: number; - search: string; - status: 'all' | 'none' | 'COMPLETED' | 'IN_PROGRESS' | 'PENDING' | 'FAILED'; - sortBy?: 'displayName' | 'indexedAt'; - sortOrder: 'asc' | 'desc'; -} - -const getRepos = async ({ skip, take, search, status, sortBy, sortOrder }: GetReposParams) => sew(() => - withOptionalAuth(async ({ prisma }) => { - const whereClause: Prisma.RepoWhereInput = { - ...(search ? { - displayName: { contains: search, mode: 'insensitive' }, - } : {}), - latestIndexingJobStatus: - status === 'all' ? undefined : - status === 'none' ? null : - status - }; - - // Build orderBy clause based on sortBy and sortOrder - const orderByClause: Prisma.RepoOrderByWithRelationInput = {}; - - if (sortBy === 'displayName') { - orderByClause.displayName = sortOrder === 'asc' ? 'asc' : 'desc'; - } else if (sortBy === 'indexedAt') { - orderByClause.indexedAt = sortOrder === 'asc' ? 'asc' : 'desc'; - } else { - // Default to displayName asc - orderByClause.displayName = 'asc'; - } - - const repos = await prisma.repo.findMany({ - skip, - take, - where: whereClause, - orderBy: orderByClause, - }); - - // Calculate total count using the filtered where clause - const totalCount = await prisma.repo.count({ - where: whereClause - }); - - // Status stats - const [ - numCompleted, - numFailed, - numPending, - numInProgress, - numNoJobs - ] = await Promise.all([ - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.PENDING, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: null, - } - }), - ]) - - return { - repos, - totalCount, - stats: { - numCompleted, - numFailed, - numPending, - numInProgress, - numNoJobs, - } - }; - })); \ No newline at end of file +
+ ({ + id: repo.id, + name: repo.name, + displayName: repo.displayName, + indexedAt: repo.indexedAt, + indexedCommitHash: repo.indexedCommitHash, + latestJob: repo.latestIndexingJobId + ? latestJobs.get(repo.latestIndexingJobId) ?? null + : null, + imageUrl: repo.imageUrl, + webUrl: repo.webUrl, + codeHostType: repo.external_codeHostType, + }))} + currentPage={page} + pageSize={DEFAULT_PAGE_SIZE} + totalCount={totalCount} + canRetry={canRetry} + retryableCount={retryableCount} + sortBy={sortBy} + sortOrder={sortOrder} + /> +
+
+ ); +}, { allowAnonymous: true }); diff --git a/packages/web/src/app/(app)/repos/types.ts b/packages/web/src/app/(app)/repos/types.ts new file mode 100644 index 000000000..499781029 --- /dev/null +++ b/packages/web/src/app/(app)/repos/types.ts @@ -0,0 +1,12 @@ +import type { WorkloadJob } from "@sourcebot/shared"; + +export type RepoIndexingStatus = { + repoId: number; + indexedAt: string | null; + indexedCommitHash: string | null; + latestJob: WorkloadJob<"repo-index"> | null; +}; + +export type RepoIndexingStatusesResponse = { + repositories: RepoIndexingStatus[]; +}; diff --git a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx index e9236607e..f83d2aed3 100644 --- a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx +++ b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx @@ -1,11 +1,10 @@ import { SourcebotLogo } from "@/app/components/sourcebotLogo" -import { RepositoryCarousel } from "../../components/repositoryCarousel" import { Separator } from "@/components/ui/separator" import { SyntaxReferenceGuideHint } from "../../components/syntaxReferenceGuideHint" import Link from "next/link" import { SearchBar } from "../../components/searchBar" import { SearchModeSelector } from "../../components/searchModeSelector" -import { getRepos, getReposStats } from "@/actions" +import { getRepos } from "@/actions" import { ServiceErrorException } from "@/lib/serviceError" import { isServiceError } from "@/lib/utils" @@ -25,10 +24,7 @@ export const SearchLandingPage = async ({ take: 10, }); - const repoStats = await getReposStats(); - if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); - if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); return (
@@ -53,15 +49,7 @@ export const SearchLandingPage = async ({
-
- -
- -
- +
How to search
) -} \ No newline at end of file +} diff --git a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx deleted file mode 100644 index edcc61069..000000000 --- a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx +++ /dev/null @@ -1,211 +0,0 @@ -import { sew } from "@/middleware/sew"; -import { BackButton } from "@/app/(app)/components/backButton"; -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; -import { notFound } from "next/navigation"; -import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; -import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; -import { env, getConfigSettings } from "@sourcebot/shared"; -import { Info } from "lucide-react"; -import Link from "next/link"; -import { Suspense } from "react"; -import { ConnectionJobsTable } from "../components/connectionJobsTable"; - -interface ConnectionDetailPageProps { - params: Promise<{ - id: string - }> -} - -export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { - const params = await props.params; - const { id } = params; - - const connectionId = Number.parseInt(id); - if (isNaN(connectionId)) { - return notFound(); - } - - const connection = await getConnectionWithJobs(connectionId); - if (isServiceError(connection)) { - throw new ServiceErrorException(connection); - } - - const configSettings = await getConfigSettings(env.CONFIG_PATH); - - const nextSyncAttempt = (() => { - const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; - if (!latestJob) { - return undefined; - } - - if (latestJob.completedAt) { - return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); - } - - return undefined; - })(); - - // Extracts the code host URL from the connection config. - const codeHostUrl: string = (() => { - const connectionType = connection.connectionType; - switch (connectionType) { - case 'github': { - const config = connection.config as unknown as GithubConnectionConfig; - return config.url ?? 'https://github.com'; - } - case 'gitlab': { - const config = connection.config as unknown as GitlabConnectionConfig; - return config.url ?? 'https://gitlab.com'; - } - case 'gitea': { - const config = connection.config as unknown as GiteaConnectionConfig; - return config.url ?? 'https://gitea.com'; - } - case 'gerrit': { - const config = connection.config as unknown as GerritConnectionConfig; - return config.url; - } - case 'bitbucket': { - const config = connection.config as unknown as BitbucketConnectionConfig; - if (config.deploymentType === 'cloud') { - return config.url ?? 'https://bitbucket.org'; - } else { - return config.url!; - } - } - case 'azuredevops': { - const config = connection.config as unknown as AzureDevOpsConnectionConfig; - return config.url ?? 'https://dev.azure.com'; - } - case 'git': { - const config = connection.config as unknown as GenericGitHostConnectionConfig; - return config.url; - } - } - })(); - - return ( -
- -
-

{connection.name}

- - - {codeHostUrl} - -
- -
- - - - Created - - - - - -

When this connection was first added to Sourcebot

- - - - - - - - - - - - - Last synced - - - - - -

The last time this connection was successfully synced

- - - - - - {connection.syncedAt ? : "Never"} - - - - - - - Scheduled - - - - - -

When the connection will be resynced next. Modifying the config will also trigger a resync.

- - - - - - {nextSyncAttempt ? : "-"} - - -
- - - - Sync History - History of all sync jobs for this connection. - - - }> - - - - -
- ) -} - -const getConnectionWithJobs = async (id: number) => sew(() => - withAuth(async ({ prisma, org }) => { - const connection = await prisma.connection.findUnique({ - where: { - id, - orgId: org.id, - }, - include: { - syncJobs: { - orderBy: { - createdAt: 'desc', - }, - }, - }, - }); - - if (!connection) { - return notFoundServiceError(); - } - - return connection; - }) -) \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionActionsMenu.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionActionsMenu.tsx new file mode 100644 index 000000000..c61b959bd --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/connectionActionsMenu.tsx @@ -0,0 +1,91 @@ +"use client"; + +import { useToast } from "@/components/hooks/use-toast"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { syncConnection } from "@/features/connections/actions"; +import { isServiceError } from "@/lib/utils"; +import { EllipsisVertical, Loader2, RefreshCw } from "lucide-react"; +import { useState } from "react"; + +type ConnectionActionsMenuProps = { + connection: { + id: number; + name: string; + }; + isSyncing: boolean; + onSyncScheduled: (connectionId: number, jobId: string) => void; +}; + +export const ConnectionActionsMenu = ({ + connection, + isSyncing, + onSyncScheduled, +}: ConnectionActionsMenuProps) => { + const [isScheduling, setIsScheduling] = useState(false); + const { toast } = useToast(); + + const scheduleSync = async () => { + setIsScheduling(true); + + try { + const response = await syncConnection(connection.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to sync connection", + description: response.message, + }); + return; + } + + onSyncScheduled(connection.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${connection.name} was queued for syncing.`, + }); + } catch { + toast({ + variant: "destructive", + title: "Failed to sync connection", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsScheduling(false); + } + }; + + return ( + + + + + + void scheduleSync()} + > + {isScheduling ? ( + + ) : ( + + )} + Sync + + + + ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx deleted file mode 100644 index 0600a5c37..000000000 --- a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx +++ /dev/null @@ -1,344 +0,0 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" -import * as React from "react" -import { CopyIconButton } from "@/app/(app)/components/copyIconButton" -import { useMemo } from "react" -import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { LoadingButton } from "@/components/ui/loading-button" -import { syncConnection } from "@/features/connections/actions" -import { isServiceError } from "@/lib/utils" - - -export type ConnectionSyncJob = { - id: string - status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" - createdAt: Date - updatedAt: Date - completedAt: Date | null - errorMessage: string | null - warningMessages: string[] -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: ConnectionSyncJob["status"]) => { - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -const getDuration = (start: Date, end: Date | null) => { - if (!end) return "-" - const diff = end.getTime() - start.getTime() - const minutes = Math.floor(diff / 60000) - const seconds = Math.floor((diff % 60000) / 1000) - return `${minutes}m ${seconds}s` -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "status", - header: "Status", - cell: ({ row }) => { - const job = row.original - return ( -
- {getStatusBadge(row.getValue("status"))} - {job.errorMessage ? ( - - - - - - - - {job.errorMessage} - - - - - ) : job.warningMessages.length > 0 ? ( - - - - - - -

{job.warningMessages.length} warning(s) while syncing:

-
- {job.warningMessages.map((warning, index) => ( -
- {index + 1}. - {warning} -
- ))} -
- - - - ) : null} -
- ) - }, - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "createdAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => , - }, - { - accessorKey: "completedAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const completedAt = row.getValue("completedAt") as Date | null; - if (!completedAt) { - return "-"; - } - - return - }, - }, - { - id: "duration", - header: "Duration", - cell: ({ row }) => { - const job = row.original - return getDuration(job.createdAt, job.completedAt) - }, - }, - { - accessorKey: "id", - header: "Job ID", - cell: ({ row }) => { - const id = row.getValue("id") as string - return ( -
- {id} - { - navigator.clipboard.writeText(id); - return true; - }} /> -
- ) - }, - }, -] - -export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { - const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) - const [columnFilters, setColumnFilters] = React.useState([]) - const [columnVisibility, setColumnVisibility] = React.useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); - const onSyncButtonClick = React.useCallback(async () => { - setIsSyncSubmitting(true); - const response = await syncConnection(connectionId); - - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Connection sync scheduled. Job ID: ${jobId}`, - }) - router.refresh(); - } else { - toast({ - description: `❌ Failed to sync connection. ${response.message}`, - }); - } - - setIsSyncSubmitting(false); - }, [connectionId, router, toast]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - state: { - sorting, - columnFilters, - columnVisibility, - }, - }) - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - } = useMemo(() => { - return { - numCompleted: data.filter((job) => job.status === "COMPLETED").length, - numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, - numPending: data.filter((job) => job.status === "PENDING").length, - numFailed: data.filter((job) => job.status === "FAILED").length, - }; - }, [data]); - - return ( -
-
- - -
- - - - - Trigger sync - -
-
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No sync jobs found. - - - )} - -
-
- -
-
- {table.getFilteredRowModel().rows.length} job(s) total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx new file mode 100644 index 000000000..a1fb02e92 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx @@ -0,0 +1,558 @@ +import type { ConnectionType } from "@sourcebot/db"; +import type { JobLogs } from "@sourcebot/shared"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, + within, +} from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import type { ConnectionSyncStatusesResponse } from "../types"; +import type { Connection } from "./connectionsTable"; + +const navigation = vi.hoisted(() => ({ + push: vi.fn(), + replace: vi.fn(), + refresh: vi.fn(), + searchParams: "", +})); +const connectionActions = vi.hoisted(() => ({ + syncConnection: vi.fn(), +})); +const toast = vi.hoisted(() => vi.fn()); + +vi.mock("next/navigation", () => ({ + usePathname: () => "/settings/connections", + useRouter: () => navigation, + useSearchParams: () => new URLSearchParams(navigation.searchParams), +})); + +vi.mock("@/features/connections/actions", () => ({ + syncConnection: connectionActions.syncConnection, +})); + +vi.mock("@/components/hooks/use-toast", () => ({ + useToast: () => ({ toast }), +})); + +const { ConnectionsTable } = await import("./connectionsTable"); + +const connections: Connection[] = [ + { + id: 1, + name: "Primary GitHub", + connectionType: "github" as ConnectionType, + syncedAt: new Date("2026-08-18T12:00:00.000Z"), + latestJob: null, + }, + { + id: 2, + name: "Internal GitLab", + connectionType: "gitlab" as ConnectionType, + syncedAt: null, + latestJob: null, + }, +]; + +type RenderTableOptions = { + data?: Connection[]; + currentPage?: number; + totalCount?: number; + sortBy?: "name" | "syncedAt"; + sortOrder?: "asc" | "desc"; +}; + +const renderTable = (options: RenderTableOptions = {}) => { + const data = options.data ?? connections; + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + + return render( + + + + + , + ); +}; + +beforeEach(() => { + vi.stubGlobal("fetch", vi.fn(() => new Promise(() => {}))); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); + vi.unstubAllGlobals(); + navigation.searchParams = ""; +}); + +describe("ConnectionsTable", () => { + test("renders connection names without detail-page links", () => { + renderTable(); + + expect(screen.getByText("Primary GitHub")).toBeTruthy(); + expect( + screen.queryByRole("link", { name: "Primary GitHub" }), + ).toBeNull(); + }); + + test("reflects the status filter from the URL", () => { + navigation.searchParams = "status=failed"; + + renderTable(); + + expect( + screen + .getByRole("combobox", { + name: "Filter connections by status", + }) + .textContent, + ).toContain("Failed"); + }); + + test("renders a status-specific empty state", () => { + navigation.searchParams = "status=warning"; + + renderTable({ data: [] }); + + expect(screen.getByText("No connections with warnings.")).toBeTruthy(); + expect(screen.queryByText("Page 1 of 1")).toBeNull(); + }); + + test("clears search and status filters from the empty state", () => { + navigation.searchParams = + "search=missing&status=failed&page=2&sortBy=syncedAt"; + + renderTable({ data: [] }); + + fireEvent.click(screen.getByRole("button", { name: "Clear filters" })); + + expect( + (screen.getByPlaceholderText("Search connections...") as HTMLInputElement) + .value, + ).toBe(""); + expect(navigation.replace).toHaveBeenCalledWith( + "/settings/connections?sortBy=syncedAt", + { scroll: false }, + ); + }); + + test.each(["search=github", "status=warning"])( + "shows clear filters in the toolbar for %s", + (searchParams) => { + navigation.searchParams = searchParams; + + renderTable({ data: [connections[0]] }); + + expect( + screen.getByRole("button", { name: "Clear filters" }), + ).toBeTruthy(); + }, + ); + + test("uses URL-driven server pagination", () => { + renderTable({ totalCount: 29 }); + + expect(screen.getByText("Showing 1-20 of 29")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Next" })); + + expect(navigation.push).toHaveBeenCalledWith( + "/settings/connections?page=2", + ); + }); + + test("toggles server-side name sorting", () => { + renderTable(); + + fireEvent.click(screen.getByRole("button", { name: "Sort by Name" })); + + expect(navigation.push).toHaveBeenCalledWith( + "/settings/connections?sortOrder=desc", + ); + }); + + test("focuses connection search when slash is pressed", () => { + renderTable(); + + const searchInput = screen.getByPlaceholderText("Search connections..."); + fireEvent.keyDown(document, { key: "/" }); + + expect(document.activeElement).toBe(searchInput); + }); + + test("shows and polls an explicitly scheduled sync", async () => { + connectionActions.syncConnection.mockResolvedValue({ jobId: "job-1" }); + renderTable(); + + fireEvent.keyDown( + screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), + { key: "Enter" }, + ); + + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + await waitFor(() => { + expect(connectionActions.syncConnection).toHaveBeenCalledWith(1); + expect(screen.getByText("Syncing")).toBeTruthy(); + expect(fetch).toHaveBeenCalledOnce(); + }); + + fireEvent.keyDown( + screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), + { key: "Enter" }, + ); + expect( + screen + .getByRole("menuitem", { name: "Sync" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("updates an actively syncing connection in place", async () => { + let resolveRequest: ((response: Response) => void) | undefined; + vi.mocked(fetch).mockImplementation(() => + new Promise((resolve) => { + resolveRequest = resolve; + }) + ); + const activeConnection: Connection = { + ...connections[0], + latestJob: { + id: "active-job", + data: { connectionId: connections[0].id }, + status: "IN_PROGRESS", + errorMessage: null, + result: null, + }, + }; + renderTable({ data: [activeConnection, connections[1]] }); + + expect(screen.getByText("Syncing")).toBeTruthy(); + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + + const response: ConnectionSyncStatusesResponse = { + connections: [{ + connectionId: activeConnection.id, + syncedAt: new Date().toISOString(), + latestJob: { + ...activeConnection.latestJob!, + status: "COMPLETED", + result: { outcome: "SUCCESS" }, + }, + }], + }; + await act(async () => { + resolveRequest?.(Response.json(response)); + }); + + await waitFor(() => expect(screen.getByText("Completed")).toBeTruthy()); + expect(screen.queryByText("Syncing")).toBeNull(); + const rows = within(screen.getByRole("table")).getAllByRole("row"); + expect(within(rows[1]).getByText("Primary GitHub")).toBeTruthy(); + expect(within(rows[1]).getByText("just now")).toBeTruthy(); + expect(within(rows[2]).getByText("Internal GitLab")).toBeTruthy(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("preserves the latest synced timestamp when scheduling another sync", async () => { + const previousSyncedAt = new Date( + Date.now() - 2 * 24 * 60 * 60 * 1_000, + ); + const completedAt = new Date(); + const connection = { + ...connections[0], + syncedAt: previousSyncedAt, + }; + connectionActions.syncConnection + .mockResolvedValueOnce({ jobId: "first-job" }) + .mockResolvedValueOnce({ jobId: "second-job" }); + vi.mocked(fetch) + .mockResolvedValueOnce(Response.json({ + connections: [{ + connectionId: connection.id, + syncedAt: completedAt.toISOString(), + latestJob: { + id: "first-job", + data: { connectionId: connection.id }, + status: "COMPLETED", + errorMessage: null, + result: { outcome: "SUCCESS" }, + }, + }], + } satisfies ConnectionSyncStatusesResponse)) + .mockImplementation(() => new Promise(() => {})); + renderTable({ data: [connection] }); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => expect(screen.getByText("just now")).toBeTruthy()); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => { + expect(fetch).toHaveBeenCalledTimes(2); + expect(screen.getByText("Syncing")).toBeTruthy(); + }); + expect(screen.getByText("just now")).toBeTruthy(); + expect(screen.queryByText("2 days ago")).toBeNull(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test.each([ + ["PENDING", "Syncing"], + ["IN_PROGRESS", "Syncing"], + ] as const)("renders a %s sync annotation", (status, label) => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "job-1", + data: { connectionId: connections[0].id }, + status, + errorMessage: null, + result: null, + }, + }], + }); + + expect(screen.getByText(label)).toBeTruthy(); + }); + + test("renders a failed annotation when the connection has never synced", () => { + renderTable({ + data: [{ + ...connections[0], + syncedAt: null, + latestJob: { + id: "failed-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "Authentication failed", + result: null, + }, + }], + }); + + const failed = screen.getByText("Failed"); + expect(failed.closest("td")).toBe( + screen.getByText("Primary GitHub").closest("td"), + ); + }); + + test("renders a warning when a previously synced connection fails", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "warning-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "Authentication failed", + result: null, + }, + }], + }); + + expect(screen.getByText("Warning")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { + name: "View warning details for Primary GitHub", + })); + + expect(screen.getByText("Latest connection sync failed")).toBeTruthy(); + expect(screen.getByText(/may be stale/)).toBeTruthy(); + expect(screen.getByText("Authentication failed")).toBeTruthy(); + expect(screen.getByText("warning-job")).toBeTruthy(); + }); + + test("renders structured warnings for a partial success", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "partial-job", + data: { connectionId: connections[0].id }, + status: "COMPLETED", + errorMessage: null, + result: { + outcome: "PARTIAL_SUCCESS", + reasons: [{ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "acme/private", + }, + message: "Not found or inaccessible", + }], + }, + }, + }], + }); + + fireEvent.click(screen.getByRole("button", { + name: "View warning details for Primary GitHub", + })); + + expect( + screen.getByText("Connection sync completed with warnings"), + ).toBeTruthy(); + expect(screen.getByText("repository")).toBeTruthy(); + expect(screen.getByText("acme/private")).toBeTruthy(); + expect(screen.getByText("Not found or inaccessible")).toBeTruthy(); + expect(screen.getByText("partial-job")).toBeTruthy(); + }); + + test("shows the worker error for a failed sync and supports retry", async () => { + connectionActions.syncConnection.mockResolvedValue({ + jobId: "retry-job", + }); + renderTable({ + data: [{ + ...connections[0], + syncedAt: null, + latestJob: { + id: "failed-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "Authentication failed while discovering repositories", + result: null, + }, + }], + }); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for Primary GitHub", + })); + + expect(screen.getByText("Connection sync failed")).toBeTruthy(); + expect(screen.getByText( + "This connection failed to sync. Its repositories are unavailable.", + )).toBeTruthy(); + expect( + screen.getByText( + "Authentication failed while discovering repositories", + ), + ).toBeTruthy(); + expect(screen.getByText("failed-job")).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Retry sync" })); + + await waitFor(() => { + expect(connectionActions.syncConnection).toHaveBeenCalledWith(1); + expect(screen.getByText("Syncing")).toBeTruthy(); + }); + expect(screen.queryByText("Connection sync failed")).toBeNull(); + }); + + test("opens retained job logs in a large dialog", async () => { + const jobLogs: JobLogs = { + count: 3, + logs: [ + { + version: 1, + timestamp: "2026-08-18T23:00:00.000Z", + level: "info", + message: "Earlier attempt failed", + attempt: 1, + }, + { + version: 1, + timestamp: "2026-08-18T23:00:01.000Z", + level: "info", + message: "Starting repository discovery", + attempt: 2, + }, + { + version: 1, + timestamp: "2026-08-18T23:00:02.000Z", + level: "error", + message: "Repository discovery failed", + attempt: 2, + fields: { provider: "gitlab" }, + }, + ], + }; + vi.mocked(fetch).mockResolvedValueOnce(Response.json(jobLogs)); + renderTable({ + data: [{ + ...connections[0], + syncedAt: null, + latestJob: { + id: "failed-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "fetch failed", + result: null, + }, + }], + }); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for Primary GitHub", + })); + fireEvent.click(screen.getByRole("button", { name: "View logs" })); + + const dialog = await screen.findByRole("dialog"); + expect(within(dialog).getByText("Job logs")).toBeTruthy(); + await waitFor(() => { + expect(dialog.textContent).toContain("Starting repository discovery"); + expect(dialog.textContent).toContain("Repository discovery failed"); + expect(dialog.textContent).toContain("gitlab"); + }); + expect(dialog.textContent).not.toContain("Earlier attempt failed"); + expect(fetch).toHaveBeenCalledWith( + "/api/job-logs", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + queue: "connection-sync", + jobId: "failed-job", + }), + }), + ); + }); + + test("does not annotate a successful sync", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "job-1", + data: { connectionId: connections[0].id }, + status: "COMPLETED", + errorMessage: null, + result: { outcome: "SUCCESS" }, + }, + }], + }); + + expect(screen.queryByText("Syncing")).toBeNull(); + expect(screen.queryByText("Failed")).toBeNull(); + expect(screen.queryByText("Warning")).toBeNull(); + expect(screen.queryByText("Completed")).toBeNull(); + }); +}); diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx index e52e7ef1e..f901f3065 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx @@ -1,294 +1,702 @@ -"use client" - -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { NotificationDot } from "@/app/(app)/components/notificationDot" -import { useToast } from "@/components/hooks/use-toast" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { getCodeHostIcon } from "@/lib/utils" -import { ConnectionType } from "@sourcebot/db" +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { Button } from "@/components/ui/button"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, +} from "@/components/ui/input-group"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { cn, getCodeHostIcon } from "@/lib/utils"; +import type { ConnectionType } from "@sourcebot/db"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { useDebounce } from "@uidotdev/usehooks"; import { type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, flexRender, getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { ArrowUpDown, RefreshCwIcon } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { useRouter } from "next/navigation" -import { useMemo, useState } from "react" +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, CircleX, Loader2, Search } from "lucide-react"; +import Image from "next/image"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import type { ConnectionSyncStatusesResponse } from "../types"; +import { ConnectionActionsMenu } from "./connectionActionsMenu"; +import { + getConnectionSyncAnnotation, + SyncAnnotation, +} from "./syncAnnotation"; +const POLL_INTERVAL_MS = 5_000; export type Connection = { - id: number - name: string - syncedAt: Date | null - connectionType: ConnectionType - latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null - isFirstTimeSync: boolean -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) + id: number; + name: string; + connectionType: ConnectionType; + syncedAt: Date | null; + latestJob: WorkloadJob<"connection-sync"> | null; +}; + +type DisplayedConnection = Connection & { + showCompleted: boolean; +}; -const getStatusBadge = (status: Connection["latestJobStatus"]) => { - if (!status) { - return "-"; +type SortBy = "name" | "syncedAt"; +type SortOrder = "asc" | "desc"; +type StatusFilter = "all" | "failed" | "warning"; + +const getStatusFilter = (value: string | null): StatusFilter => { + if (value === "failed" || value === "warning") { + return value; } - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", + return "all"; +}; + +const getConnectionSyncStatuses = async ( + connectionIds: number[], + signal: AbortSignal, +): Promise => { + const response = await fetch("/api/connection-sync-status", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ connectionIds }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load connection sync statuses"); } - return {labels[status]} -} + return response.json() as Promise; +}; + +const SortableHeader = ({ + label, + column, + sortBy, + sortOrder, + onSortChange, +}: { + label: string; + column: SortBy; + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; +}) => { + const isActive = sortBy === column; + const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; -export const columns: ColumnDef[] = [ + return ( + + ); +}; + +const getColumns = ({ + sortBy, + sortOrder, + onSortChange, + onSyncScheduled, +}: { + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + onSyncScheduled: (connectionId: number, jobId: string) => void; +}): ColumnDef[] => [ { accessorKey: "name", - size: 400, - header: ({ column }) => { - return ( - - ) - }, + header: () => ( + + ), cell: ({ row }) => { const connection = row.original; const codeHostIcon = getCodeHostIcon(connection.connectionType); return ( -
+
{`${connection.connectionType} - + {connection.name} - - {connection.isFirstTimeSync && ( - - - - - - - - This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. - - - )} + +
- ) + ); }, }, { - accessorKey: "latestJobStatus", - size: 150, - header: "Lastest status", - cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), + accessorKey: "syncedAt", + header: () => ( + + ), + cell: ({ row }) => row.original.syncedAt + ? + : "-", }, { - accessorKey: "syncedAt", - size: 200, - header: ({ column }) => { - return ( - - ) + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ +
+ ), + }, +]; + +type ConnectionsTableProps = { + data: Connection[]; + currentPage: number; + pageSize: number; + totalCount: number; + sortBy: SortBy; + sortOrder: SortOrder; +}; + +export const ConnectionsTable = ({ + data, + currentPage, + pageSize, + totalCount, + sortBy, + sortOrder, +}: ConnectionsTableProps) => { + const pathname = usePathname(); + const router = useRouter(); + const searchParams = useSearchParams(); + const searchParamsString = searchParams.toString(); + const urlSearchValue = searchParams.get("search") ?? ""; + const statusFilter = getStatusFilter(searchParams.get("status")); + const [searchValue, setSearchValue] = useState(urlSearchValue); + const [scheduledSyncJobs, setScheduledSyncJobs] = useState< + Map> + >(() => new Map()); + const debouncedSearchValue = useDebounce(searchValue, 300); + const [isSearchNavigationPending, startSearchTransition] = useTransition(); + const pendingSearchValuesRef = useRef>(new Set()); + const searchInputRef = useRef(null); + const isSearchPending = searchValue !== debouncedSearchValue + || isSearchNavigationPending; + + useHotkeys("/", (event) => { + event.preventDefault(); + searchInputRef.current?.focus(); + }); + + useEffect(() => { + if (pendingSearchValuesRef.current.delete(urlSearchValue)) { + return; + } + + setSearchValue(urlSearchValue); + }, [urlSearchValue]); + + useEffect(() => { + if (debouncedSearchValue !== searchValue) { + return; + } + + const nextSearchValue = debouncedSearchValue.trim(); + if (nextSearchValue === urlSearchValue) { + return; + } + + const params = new URLSearchParams(searchParamsString); + if (nextSearchValue) { + params.set("search", nextSearchValue); + } else { + params.delete("search"); + } + params.delete("page"); + + const nextSearchParamsString = params.toString(); + if (nextSearchParamsString === searchParamsString) { + return; + } + + pendingSearchValuesRef.current.add(nextSearchValue); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }, [ + debouncedSearchValue, + pathname, + router, + searchParamsString, + searchValue, + urlSearchValue, + ]); + + const onSyncScheduled = useCallback(( + connectionId: number, + jobId: string, + ) => { + setScheduledSyncJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + nextJobs.set(connectionId, { + id: jobId, + data: { connectionId }, + status: "PENDING", + errorMessage: null, + result: null, + }); + return nextJobs; + }); + }, []); + const syncAwareData = useMemo( + () => data.map((connection) => { + const scheduledJob = scheduledSyncJobs.get(connection.id); + return scheduledJob + ? { ...connection, latestJob: scheduledJob } + : connection; + }), + [data, scheduledSyncJobs], + ); + const pollingTargets = useMemo( + () => syncAwareData.flatMap((connection) => { + const latestJob = connection.latestJob; + if ( + !latestJob + || getConnectionSyncAnnotation( + connection.id, + latestJob, + connection.syncedAt, + ) !== "SYNCING" + ) { + return []; + } + + return [{ connectionId: connection.id, jobId: latestJob.id }]; + }), + [syncAwareData], + ); + const pollingConnectionIds = useMemo( + () => pollingTargets.map(({ connectionId }) => connectionId), + [pollingTargets], + ); + const pollingKey = useMemo( + () => pollingTargets.map(({ connectionId, jobId }) => + `${connectionId}:${jobId}` + ), + [pollingTargets], + ); + const { data: polledStatuses } = useQuery({ + queryKey: [ + "connections-sync-status", + pollingConnectionIds, + pollingKey, + ], + queryFn: ({ signal }) => + getConnectionSyncStatuses(pollingConnectionIds, signal), + enabled: pollingConnectionIds.length > 0, + placeholderData: (previousData) => previousData, + refetchInterval: (query) => { + const statuses = query.state.data?.connections; + if (!statuses) { + return POLL_INTERVAL_MS; + } + + return pollingTargets.some((target) => { + const status = statuses.find( + ({ connectionId }) => + connectionId === target.connectionId, + ); + if (!status || status.latestJob?.id !== target.jobId) { + return true; + } + + return status.latestJob.status === "PENDING" + || status.latestJob.status === "IN_PROGRESS"; + }) + ? POLL_INTERVAL_MS + : false; }, - cell: ({ row }) => { - const syncedAt = row.getValue("syncedAt") as Date | null; - if (!syncedAt) { - return "-"; + }); + const completedDuringPollingConnectionIds = useMemo(() => { + const pollingTargetsByConnectionId = new Map( + pollingTargets.map((target) => [target.connectionId, target]), + ); + return new Set( + polledStatuses?.connections.flatMap((status) => { + const target = pollingTargetsByConnectionId.get( + status.connectionId, + ); + return target + && status.latestJob?.id === target.jobId + && status.latestJob.status === "COMPLETED" + && status.syncedAt + ? [status.connectionId] + : []; + }) ?? [], + ); + }, [polledStatuses, pollingTargets]); + const displayedData = useMemo(() => { + const statusesByConnectionId = new Map( + polledStatuses?.connections.map((status) => [ + status.connectionId, + status, + ]) ?? [], + ); + + return syncAwareData.map((connection): DisplayedConnection => { + const showCompleted = completedDuringPollingConnectionIds.has( + connection.id, + ); + const status = statusesByConnectionId.get(connection.id); + const scheduledJob = scheduledSyncJobs.get(connection.id); + if (!status) { + return { ...connection, showCompleted }; + } + if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { + return { + ...connection, + syncedAt: status.syncedAt + ? new Date(status.syncedAt) + : connection.syncedAt, + showCompleted, + }; } - return ( - - ) - } - }, -] + return { + ...connection, + syncedAt: status.syncedAt + ? new Date(status.syncedAt) + : connection.syncedAt, + latestJob: status.latestJob, + showCompleted, + }; + }); + }, [ + completedDuringPollingConnectionIds, + polledStatuses, + scheduledSyncJobs, + syncAwareData, + ]); -export const ConnectionsTable = ({ data }: { data: Connection[] }) => { - const [sorting, setSorting] = useState([]) - const [columnFilters, setColumnFilters] = useState([]) - const [columnVisibility, setColumnVisibility] = useState({}) - const [rowSelection, setRowSelection] = useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - numNoJobs, - } = useMemo(() => { - return { - numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, - numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, - numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, - numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, - numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, + const onSortChange = useCallback((column: SortBy) => { + const params = new URLSearchParams(searchParams.toString()); + const nextSortOrder = sortBy === column && sortOrder === "asc" + ? "desc" + : "asc"; + params.delete("page"); + if (column === "name") { + params.delete("sortBy"); + } else { + params.set("sortBy", column); + } + if (nextSortOrder === "asc") { + params.delete("sortOrder"); + } else { + params.set("sortOrder", nextSortOrder); } - }, [data]); + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }, [pathname, router, searchParams, sortBy, sortOrder]); + const columns = useMemo( + () => getColumns({ + sortBy, + sortOrder, + onSortChange, + onSyncScheduled, + }), + [onSortChange, onSyncScheduled, sortBy, sortOrder], + ); const table = useReactTable({ - data, + data: displayedData, columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - columnResizeMode: 'onChange', - enableColumnResizing: false, + manualPagination: true, + manualSorting: true, + rowCount: totalCount, state: { - sorting, - columnFilters, - columnVisibility, - rowSelection, + pagination: { + pageIndex: currentPage - 1, + pageSize, + }, + sorting: [{ id: sortBy, desc: sortOrder === "desc" }], }, - }) + }); + const totalPages = Math.max(1, table.getPageCount()); + const firstVisibleConnection = totalCount === 0 + ? 0 + : (currentPage - 1) * pageSize + 1; + const lastVisibleConnection = Math.min(currentPage * pageSize, totalCount); + const hasActiveFilters = statusFilter !== "all" + || urlSearchValue.trim().length > 0; + + const goToPage = (page: number) => { + const params = new URLSearchParams(searchParams.toString()); + if (page === 1) { + params.delete("page"); + } else { + params.set("page", page.toString()); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }; + + const onStatusFilterChange = (status: StatusFilter) => { + const params = new URLSearchParams(searchParams.toString()); + if (status === "all") { + params.delete("status"); + } else { + params.set("status", status); + } + params.delete("page"); + + const query = params.toString(); + router.replace(query ? `${pathname}?${query}` : pathname, { + scroll: false, + }); + }; + + const clearFilters = () => { + const params = new URLSearchParams(searchParamsString); + params.delete("search"); + params.delete("status"); + params.delete("page"); + + const nextSearchParamsString = params.toString(); + setSearchValue(""); + pendingSearchValuesRef.current.add(""); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }; + + const emptyMessage = statusFilter === "failed" + ? "No failed connections." + : statusFilter === "warning" + ? "No connections with warnings." + : "No connections found."; return ( -
-
- table.getColumn("name")?.setFilterValue(event.target.value)} - className="max-w-sm" - /> +
+
+ + + + + setSearchValue(event.target.value)} + placeholder="Search connections..." + /> + {isSearchPending && ( + + + + )} + - + {hasActiveFilters && ( + + )}
- +
{table.getHeaderGroups().map((headerGroup) => ( - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} ))} - {table.getRowModel().rows?.length ? ( + {table.getRowModel().rows.length > 0 ? ( table.getRowModel().rows.map((row) => ( - + {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} ))} )) ) : ( - - No results. + +

{emptyMessage}

 )}
-
-
- {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total -
-
- - + {totalCount > 0 && ( +
+

+ Showing {firstVisibleConnection}-{lastVisibleConnection} of {totalCount} +

+
+

+ Page {currentPage} of {totalPages} +

+
+ + +
+
-
+ )}
- ) -} + ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/components/syncAnnotation.tsx b/packages/web/src/app/(app)/settings/connections/components/syncAnnotation.tsx new file mode 100644 index 000000000..6a7fca507 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/syncAnnotation.tsx @@ -0,0 +1,170 @@ +import { Badge } from "@/components/ui/badge"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { Check, Loader2 } from "lucide-react"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; +import { useEffect, useState } from "react"; +import { SyncIssuePopover } from "./syncIssuePopover"; + +const COMPLETED_BADGE_VISIBLE_MS = 5_000; + +type SyncAnnotationProps = { + connectionId: number; + latestJob: WorkloadJob<"connection-sync"> | null; + showCompleted: boolean; + connectionName: string; + syncedAt: Date | null; + onRetryScheduled: (connectionId: number, jobId: string) => void; +}; + +export type ConnectionSyncAnnotation = + | "SYNCING" + | "WARNING" + | "FAILED" + | null; + +export const getConnectionSyncAnnotation = ( + connectionId: number, + latestJob: WorkloadJob<"connection-sync"> | null, + syncedAt: Date | null, +): ConnectionSyncAnnotation => { + if (!latestJob || latestJob.data.connectionId !== connectionId) { + return null; + } + + if ( + latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ) { + return "SYNCING"; + } + + if (latestJob.status === "FAILED") { + return syncedAt ? "WARNING" : "FAILED"; + } + + if ( + latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ) { + return "WARNING"; + } + + return null; +}; + +export const SyncAnnotation = ({ + connectionId, + latestJob, + showCompleted, + connectionName, + syncedAt, + onRetryScheduled, +}: SyncAnnotationProps) => { + const prefersReducedMotion = useReducedMotion(); + const completionKey = showCompleted + ? latestJob?.id ?? `connection:${connectionId}` + : null; + const [expiredCompletionKey, setExpiredCompletionKey] = useState< + string | null + >(null); + useEffect(() => { + if (!completionKey || expiredCompletionKey === completionKey) { + return; + } + + const timeout = window.setTimeout(() => { + setExpiredCompletionKey(completionKey); + }, COMPLETED_BADGE_VISIBLE_MS); + return () => window.clearTimeout(timeout); + }, [completionKey, expiredCompletionKey]); + const annotation = completionKey !== null + && expiredCompletionKey !== completionKey + ? "COMPLETED" + : getConnectionSyncAnnotation(connectionId, latestJob, syncedAt); + const badge = (() => { + switch (annotation) { + case "COMPLETED": + return ( + + + Completed + + ); + case "SYNCING": + return ( + + + Syncing + + ); + case "FAILED": + return latestJob + ? ( + + ) + : null; + case "WARNING": + return latestJob + ? ( + + ) + : null; + default: + return null; + } + })(); + const isCompleted = annotation === "COMPLETED"; + + return ( + + {annotation && badge && ( + + {badge} + + )} + + ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx new file mode 100644 index 000000000..3eb77d9ad --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx @@ -0,0 +1,276 @@ +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { JobLogsDialog } from "@/app/(app)/components/jobLogsDialog"; +import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter"; +import { useToast } from "@/components/hooks/use-toast"; +import { badgeVariants } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { syncConnection } from "@/features/connections/actions"; +import { cn, isServiceError } from "@/lib/utils"; +import type { + RepositoryDiscoveryIssue, + WorkloadJob, +} from "@sourcebot/shared"; +import { + ChevronDown, + CircleX, + Loader2, + RotateCw, + ScrollText, + TriangleAlert, +} from "lucide-react"; +import { useState } from "react"; + +type SyncIssuePopoverProps = { + connection: { + id: number; + name: string; + syncedAt: Date | null; + }; + latestJob: WorkloadJob<"connection-sync">; + annotation: "WARNING" | "FAILED"; + onRetryScheduled: (connectionId: number, jobId: string) => void; +}; + +const DiscoveryIssues = ({ + issues, +}: { + issues: RepositoryDiscoveryIssue[]; +}) => ( +
+
+ {issues.map((issue, index) => ( +
+ {issue.subject && ( +
+ + {issue.subject.kind} + + + {issue.subject.value} + +
+ )} +

+ {issue.message} +

+
+ ))} +
+
+); + +export const SyncIssuePopover = ({ + connection, + latestJob, + annotation, + onRetryScheduled, +}: SyncIssuePopoverProps) => { + const [isOpen, setIsOpen] = useState(false); + const [isLogsOpen, setIsLogsOpen] = useState(false); + const [isRetrying, setIsRetrying] = useState(false); + const { toast } = useToast(); + const isWarning = annotation === "WARNING"; + const reasons = latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ? latestJob.result.reasons + : null; + if ( + latestJob.status !== "FAILED" + && (!isWarning || !reasons) + ) { + return null; + } + + const label = isWarning ? "Warning" : "Failed"; + const Icon = isWarning ? TriangleAlert : CircleX; + + const retrySync = async () => { + setIsRetrying(true); + + try { + const response = await syncConnection(connection.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: response.message, + }); + return; + } + + onRetryScheduled(connection.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${connection.name} was queued for syncing.`, + }); + setIsOpen(false); + } catch { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsRetrying(false); + } + }; + + return ( + <> + + + + + + + + +

+ {isWarning + ? "View warning details" + : "View failure details"} +

+ + + +
+ +
+
+

+ {reasons + ? "Connection sync completed with warnings" + : isWarning + ? "Latest connection sync failed" + : "Connection sync failed"} +

+

+ {reasons + ? "Sourcebot could not honor the full configured discovery scope. Some repositories may be missing." + : isWarning + ? "Previously discovered repositories remain available, but they may be stale." + : "This connection failed to sync. Its repositories are unavailable."} +

+
+ {reasons + ? + : ( +
+

+ Error +

+
+ + {latestJob.errorMessage + ?? "No error details were reported."} + +
+
+ )} +
+ {connection.syncedAt && ( + <> +
+ Last synced +
+
+ +
+ + )} +
Job ID
+
+ {latestJob.id} +
+
+
+ + +
+
+
+ + + + + ); +}; diff --git a/packages/web/src/app/(app)/settings/connections/layout.tsx b/packages/web/src/app/(app)/settings/connections/layout.tsx index c89c498e6..feb4a09a8 100644 --- a/packages/web/src/app/(app)/settings/connections/layout.tsx +++ b/packages/web/src/app/(app)/settings/connections/layout.tsx @@ -2,6 +2,9 @@ import { authenticatedPage } from "@/middleware/authenticatedPage"; import { OrgRole } from "@sourcebot/db"; import { SettingsContainer } from "../components/settingsContainer"; -export default authenticatedPage<{ children: React.ReactNode }>(async (_auth, { children }) => { - return {children}; -}, { minRole: OrgRole.OWNER, redirectTo: '/settings' }); +export default authenticatedPage<{ children: React.ReactNode }>( + async (_auth, { children }) => ( + {children} + ), + { minRole: OrgRole.OWNER, redirectTo: "/settings" }, +); diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index fdce7b08f..5b1a3988f 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -1,77 +1,157 @@ -import { sew } from "@/middleware/sew"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { authenticatedPage } from "@/middleware/authenticatedPage"; +import { OrgRole, type Prisma } from "@sourcebot/db"; +import { + CONNECTION_QUEUE, + type WorkloadJob, +} from "@sourcebot/shared"; import Link from "next/link"; +import { z } from "zod"; import { ConnectionsTable } from "./components/connectionsTable"; -import { ConnectionSyncJobStatus } from "@prisma/client"; const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; +const DEFAULT_PAGE_SIZE = 20; +const pageSchema = z.coerce.number().int().positive(); +const sortBySchema = z.enum(["name", "syncedAt"]); +const sortOrderSchema = z.enum(["asc", "desc"]); +const statusSchema = z.enum(["failed", "warning"]); -export default async function ConnectionsPage() { - const _connections = await getConnectionsWithLatestJob(); - if (isServiceError(_connections)) { - throw new ServiceErrorException(_connections); - } +type ConnectionsPageProps = { + searchParams: Promise<{ + page?: string; + search?: string; + status?: string; + sortBy?: string; + sortOrder?: string; + }>; +}; - // Sort connections so that first time syncs are at the top. - const connections = _connections - .map((connection) => ({ - ...connection, - isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, - latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, - })) - .sort((a, b) => { - if (a.isFirstTimeSync && !b.isFirstTimeSync) { - return -1; - } - if (!a.isFirstTimeSync && b.isFirstTimeSync) { - return 1; +export default authenticatedPage(async ( + { org, prisma }, + { searchParams }, +) => { + const params = await searchParams; + const page = pageSchema.safeParse(params.page).data ?? 1; + const search = z.string().optional().safeParse(params.search).data?.trim() + ?? ""; + const status = statusSchema.safeParse(params.status).data ?? "all"; + const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; + const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const skip = (page - 1) * DEFAULT_PAGE_SIZE; + const baseWhere: Prisma.ConnectionWhereInput = { + orgId: org.id, + ...(search + ? { + name: { + contains: search, + mode: "insensitive" as const, + }, + } + : {}), + }; + let where = baseWhere; + let latestJobs = new Map< + string, + WorkloadJob<"connection-sync"> | null + >(); + if (status !== "all") { + const candidates = await prisma.connection.findMany({ + where: baseWhere, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + }, + }); + const candidateJobIds = candidates.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + candidateJobIds, + ); + const matchingConnectionIds = candidates.flatMap((connection) => { + if (!connection.latestSyncJobId) { + return []; } - return a.name.localeCompare(b.name); + + const job = latestJobs.get(connection.latestSyncJobId); + const matches = status === "failed" + ? job?.status === "FAILED" && connection.syncedAt === null + : (job?.status === "FAILED" && connection.syncedAt !== null) + || (job?.status === "COMPLETED" + && job.result?.outcome === "PARTIAL_SUCCESS"); + return matches ? [connection.id] : []; }); + where = { + ...baseWhere, + id: { in: matchingConnectionIds }, + }; + } + const orderBy: Prisma.ConnectionOrderByWithRelationInput[] = sortBy === "syncedAt" + ? [{ syncedAt: sortOrder }, { id: "asc" }] + : [{ name: sortOrder }, { id: "asc" }]; + + const [connections, totalCount] = await Promise.all([ + prisma.connection.findMany({ + where, + orderBy, + skip, + take: DEFAULT_PAGE_SIZE, + select: { + id: true, + name: true, + connectionType: true, + syncedAt: true, + latestSyncJobId: true, + }, + }), + prisma.connection.count({ where }), + ]); + const latestJobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + if (status === "all") { + latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); + } return (

Code Host Connections

-

Manage your connections to external code hosts. Learn more

+

+ Manage connections to external code hosts.{" "} + + Learn more + +

- ({ - id: connection.id, - name: connection.name, - connectionType: connection.connectionType, - syncedAt: connection.syncedAt, - latestJobStatus: connection.latestJobStatus, - isFirstTimeSync: connection.isFirstTimeSync, - }))} /> + ({ + id: connection.id, + name: connection.name, + connectionType: connection.connectionType, + syncedAt: connection.syncedAt, + latestJob: connection.latestSyncJobId + ? latestJobs.get(connection.latestSyncJobId) ?? null + : null, + }))} + currentPage={page} + pageSize={DEFAULT_PAGE_SIZE} + totalCount={totalCount} + sortBy={sortBy} + sortOrder={sortOrder} + />
- ) -} - -const getConnectionsWithLatestJob = async () => sew(() => - withAuth(async ({ prisma, org }) => { - const connections = await prisma.connection.findMany({ - where: { - orgId: org.id, - }, - include: { - _count: { - select: { - syncJobs: true, - } - }, - syncJobs: { - orderBy: { - createdAt: 'desc' - }, - take: 1 - }, - }, - orderBy: { - name: 'asc' - }, - }); - - return connections; - })); \ No newline at end of file + ); +}, { + minRole: OrgRole.OWNER, + redirectTo: "/settings", +}); diff --git a/packages/web/src/app/(app)/settings/connections/types.ts b/packages/web/src/app/(app)/settings/connections/types.ts new file mode 100644 index 000000000..faf6ce997 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connections/types.ts @@ -0,0 +1,11 @@ +import type { WorkloadJob } from "@sourcebot/shared"; + +export type ConnectionSyncStatus = { + connectionId: number; + syncedAt: string | null; + latestJob: WorkloadJob<"connection-sync"> | null; +}; + +export type ConnectionSyncStatusesResponse = { + connections: ConnectionSyncStatus[]; +}; diff --git a/packages/web/src/app/(app)/settings/layout.tsx b/packages/web/src/app/(app)/settings/layout.tsx index 655ca28f5..40f53ef1b 100644 --- a/packages/web/src/app/(app)/settings/layout.tsx +++ b/packages/web/src/app/(app)/settings/layout.tsx @@ -3,7 +3,6 @@ import { Metadata } from "next" import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { OrgRole } from "@prisma/client"; @@ -48,10 +47,6 @@ export const getSidebarNavGroups = async () => numJoinRequests = requests.length; } - const connectionStats = await getConnectionStats(); - if (isServiceError(connectionStats)) { - throw new ServiceErrorException(connectionStats); - } const hasAskEntitlement = await hasEntitlement("ask"); const groups: NavGroup[] = [ @@ -120,7 +115,6 @@ export const getSidebarNavGroups = async () => title: "Connections", href: `/settings/connections`, hrefRegex: `/settings/connections(/[^/]+)?$`, - isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, icon: "plug" as const, }, { diff --git a/packages/web/src/app/api/(client)/client.ts b/packages/web/src/app/api/(client)/client.ts index 17326787e..4f0a714bd 100644 --- a/packages/web/src/app/api/(client)/client.ts +++ b/packages/web/src/app/api/(client)/client.ts @@ -36,6 +36,9 @@ import type { SearchChatShareableMembersQueryParams, SearchChatShareableMembersResponse, } from "../(server)/ee/chat/[chatId]/searchMembers/route"; +import type { JobLogs, QueueName } from "@sourcebot/shared"; +import type { RepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import type { ConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; import type { OffersResponse } from "@sourcebot/shared/client"; import { ConnectMcpResponse } from "../(server)/ee/askmcp/connect/types"; import type { GetMcpServersResponse } from "../(server)/ee/askmcp/servers/route"; @@ -260,6 +263,28 @@ export const getPermissionSyncStatus = async (): Promise => { + const result = await fetch("/api/repository-sync-counts", { + method: "GET", + headers: { + "X-Sourcebot-Client-Source": "sourcebot-web-client", + }, + }).then(response => response.json()); + + return result as RepositorySyncCounts | ServiceError; +} + +export const getConnectionSyncCounts = async (): Promise => { + const result = await fetch("/api/connection-sync-counts", { + method: "GET", + headers: { + "X-Sourcebot-Client-Source": "sourcebot-web-client", + }, + }).then(response => response.json()); + + return result as ConnectionSyncCounts | ServiceError; +} + export const getAccountSyncStatus = async (jobId: string): Promise => { const url = new URL("/api/ee/accountPermissionSyncJobStatus", window.location.origin); url.searchParams.set("jobId", jobId); @@ -395,3 +420,25 @@ export const getMcpServerToolPermissions = async (serverId: string): Promise => { + const response = await fetch("/api/job-logs", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Sourcebot-Client-Source": "sourcebot-web-client", + }, + body: JSON.stringify({ queue, jobId }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load job logs"); + } + + return response.json() as Promise; +}; diff --git a/packages/web/src/app/api/(server)/connection-sync-counts/route.ts b/packages/web/src/app/api/(server)/connection-sync-counts/route.ts new file mode 100644 index 000000000..55c6030e3 --- /dev/null +++ b/packages/web/src/app/api/(server)/connection-sync-counts/route.ts @@ -0,0 +1,17 @@ +import { getConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; +import { apiHandler } from "@/lib/apiHandler"; +import { serviceErrorResponse } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { sew } from "@/middleware/sew"; +import { StatusCodes } from "http-status-codes"; + +// eslint-disable-next-line authz/require-auth-wrapper -- Authentication and owner authorization are enforced by getConnectionSyncCounts. +export const GET = apiHandler(async () => { + const result = await sew(() => getConnectionSyncCounts()); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result, { status: StatusCodes.OK }); +}); diff --git a/packages/web/src/app/api/(server)/connection-sync-status/route.ts b/packages/web/src/app/api/(server)/connection-sync-status/route.ts new file mode 100644 index 000000000..16448c50e --- /dev/null +++ b/packages/web/src/app/api/(server)/connection-sync-status/route.ts @@ -0,0 +1,63 @@ +import type { ConnectionSyncStatusesResponse } from "@/app/(app)/settings/connections/types"; +import { apiHandler } from "@/lib/apiHandler"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + requestBodySchemaValidationError, + serviceErrorResponse, +} from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withOptionalAuth } from "@/middleware/withAuth"; +import { CONNECTION_QUEUE } from "@sourcebot/shared"; +import { z } from "zod"; + +const requestSchema = z.object({ + connectionIds: z.array(z.number().int().positive()).min(1).max(100), +}); + +export const POST = apiHandler(async (request) => { + const parsed = requestSchema.safeParse( + await request.json().catch(() => null), + ); + if (!parsed.success) { + return serviceErrorResponse( + requestBodySchemaValidationError(parsed.error), + ); + } + + const result = await withOptionalAuth(async ({ org, prisma }) => { + const connections = await prisma.connection.findMany({ + where: { + orgId: org.id, + id: { in: parsed.data.connectionIds }, + }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + }, + }); + const jobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + const jobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + jobIds, + ); + + return { + connections: connections.map((connection) => ({ + connectionId: connection.id, + syncedAt: connection.syncedAt?.toISOString() ?? null, + latestJob: connection.latestSyncJobId + ? jobs.get(connection.latestSyncJobId) ?? null + : null, + })), + } satisfies ConnectionSyncStatusesResponse; + }); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result); +}); diff --git a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts index 41715adef..1ed721985 100644 --- a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts +++ b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts @@ -2,13 +2,19 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ authContext: undefined as unknown, + getJob: vi.fn(), })); vi.mock('@/middleware/withAuth', () => ({ withAuth: vi.fn((callback: (context: unknown) => unknown) => callback(mocks.authContext)), })); +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ getJob: mocks.getJob }), +})); + vi.mock('@sourcebot/shared', () => ({ + ACCOUNT_PERMISSION_SYNC_QUEUE: { name: 'account-permission-sync' }, createLogger: () => ({ error: vi.fn() }), })); @@ -16,25 +22,70 @@ const { getAccountSyncStatus } = await import('./api'); beforeEach(() => { vi.clearAllMocks(); + mocks.getJob.mockReset(); }); describe('getAccountSyncStatus', () => { test.each(['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const)( 'returns the underlying %s job status', async (status) => { - const findFirst = vi.fn().mockResolvedValue({ status }); + const findFirst = vi.fn().mockResolvedValue({ id: 'account_1' }); + mocks.getJob.mockResolvedValue({ + id: 'job_1', + data: { accountId: 'account_1' }, + status, + errorMessage: null, + }); mocks.authContext = { user: { id: 'user_1' }, - prisma: { accountPermissionSyncJob: { findFirst } }, + prisma: { account: { findFirst } }, }; await expect(getAccountSyncStatus('job_1')).resolves.toEqual({ status }); + expect(mocks.getJob).toHaveBeenCalledWith( + { name: 'account-permission-sync' }, + 'job_1', + ); expect(findFirst).toHaveBeenCalledWith({ where: { - id: 'job_1', - account: { userId: 'user_1' }, + id: 'account_1', + userId: 'user_1', }, + select: { id: true }, }); }, ); + + test('does not expose a job belonging to another user', async () => { + mocks.getJob.mockResolvedValue({ + id: 'job_1', + data: { accountId: 'account_1' }, + status: 'IN_PROGRESS', + errorMessage: null, + }); + mocks.authContext = { + user: { id: 'user_1' }, + prisma: { + account: { findFirst: vi.fn().mockResolvedValue(null) }, + }, + }; + + await expect(getAccountSyncStatus('job_1')).resolves.toMatchObject({ + statusCode: 404, + }); + }); + + test('returns not found when Redis no longer has the job', async () => { + const findFirst = vi.fn(); + mocks.getJob.mockResolvedValue(null); + mocks.authContext = { + user: { id: 'user_1' }, + prisma: { account: { findFirst } }, + }; + + await expect(getAccountSyncStatus('missing')).resolves.toMatchObject({ + statusCode: 404, + }); + expect(findFirst).not.toHaveBeenCalled(); + }); }); diff --git a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts index e92c549d1..e3c034e16 100644 --- a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts +++ b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts @@ -1,24 +1,38 @@ 'use server'; import { ServiceError, notFound } from "@/lib/serviceError"; +import { getBullMQClient } from "@/lib/bullmqClient"; import { withAuth } from "@/middleware/withAuth"; -import { AccountPermissionSyncJobStatus } from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + type WorkloadJobStatus, +} from "@sourcebot/shared"; import { sew } from "@/middleware/sew"; export interface AccountSyncStatusResponse { - status: AccountPermissionSyncJobStatus; + status: WorkloadJobStatus; } export const getAccountSyncStatus = async (jobId: string): Promise => sew(() => withAuth(async ({ prisma, user }) => { - const job = await prisma.accountPermissionSyncJob.findFirst({ + const job = await getBullMQClient().getJob( + ACCOUNT_PERMISSION_SYNC_QUEUE, + jobId, + ); + if (!job) { + return notFound(); + } + + const account = await prisma.account.findFirst({ where: { - id: jobId, - account: { userId: user.id }, + id: job.data.accountId, + userId: user.id, }, + select: { id: true }, }); - - if (!job) return notFound(); + if (!account) { + return notFound(); + } return { status: job.status } satisfies AccountSyncStatusResponse; })); diff --git a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts index 4cf5dbe8c..c28765c78 100644 --- a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts +++ b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ authContext: undefined as unknown, getEntitlements: vi.fn(), + getJobs: vi.fn(), })); vi.mock('@/middleware/withAuth', () => ({ @@ -13,8 +14,13 @@ vi.mock('@/lib/entitlements', () => ({ getEntitlements: mocks.getEntitlements, })); +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ getJobs: mocks.getJobs }), +})); + vi.mock('@sourcebot/shared', () => ({ createLogger: () => ({ error: vi.fn() }), + ACCOUNT_PERMISSION_SYNC_QUEUE: { name: 'account-permission-sync' }, env: { PERMISSION_SYNC_ENABLED: 'true' }, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ 'github', @@ -29,6 +35,7 @@ const { getPermissionSyncStatus } = await import('./api'); beforeEach(() => { vi.clearAllMocks(); mocks.getEntitlements.mockResolvedValue(['permission-syncing']); + mocks.getJobs.mockResolvedValue(new Map()); }); describe('getPermissionSyncStatus', () => { @@ -41,7 +48,7 @@ describe('getPermissionSyncStatus', () => { permissionSyncedAt: null, permissionSyncIssue: null, permissionSyncIssueAt: null, - permissionSyncJobs: [{ status: 'PENDING' }], + latestPermissionSyncJobId: 'job_pending', }, { id: 'account_action_required', @@ -50,9 +57,23 @@ describe('getPermissionSyncStatus', () => { permissionSyncedAt: new Date('2026-07-01T00:00:00Z'), permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', permissionSyncIssueAt: new Date('2026-07-22T12:00:00Z'), - permissionSyncJobs: [{ status: 'PENDING' }], + latestPermissionSyncJobId: 'job_recovery', }, ]); + mocks.getJobs.mockResolvedValue(new Map([ + ['job_pending', { + id: 'job_pending', + data: { accountId: 'account_pending' }, + status: 'PENDING', + errorMessage: null, + }], + ['job_recovery', { + id: 'job_recovery', + data: { accountId: 'account_action_required' }, + status: 'IN_PROGRESS', + errorMessage: null, + }], + ])); mocks.authContext = { user: { id: 'user_1' }, prisma: { account: { findMany } }, @@ -72,6 +93,10 @@ describe('getPermissionSyncStatus', () => { expect(findMany).toHaveBeenCalledWith(expect.objectContaining({ where: expect.objectContaining({ userId: 'user_1' }), })); + expect(mocks.getJobs).toHaveBeenCalledWith( + { name: 'account-permission-sync' }, + ['job_pending', 'job_recovery'], + ); }); test('returns an issue even when the account has no issue timestamp', async () => { @@ -86,14 +111,44 @@ describe('getPermissionSyncStatus', () => { permissionSyncedAt: new Date('2026-07-01T00:00:00Z'), permissionSyncIssue: 'INSUFFICIENT_SCOPE', permissionSyncIssueAt: null, - permissionSyncJobs: [{ status: 'FAILED' }], + latestPermissionSyncJobId: 'job_failed', }]), }, }, }; + mocks.getJobs.mockResolvedValue(new Map([['job_failed', { + id: 'job_failed', + data: { accountId: 'account_1' }, + status: 'FAILED', + errorMessage: 'Insufficient scope', + }]])); await expect(getPermissionSyncStatus()).resolves.toMatchObject({ issues: [{ reason: 'INSUFFICIENT_SCOPE', occurredAt: null, isSyncing: false }], }); }); + + test('treats a missing first-sync job as pending', async () => { + mocks.authContext = { + user: { id: 'user_1' }, + prisma: { + account: { + findMany: vi.fn().mockResolvedValue([{ + id: 'account_1', + providerId: 'github', + providerType: 'github', + permissionSyncedAt: null, + permissionSyncIssue: null, + permissionSyncIssueAt: null, + latestPermissionSyncJobId: null, + }]), + }, + }, + }; + + await expect(getPermissionSyncStatus()).resolves.toMatchObject({ + hasPendingFirstSync: true, + }); + expect(mocks.getJobs).not.toHaveBeenCalled(); + }); }); diff --git a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts index 804016903..2c1e0708d 100644 --- a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts +++ b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts @@ -1,10 +1,17 @@ 'use server'; import { ServiceError } from "@/lib/serviceError"; +import { getBullMQClient } from "@/lib/bullmqClient"; import { withAuth } from "@/middleware/withAuth"; import { getEntitlements } from "@/lib/entitlements"; -import { env, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS } from "@sourcebot/shared"; -import { AccountPermissionSyncJobStatus, type AccountPermissionSyncIssue } from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + env, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, + type WorkloadJob, + type WorkloadJobStatus, +} from "@sourcebot/shared"; +import type { AccountPermissionSyncIssue } from "@sourcebot/db"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "@/lib/errorCodes"; import { sew } from "@/middleware/sew"; @@ -21,6 +28,9 @@ export interface PermissionSyncStatusResponse { }>; } +const isActiveStatus = (status: WorkloadJobStatus | undefined) => + status === "PENDING" || status === "IN_PROGRESS"; + /** * Returns initial-sync progress and action-required permission sync issues * for the authenticated user's linked accounts. @@ -49,35 +59,67 @@ export const getPermissionSyncStatus = async (): Promise + account.latestPermissionSyncJobId + ? [account.latestPermissionSyncJobId] + : [], + ); + const latestJobs = latestJobIds.length > 0 + ? await getBullMQClient().getJobs( + ACCOUNT_PERMISSION_SYNC_QUEUE, + latestJobIds, + ) + : new Map | null>(); + const latestJobsByAccountId = new Map< + string, + WorkloadJob<"account-permission-sync"> | null + >( + accounts.map((account): [ + string, + WorkloadJob<"account-permission-sync"> | null, + ] => { + const job = account.latestPermissionSyncJobId + ? latestJobs.get(account.latestPermissionSyncJobId) ?? null + : null; + return [ + account.id, + job?.data.accountId === account.id ? job : null, + ]; + }), + ); const hasPendingFirstSync = env.PERMISSION_SYNC_ENABLED === 'true' && - accounts.some(account => - account.permissionSyncedAt === null && - // @note: to handle the case where the permission sync job - // has not yet been scheduled for a new account, we consider - // accounts with no permission sync jobs as having a pending first sync. - (account.permissionSyncJobs.length === 0 || (account.permissionSyncJobs.length > 0 && activeStatuses.includes(account.permissionSyncJobs[0].status))) - ); + accounts.some((account) => { + const latestJob = latestJobsByAccountId.get(account.id); + return ( + account.permissionSyncedAt === null + // @note: to handle the case where the permission sync job + // has not yet been scheduled for a new account, we consider + // accounts with no available job as having a pending first sync. + && (!latestJob || isActiveStatus(latestJob.status)) + ); + }); + + const issues = accounts.flatMap((account) => { + if (account.permissionSyncIssue === null) { + return []; + } - const issues = accounts.flatMap(account => account.permissionSyncIssue === null ? [] : [{ - accountId: account.id, - providerId: account.providerId, - providerType: account.providerType, - reason: account.permissionSyncIssue, - occurredAt: account.permissionSyncIssueAt?.toISOString() ?? null, - isSyncing: account.permissionSyncJobs.some(job => activeStatuses.includes(job.status)), - }]); + return [{ + accountId: account.id, + providerId: account.providerId, + providerType: account.providerType, + reason: account.permissionSyncIssue, + occurredAt: account.permissionSyncIssueAt?.toISOString() ?? null, + isSyncing: isActiveStatus( + latestJobsByAccountId.get(account.id)?.status, + ), + }]; + }); return { hasPendingFirstSync, issues } satisfies PermissionSyncStatusResponse; }) diff --git a/packages/web/src/app/api/(server)/job-logs/route.ts b/packages/web/src/app/api/(server)/job-logs/route.ts new file mode 100644 index 000000000..5704bd493 --- /dev/null +++ b/packages/web/src/app/api/(server)/job-logs/route.ts @@ -0,0 +1,73 @@ +import { apiHandler } from "@/lib/apiHandler"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + notFound, + requestBodySchemaValidationError, + serviceErrorResponse, +} from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { OrgRole } from "@sourcebot/db"; +import { + QUEUE_SPECS, + type QueueName, + type QueueSpec, +} from "@sourcebot/shared"; +import { z } from "zod"; + +const queueSchema = z.custom( + (value) => + typeof value === "string" && Object.hasOwn(QUEUE_SPECS, value), + "Unsupported queue", +); + +const requestSchema = z.object({ + queue: queueSchema, + jobId: z.string().min(1).max(200), +}); + +const getJobLogs = async ( + spec: QueueSpec, + jobId: string, +) => { + const client = getBullMQClient(); + const job = await client.getJob(spec, jobId); + if (!job) { + return null; + } + + return client.getJobLogs(spec, jobId, { ascending: true }); +}; + +export const POST = apiHandler(async (request) => { + const parsed = requestSchema.safeParse( + await request.json().catch(() => null), + ); + if (!parsed.success) { + return serviceErrorResponse( + requestBodySchemaValidationError(parsed.error), + ); + } + + // TODO(SINGLE_TENANT_ORG_ID migration): Before supporting multiple organizations, + // map each queue's job data to an organization-scoped record and verify that the + // requested job belongs to the authenticated organization before returning logs. + const result = await withAuth(({ role }) => + withMinimumOrgRole(role, OrgRole.OWNER, () => + getJobLogs( + QUEUE_SPECS[parsed.data.queue], + parsed.data.jobId, + ) + ) + ); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + if (!result) { + return serviceErrorResponse(notFound("Job not found")); + } + + return Response.json(result); +}); diff --git a/packages/web/src/app/api/(server)/repo-index-status/route.ts b/packages/web/src/app/api/(server)/repo-index-status/route.ts new file mode 100644 index 000000000..5f5b29830 --- /dev/null +++ b/packages/web/src/app/api/(server)/repo-index-status/route.ts @@ -0,0 +1,65 @@ +import type { RepoIndexingStatusesResponse } from "@/app/(app)/repos/types"; +import { apiHandler } from "@/lib/apiHandler"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + requestBodySchemaValidationError, + serviceErrorResponse, +} from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withOptionalAuth } from "@/middleware/withAuth"; +import { REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { z } from "zod"; + +const requestSchema = z.object({ + repoIds: z.array(z.number().int().positive()).min(1).max(100), +}); + +export const POST = apiHandler(async (request) => { + const parsed = requestSchema.safeParse( + await request.json().catch(() => null), + ); + if (!parsed.success) { + return serviceErrorResponse( + requestBodySchemaValidationError(parsed.error), + ); + } + + const result = await withOptionalAuth(async ({ org, prisma }) => { + const repositories = await prisma.repo.findMany({ + where: { + orgId: org.id, + id: { in: parsed.data.repoIds }, + }, + select: { + id: true, + indexedAt: true, + indexedCommitHash: true, + latestIndexingJobId: true, + }, + }); + const jobIds = repositories.flatMap((repo) => + repo.latestIndexingJobId ? [repo.latestIndexingJobId] : [], + ); + const jobs = await getBullMQClient().getJobs( + REPO_INDEX_QUEUE, + jobIds, + ); + + return { + repositories: repositories.map((repo) => ({ + repoId: repo.id, + indexedAt: repo.indexedAt?.toISOString() ?? null, + indexedCommitHash: repo.indexedCommitHash, + latestJob: repo.latestIndexingJobId + ? jobs.get(repo.latestIndexingJobId) ?? null + : null, + })), + } satisfies RepoIndexingStatusesResponse; + }); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result); +}); diff --git a/packages/web/src/app/api/(server)/repository-sync-counts/route.ts b/packages/web/src/app/api/(server)/repository-sync-counts/route.ts new file mode 100644 index 000000000..b82e58063 --- /dev/null +++ b/packages/web/src/app/api/(server)/repository-sync-counts/route.ts @@ -0,0 +1,17 @@ +import { apiHandler } from "@/lib/apiHandler"; +import { serviceErrorResponse } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { sew } from "@/middleware/sew"; +import { getRepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import { StatusCodes } from "http-status-codes"; + +// eslint-disable-next-line authz/require-auth-wrapper -- Authentication and owner authorization are enforced by getRepositorySyncCounts. +export const GET = apiHandler(async () => { + const result = await sew(() => getRepositorySyncCounts()); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result, { status: StatusCodes.OK }); +}); diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.test.ts b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts new file mode 100644 index 000000000..c5c40bbc0 --- /dev/null +++ b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts @@ -0,0 +1,101 @@ +import type { WorkloadJob } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const findMany = vi.fn(); + return { + findMany, + getJobs: vi.fn(), + prisma: { + connection: { findMany }, + }, + }; +}); + +vi.mock("server-only", () => ({})); +vi.mock("@/middleware/withAuth", () => ({ + withAuth: (fn: (context: unknown) => unknown) => fn({ + org: { id: 42 }, + prisma: mocks.prisma, + role: "OWNER", + }), +})); +vi.mock("@/lib/bullmqClient", () => ({ + getBullMQClient: () => ({ + getJobs: mocks.getJobs, + }), +})); + +const { getConnectionSyncCounts } = await import( + "./connectionSyncCounts.server" +); + +const job = ( + id: string, + connectionId: number, + status: WorkloadJob<"connection-sync">["status"], + result: WorkloadJob<"connection-sync">["result"] = null, +): WorkloadJob<"connection-sync"> => ({ + id, + data: { connectionId }, + status, + errorMessage: status === "FAILED" ? "sync failed" : null, + result, +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("getConnectionSyncCounts", () => { + test("classifies never-synced failures separately from warnings", async () => { + mocks.findMany.mockResolvedValue([ + { id: 1, syncedAt: null, latestSyncJobId: "failed-first-sync", firstSyncJobFinishedAt: new Date() }, + { id: 2, syncedAt: new Date(), latestSyncJobId: "failed-resync", firstSyncJobFinishedAt: new Date() }, + { id: 3, syncedAt: new Date(), latestSyncJobId: "partial-success", firstSyncJobFinishedAt: new Date() }, + { id: 4, syncedAt: new Date(), latestSyncJobId: "success", firstSyncJobFinishedAt: new Date() }, + { id: 5, syncedAt: null, latestSyncJobId: "mismatched-job", firstSyncJobFinishedAt: new Date() }, + { id: 6, syncedAt: null, latestSyncJobId: null, firstSyncJobFinishedAt: null }, + { id: 7, syncedAt: null, latestSyncJobId: "active-first-sync", firstSyncJobFinishedAt: null }, + ]); + mocks.getJobs.mockResolvedValue(new Map([ + ["failed-first-sync", job("failed-first-sync", 1, "FAILED")], + ["failed-resync", job("failed-resync", 2, "FAILED")], + ["partial-success", job( + "partial-success", + 3, + "COMPLETED", + { outcome: "PARTIAL_SUCCESS", reasons: [] }, + )], + ["success", job("success", 4, "COMPLETED", { outcome: "SUCCESS" })], + ["mismatched-job", job("mismatched-job", 999, "FAILED")], + ["active-first-sync", job("active-first-sync", 7, "IN_PROGRESS")], + ])); + + await expect(getConnectionSyncCounts()).resolves.toEqual({ + firstTimeSyncingCount: 2, + failedCount: 1, + warningCount: 2, + }); + expect(mocks.findMany).toHaveBeenCalledWith({ + where: { orgId: 42 }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + firstSyncJobFinishedAt: true, + }, + }); + expect(mocks.getJobs).toHaveBeenCalledWith( + expect.objectContaining({ name: "connection-sync" }), + [ + "failed-first-sync", + "failed-resync", + "partial-success", + "success", + "mismatched-job", + "active-first-sync", + ], + ); + }); +}); diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.ts b/packages/web/src/features/connections/connectionSyncCounts.server.ts new file mode 100644 index 000000000..43239d377 --- /dev/null +++ b/packages/web/src/features/connections/connectionSyncCounts.server.ts @@ -0,0 +1,67 @@ +import "server-only"; + +import { getBullMQClient } from "@/lib/bullmqClient"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { OrgRole } from "@sourcebot/db"; +import { CONNECTION_QUEUE } from "@sourcebot/shared"; +import { cache } from "react"; + +export interface ConnectionSyncCounts { + firstTimeSyncingCount: number; + failedCount: number; + warningCount: number; +} + +export const getConnectionSyncCounts = cache(async () => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const connections = await prisma.connection.findMany({ + where: { orgId: org.id }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + firstSyncJobFinishedAt: true, + }, + }); + const latestJobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + const latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); + + return connections.reduce((counts, connection) => { + if ( + connection.syncedAt === null + && connection.firstSyncJobFinishedAt === null + ) { + counts.firstTimeSyncingCount += 1; + } + + const latestJob = connection.latestSyncJobId + ? latestJobs.get(connection.latestSyncJobId) + : null; + if (!latestJob || latestJob.data.connectionId !== connection.id) { + return counts; + } + + if (latestJob.status === "FAILED") { + if (connection.syncedAt) { + counts.warningCount += 1; + } else { + counts.failedCount += 1; + } + } else if ( + latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ) { + counts.warningCount += 1; + } + + return counts; + }, { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }); + }) + )); diff --git a/packages/web/src/features/repos/actions.test.ts b/packages/web/src/features/repos/actions.test.ts index c64d5c0ac..02452ba12 100644 --- a/packages/web/src/features/repos/actions.test.ts +++ b/packages/web/src/features/repos/actions.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ authContext: undefined as unknown, enqueue: vi.fn(), + getFailedJobIds: vi.fn(), })); const repoIndexQueue = { @@ -10,7 +11,10 @@ const repoIndexQueue = { }; vi.mock('@/lib/bullmqClient', () => ({ - getBullMQClient: () => ({ enqueue: mocks.enqueue }), + getBullMQClient: () => ({ + enqueue: mocks.enqueue, + getFailedJobIds: mocks.getFailedJobIds, + }), })); vi.mock('@/lib/serviceError', () => ({ unexpectedError: (message: string) => ({ error: message }), @@ -34,17 +38,21 @@ vi.mock('@sourcebot/shared', () => ({ REPO_INDEX_QUEUE: repoIndexQueue, })); -const { indexRepo } = await import('./actions'); +const { indexRepo, retryReposWithSyncIssues } = await import('./actions'); beforeEach(() => { vi.clearAllMocks(); mocks.enqueue.mockResolvedValue('job-1'); + mocks.getFailedJobIds.mockResolvedValue([]); }); -const setAuthContext = (findFirst: ReturnType) => { +const setAuthContext = ( + findFirst: ReturnType, + findMany = vi.fn().mockResolvedValue([]), +) => { mocks.authContext = { org: { id: 1 }, - prisma: { repo: { findFirst } }, + prisma: { repo: { findFirst, findMany } }, role: 'OWNER', }; }; @@ -67,7 +75,7 @@ describe('indexRepo', () => { }); expect(mocks.enqueue).toHaveBeenCalledWith( repoIndexQueue, - { repoId: 42, type: 'INDEX' }, + { repoId: 42 }, { priority: 1 }, ); }); @@ -90,3 +98,57 @@ describe('indexRepo', () => { }); }); }); + +describe('retryReposWithSyncIssues', () => { + test('retries every repository whose latest job failed', async () => { + const findMany = vi.fn().mockResolvedValue([{ id: 10 }, { id: 20 }]); + setAuthContext(vi.fn(), findMany); + mocks.getFailedJobIds.mockResolvedValue(['failed-1', 'failed-2']); + mocks.enqueue + .mockResolvedValueOnce('retry-10') + .mockResolvedValueOnce('retry-20'); + + await expect(retryReposWithSyncIssues()).resolves.toEqual({ + jobs: [ + { repoId: 10, jobId: 'retry-10' }, + { repoId: 20, jobId: 'retry-20' }, + ], + failedCount: 0, + }); + + expect(findMany).toHaveBeenCalledWith({ + where: { + orgId: 1, + latestIndexingJobId: { in: ['failed-1', 'failed-2'] }, + }, + orderBy: { id: 'asc' }, + select: { id: true }, + }); + expect(mocks.enqueue).toHaveBeenNthCalledWith( + 1, + repoIndexQueue, + { repoId: 10 }, + { priority: 1 }, + ); + expect(mocks.enqueue).toHaveBeenNthCalledWith( + 2, + repoIndexQueue, + { repoId: 20 }, + { priority: 1 }, + ); + }); + + test('returns successful jobs when part of the batch fails', async () => { + const findMany = vi.fn().mockResolvedValue([{ id: 10 }, { id: 20 }]); + setAuthContext(vi.fn(), findMany); + mocks.getFailedJobIds.mockResolvedValue(['failed-1']); + mocks.enqueue + .mockResolvedValueOnce('retry-10') + .mockRejectedValueOnce(new Error('Redis unavailable')); + + await expect(retryReposWithSyncIssues()).resolves.toEqual({ + jobs: [{ repoId: 10, jobId: 'retry-10' }], + failedCount: 1, + }); + }); +}); diff --git a/packages/web/src/features/repos/actions.ts b/packages/web/src/features/repos/actions.ts index 286fb1ab8..29d2753a5 100644 --- a/packages/web/src/features/repos/actions.ts +++ b/packages/web/src/features/repos/actions.ts @@ -8,6 +8,13 @@ import { withMinimumOrgRole } from '@/middleware/withMinimumOrgRole'; import { OrgRole } from '@sourcebot/db'; import { JOB_PRIORITIES, REPO_INDEX_QUEUE } from '@sourcebot/shared'; +const MAX_CONCURRENT_REPO_RETRIES = 10; + +export type ScheduledRepoIndexJob = { + repoId: number; + jobId: string; +}; + export const indexRepo = async (repoId: number) => sew(() => withAuth(({ org, prisma, role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { @@ -27,7 +34,7 @@ export const indexRepo = async (repoId: number) => sew(() => const jobId = await getBullMQClient().enqueue( REPO_INDEX_QUEUE, - { repoId: repo.id, type: 'INDEX' }, + { repoId: repo.id }, { priority: JOB_PRIORITIES.INTERACTIVE }, ); @@ -38,3 +45,60 @@ export const indexRepo = async (repoId: number) => sew(() => }) ) ); + +export const retryReposWithSyncIssues = async () => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + try { + const client = getBullMQClient(); + const failedJobIds = await client.getFailedJobIds( + REPO_INDEX_QUEUE, + ); + const repos = await prisma.repo.findMany({ + where: { + orgId: org.id, + latestIndexingJobId: { in: failedJobIds }, + }, + orderBy: { id: 'asc' }, + select: { id: true }, + }); + + const jobs: ScheduledRepoIndexJob[] = []; + let failedCount = 0; + + for ( + let offset = 0; + offset < repos.length; + offset += MAX_CONCURRENT_REPO_RETRIES + ) { + const batch = repos.slice( + offset, + offset + MAX_CONCURRENT_REPO_RETRIES, + ); + const results = await Promise.allSettled( + batch.map(async ({ id: repoId }) => ({ + repoId, + jobId: await client.enqueue( + REPO_INDEX_QUEUE, + { repoId }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ), + })), + ); + + for (const result of results) { + if (result.status === 'fulfilled') { + jobs.push(result.value); + } else { + failedCount += 1; + } + } + } + + return { jobs, failedCount }; + } catch { + return unexpectedError('Failed to retry repository syncs'); + } + }) + ) +); diff --git a/packages/web/src/features/repos/repositorySyncCounts.server.ts b/packages/web/src/features/repos/repositorySyncCounts.server.ts new file mode 100644 index 000000000..772ef1c1d --- /dev/null +++ b/packages/web/src/features/repos/repositorySyncCounts.server.ts @@ -0,0 +1,57 @@ +import "server-only"; + +import { getBullMQClient } from "@/lib/bullmqClient"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { OrgRole } from "@sourcebot/db"; +import { REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { cache } from "react"; + +export interface RepositorySyncCounts { + firstTimeSyncingCount: number; + failedCount: number; + warningCount: number; +} + +export const getRepositorySyncCounts = cache(async () => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const failedJobIds = await getBullMQClient().getFailedJobIds( + REPO_INDEX_QUEUE, + ); + + const [ + firstTimeSyncingCount, + failedCount, + warningCount, + ] = await Promise.all([ + prisma.repo.count({ + where: { + orgId: org.id, + indexedAt: null, + firstIndexingJobFinishedAt: null, + }, + }), + prisma.repo.count({ + where: { + orgId: org.id, + latestIndexingJobId: { in: failedJobIds }, + indexedAt: null, + }, + }), + prisma.repo.count({ + where: { + orgId: org.id, + latestIndexingJobId: { in: failedJobIds }, + indexedAt: { not: null }, + }, + }), + ]); + + return { + firstTimeSyncingCount, + failedCount, + warningCount, + }; + }) + )); diff --git a/packages/web/src/types.ts b/packages/web/src/types.ts deleted file mode 100644 index 73ff02391..000000000 --- a/packages/web/src/types.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { z } from "zod"; - -export const demoSearchScopeSchema = z.object({ - id: z.number(), - displayName: z.string(), - value: z.string(), - type: z.enum(["repo", "reposet"]), - codeHostType: z.string().optional(), -}) - -export const demoSearchExampleSchema = z.object({ - title: z.string(), - description: z.string(), - url: z.string(), - searchScopes: z.array(z.number()) -}) - -export const demoExamplesSchema = z.object({ - searchScopes: demoSearchScopeSchema.array(), - searchExamples: demoSearchExampleSchema.array(), -}) - -export type DemoExamples = z.infer; -export type DemoSearchScope = z.infer; -export type DemoSearchExample = z.infer; \ No newline at end of file