From d2745e04dba674892297398271b8990c11ce2d41 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 21:20:26 -0700 Subject: [PATCH 01/34] wip on reposv2 table --- packages/shared/src/bullmqClient.test.ts | 83 ++ packages/shared/src/bullmqClient.ts | 27 + .../web/src/app/(app)/chats/chatsPage.tsx | 2 +- .../components/lightweightCodeHighlighter.tsx | 8 +- .../reposv2/components/repoActionsMenu.tsx | 112 +++ .../reposv2/components/reposTable.test.tsx | 434 +++++++++ .../(app)/reposv2/components/reposTable.tsx | 900 ++++++++++++++++++ .../reposv2/components/syncIssuePopover.tsx | 184 ++++ packages/web/src/app/(app)/reposv2/page.tsx | 130 +++ packages/web/src/app/(app)/reposv2/types.ts | 12 + .../api/(server)/repo-index-status/route.ts | 65 ++ 11 files changed, 1954 insertions(+), 3 deletions(-) create mode 100644 packages/web/src/app/(app)/reposv2/components/repoActionsMenu.tsx create mode 100644 packages/web/src/app/(app)/reposv2/components/reposTable.test.tsx create mode 100644 packages/web/src/app/(app)/reposv2/components/reposTable.tsx create mode 100644 packages/web/src/app/(app)/reposv2/components/syncIssuePopover.tsx create mode 100644 packages/web/src/app/(app)/reposv2/page.tsx create mode 100644 packages/web/src/app/(app)/reposv2/types.ts create mode 100644 packages/web/src/app/api/(server)/repo-index-status/route.ts diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index 8cf08309f..3f49990b6 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -3,6 +3,8 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ add: vi.fn(async () => ({ id: "job-1" })), + getJob: vi.fn(), + listJobs: vi.fn(), upsertJobScheduler: vi.fn(async () => ({ id: "scheduled-job" })), getJobScheduler: vi.fn(), getJobSchedulers: vi.fn(async () => [ @@ -15,6 +17,8 @@ const mocks = vi.hoisted(() => ({ vi.mock("bullmq", () => ({ Queue: class { add = mocks.add; + getJob = mocks.getJob; + getJobs = mocks.listJobs; upsertJobScheduler = mocks.upsertJobScheduler; getJobScheduler = mocks.getJobScheduler; getJobSchedulers = mocks.getJobSchedulers; @@ -35,9 +39,88 @@ describe("BullMQClient", () => { vi.restoreAllMocks(); vi.clearAllMocks(); vi.spyOn(Date, "now").mockReturnValue(1_000_000); + mocks.getJob.mockResolvedValue(undefined); + mocks.listJobs.mockResolvedValue([]); mocks.getJobScheduler.mockResolvedValue(undefined); }); + test("gets jobs by id and preserves missing jobs", async () => { + mocks.getJob.mockImplementation(async (jobId: string) => { + if (jobId === "job-1") { + return { + id: jobId, + data: { connectionId: 1 }, + failedReason: "", + getState: vi.fn(async () => "active"), + }; + } + if (jobId === "job-2") { + return { + id: jobId, + data: { connectionId: 2 }, + failedReason: "", + getState: vi.fn(async () => "completed"), + }; + } + return undefined; + }); + const client = new BullMQClient({} as Redis); + + await expect( + client.getJobs(CONNECTION_QUEUE, ["job-1", "missing", "job-2"]), + ).resolves.toEqual(new Map([ + ["job-1", { + id: "job-1", + data: { connectionId: 1 }, + status: "IN_PROGRESS", + errorMessage: null, + }], + ["missing", null], + ["job-2", { + id: "job-2", + data: { connectionId: 2 }, + status: "COMPLETED", + errorMessage: null, + }], + ])); + }); + + test("deduplicates job ids when getting jobs", async () => { + mocks.getJob.mockResolvedValue({ + id: "job-1", + data: { connectionId: 1 }, + failedReason: "", + getState: vi.fn(async () => "waiting"), + }); + const client = new BullMQClient({} as Redis); + + const jobs = await client.getJobs(CONNECTION_QUEUE, [ + "job-1", + "job-1", + ]); + + expect(jobs).toHaveLength(1); + expect(mocks.getJob).toHaveBeenCalledTimes(1); + }); + + test("lists failed job ids", async () => { + mocks.listJobs.mockResolvedValue([ + { id: "failed-1" }, + { id: "failed-2" }, + ]); + const client = new BullMQClient({} as Redis); + + await expect( + client.getFailedJobIds(CONNECTION_QUEUE), + ).resolves.toEqual(["failed-1", "failed-2"]); + expect(mocks.listJobs).toHaveBeenCalledWith( + ["failed"], + 0, + -1, + true, + ); + }); + test("includes workload data in scheduled jobs", async () => { const client = new BullMQClient({} as Redis); const data = { connectionId: 42 }; diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index b92f006bb..527e3c426 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -93,6 +93,33 @@ export class BullMQClient { }; } + async getJobs( + spec: QueueSpec, + jobIds: readonly string[], + ): Promise | null>> { + const uniqueJobIds = [...new Set(jobIds)]; + const jobs = await Promise.all( + uniqueJobIds.map((jobId) => this.getJob(spec, jobId)), + ); + + return new Map( + uniqueJobIds.map((jobId, index) => [jobId, jobs[index] ?? null]), + ); + } + + async getFailedJobIds( + spec: QueueSpec, + ): Promise { + const jobs = await this.getQueue(spec).getJobs( + ["failed"], + 0, + -1, + true, + ); + + return jobs.flatMap((job) => job.id ? [job.id] : []); + } + async getJobLogs( spec: QueueSpec, jobId: string, diff --git a/packages/web/src/app/(app)/chats/chatsPage.tsx b/packages/web/src/app/(app)/chats/chatsPage.tsx index 421cbce19..d522cb1b5 100644 --- a/packages/web/src/app/(app)/chats/chatsPage.tsx +++ b/packages/web/src/app/(app)/chats/chatsPage.tsx @@ -322,7 +322,7 @@ export function ChatsPage() { return (
-

Chats

+

Chats

diff --git a/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx b/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx index a88a3911f..cbbe416d6 100644 --- a/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx +++ b/packages/web/src/app/(app)/components/lightweightCodeHighlighter.tsx @@ -15,6 +15,7 @@ interface LightweightCodeHighlighter { /* 1-based line number offset */ lineNumbersOffset?: number; renderWhitespace?: boolean; + wrapLines?: boolean; isCopyButtonVisible?: boolean; } @@ -37,6 +38,7 @@ export const LightweightCodeHighlighter = memo((prop lineNumbers = false, lineNumbersOffset = 1, renderWhitespace = false, + wrapLines = true, isCopyButtonVisible = false, } = props; @@ -135,8 +137,10 @@ export const LightweightCodeHighlighter = memo((prop style={{ fontFamily: tailwind.theme.fontFamily.editor, fontSize: tailwind.theme.fontSize.editor, - whiteSpace: renderWhitespace ? 'pre-wrap' : 'none', - wordBreak: 'break-all', + whiteSpace: wrapLines + ? (renderWhitespace ? 'pre-wrap' : 'normal') + : 'pre', + wordBreak: wrapLines ? 'break-all' : 'normal', }} > {(highlightedLines ?? unhighlightedLines).map((line, index) => ( diff --git a/packages/web/src/app/(app)/reposv2/components/repoActionsMenu.tsx b/packages/web/src/app/(app)/reposv2/components/repoActionsMenu.tsx new file mode 100644 index 000000000..1cfdf2d74 --- /dev/null +++ b/packages/web/src/app/(app)/reposv2/components/repoActionsMenu.tsx @@ -0,0 +1,112 @@ +"use client"; + +import { useToast } from "@/components/hooks/use-toast"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { indexRepo } from "@/features/repos/actions"; +import { getCodeHostInfoForRepo, isServiceError } from "@/lib/utils"; +import { EllipsisVertical, ExternalLink, Loader2, RefreshCw } from "lucide-react"; +import { useState } from "react"; +import type { Repo } from "./reposTable"; + +type RepoActionsMenuProps = { + repo: Repo; + canSync: boolean; + isSyncing: boolean; + onSyncScheduled: (repoId: number, jobId: string) => void; +}; + +export const RepoActionsMenu = ({ + repo, + canSync, + isSyncing, + onSyncScheduled, +}: RepoActionsMenuProps) => { + const [isScheduling, setIsScheduling] = useState(false); + const { toast } = useToast(); + const displayName = repo.displayName ?? repo.name; + const codeHostInfo = getCodeHostInfoForRepo({ + codeHostType: repo.codeHostType, + name: repo.name, + displayName, + externalWebUrl: repo.webUrl ?? undefined, + }); + + const syncRepo = async () => { + setIsScheduling(true); + + try { + const response = await indexRepo(repo.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to sync repository", + description: response.message, + }); + return; + } + + onSyncScheduled(repo.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${displayName} was queued for indexing.`, + }); + } catch { + toast({ + variant: "destructive", + title: "Failed to sync repository", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsScheduling(false); + } + }; + + return ( + + + + + + {canSync && ( + void syncRepo()} + > + {isScheduling ? ( + + ) : ( + + )} + Sync + + )} + {codeHostInfo.externalWebUrl && ( + + + + Open in {codeHostInfo.codeHostName} + + + )} + + + ); +}; diff --git a/packages/web/src/app/(app)/reposv2/components/reposTable.test.tsx b/packages/web/src/app/(app)/reposv2/components/reposTable.test.tsx new file mode 100644 index 000000000..e2c61d00f --- /dev/null +++ b/packages/web/src/app/(app)/reposv2/components/reposTable.test.tsx @@ -0,0 +1,434 @@ +import { TooltipProvider } from "@/components/ui/tooltip"; +import type { CodeHostType } from "@sourcebot/db"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import type { RepoIndexingStatusesResponse } from "../types"; +import type { Repo } from "./reposTable"; + +const navigation = vi.hoisted(() => ({ + push: vi.fn(), + replace: vi.fn(), + refresh: vi.fn(), + searchParams: "", +})); +const reposActions = vi.hoisted(() => ({ + indexRepo: vi.fn(), +})); + +vi.mock("@/features/repos/actions", () => ({ + indexRepo: reposActions.indexRepo, +})); + +vi.mock("next/navigation", () => ({ + usePathname: () => "/reposv2", + useRouter: () => navigation, + useSearchParams: () => new URLSearchParams(navigation.searchParams), +})); + +const { ReposTable } = await import("./reposTable"); + +const repos: Repo[] = [ + { + id: 1, + name: "github.com/acme/first", + displayName: "acme/first", + indexedAt: null, + indexedCommitHash: null, + latestJob: { + id: "job-1", + data: { repoId: 1, type: "INDEX" }, + status: "IN_PROGRESS", + errorMessage: null, + }, + imageUrl: null, + webUrl: "https://github.com/acme/first", + codeHostType: "github" as CodeHostType, + }, + { + id: 2, + name: "github.com/acme/second", + displayName: "acme/second", + indexedAt: new Date("2026-08-16T12:00:00.000Z"), + indexedCommitHash: "2222222222222222222222222222222222222222", + latestJob: null, + imageUrl: null, + webUrl: "https://github.com/acme/second", + codeHostType: "github" as CodeHostType, + }, +]; + +const renderTable = (data: Repo[] = repos, canRetry = true) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + + return render( + + + + + , + ); +}; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); + vi.unstubAllGlobals(); + navigation.searchParams = ""; +}); + +describe("ReposTable", () => { + test("reflects the status filter from the URL", () => { + navigation.searchParams = "status=failed"; + + renderTable([repos[1]]); + + expect( + screen + .getByRole("combobox", { + name: "Filter repositories by status", + }) + .textContent, + ).toContain("Failed"); + }); + + test("centers the empty state across the table and hides pagination", () => { + navigation.searchParams = "status=warning"; + + renderTable([]); + + const emptyState = screen.getByText("No repositories with warnings."); + expect(emptyState.closest("td")?.getAttribute("colspan")).toBe("4"); + expect(screen.queryByText("Page 1 of 1")).toBeNull(); + expect(screen.queryByRole("button", { name: "Previous" })).toBeNull(); + expect(screen.queryByRole("button", { name: "Next" })).toBeNull(); + }); + + test("does not reset pagination when the search value is unchanged", () => { + navigation.searchParams = "page=2"; + + renderTable([repos[1]]); + + expect(navigation.replace).not.toHaveBeenCalled(); + }); + + test("focuses repository search when slash is pressed", () => { + renderTable([repos[1]]); + + const searchInput = screen.getByPlaceholderText("Search repositories..."); + fireEvent.keyDown(document, { key: "/" }); + + expect(document.activeElement).toBe(searchInput); + }); + + test("shows a loading indicator while search is being debounced", () => { + renderTable([repos[1]]); + + fireEvent.change( + screen.getByPlaceholderText("Search repositories..."), + { target: { value: "sourcebot" } }, + ); + + expect(screen.getByLabelText("Searching repositories")).toBeTruthy(); + }); + + test("does not show completed for a repository already indexed on page load", () => { + renderTable([{ + ...repos[1], + latestJob: { + id: "completed-job", + data: { repoId: 2, type: "INDEX" }, + status: "COMPLETED", + errorMessage: null, + }, + }]); + + expect(screen.queryByText("Completed")).toBeNull(); + }); + + test("links only synced repository names to their root browse page", () => { + renderTable(); + + expect( + screen.queryByRole("link", { name: "acme/first" }), + ).toBeNull(); + expect( + screen + .getByRole("link", { name: "acme/second" }) + .getAttribute("href"), + ).toBe("/browse/github.com/acme/second/-/tree"); + }); + + test("shows repository actions to everyone and sync only to owners", () => { + const { unmount } = renderTable([repos[1]], false); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect(screen.queryByRole("menuitem", { name: "Sync" })).toBeNull(); + expect( + screen + .getByRole("menuitem", { name: "Open in GitHub" }) + .getAttribute("href"), + ).toBe("https://github.com/acme/second"); + + unmount(); + renderTable([repos[1]]); + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect(screen.getByRole("menuitem", { name: "Sync" })).toBeTruthy(); + }); + + test("shows client-only syncing after an owner explicitly schedules a sync", async () => { + reposActions.indexRepo.mockResolvedValue({ jobId: "interactive-job" }); + vi.stubGlobal("fetch", vi.fn(() => new Promise(() => {}))); + renderTable([repos[1]]); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => { + expect(reposActions.indexRepo).toHaveBeenCalledWith(2); + expect(screen.getByText("Syncing")).toBeTruthy(); + expect(fetch).toHaveBeenCalledOnce(); + }); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + expect( + screen + .getByRole("menuitem", { name: "Sync" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("preserves a completed sync timestamp when another sync starts", async () => { + const firstRepo: Repo = { + ...repos[0], + indexedAt: new Date(Date.now() - 2 * 24 * 60 * 60 * 1000), + indexedCommitHash: "1111111111111111111111111111111111111111", + latestJob: null, + }; + reposActions.indexRepo + .mockResolvedValueOnce({ jobId: "first-interactive-job" }) + .mockResolvedValueOnce({ jobId: "second-interactive-job" }); + const firstCompletedResponse: RepoIndexingStatusesResponse = { + repositories: [{ + repoId: 1, + indexedAt: new Date().toISOString(), + indexedCommitHash: "3333333333333333333333333333333333333333", + latestJob: { + id: "first-interactive-job", + data: { repoId: 1, type: "INDEX" }, + status: "COMPLETED", + errorMessage: null, + }, + }], + }; + vi.stubGlobal( + "fetch", + vi.fn() + .mockResolvedValueOnce(Response.json(firstCompletedResponse)) + .mockImplementation(() => new Promise(() => {})), + ); + renderTable([firstRepo, repos[1]]); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/first", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => { + const firstRow = screen + .getByRole("link", { name: "acme/first" }) + .closest("tr"); + expect(firstRow && within(firstRow).getByText("just now")).toBeTruthy(); + }); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for acme/second", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(2)); + const firstRow = screen + .getByRole("link", { name: "acme/first" }) + .closest("tr"); + expect(firstRow && within(firstRow).getByText("just now")).toBeTruthy(); + }); + + test("shows the impact and worker error for a warning", () => { + renderTable([{ + ...repos[1], + latestJob: { + id: "warning-job", + data: { repoId: 2, type: "INDEX" }, + status: "FAILED", + errorMessage: "The remote repository could not be reached", + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View warning details for acme/second", + })); + + expect(screen.getByText("Latest sync failed")).toBeTruthy(); + expect(screen.getByText(/results may be stale/)).toBeTruthy(); + expect( + screen.getByText("The remote repository could not be reached"), + ).toBeTruthy(); + expect(screen.getByText("warning-job")).toBeTruthy(); + }); + + test("shows the impact and worker error for a failed repository", () => { + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + + expect(screen.getByText("Repository sync failed")).toBeTruthy(); + expect(screen.getByText(/not available in search/)).toBeTruthy(); + expect( + screen.getByText("Authentication failed while cloning"), + ).toBeTruthy(); + expect(screen.getByText("job-1")).toBeTruthy(); + }); + + test("schedules a retry and transitions an unindexed repository to syncing", async () => { + reposActions.indexRepo.mockResolvedValue({ jobId: "retry-job" }); + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ + repositories: [{ + repoId: 1, + indexedAt: null, + indexedCommitHash: null, + latestJob: { + id: "job-1", + data: { repoId: 1, type: "INDEX" }, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + }, + }], + } satisfies RepoIndexingStatusesResponse))); + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + fireEvent.click(screen.getByRole("button", { name: "Retry sync" })); + + await waitFor(() => { + expect(reposActions.indexRepo).toHaveBeenCalledWith(1); + expect(screen.getByText("Syncing")).toBeTruthy(); + expect(fetch).toHaveBeenCalledOnce(); + }); + expect(screen.queryByText("Repository sync failed")).toBeNull(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("updates a completed repository in place without refreshing or reordering", async () => { + let resolveRequest: ((response: Response) => void) | undefined; + vi.stubGlobal("fetch", vi.fn(() => new Promise((resolve) => { + resolveRequest = resolve; + }))); + + renderTable(); + + expect(screen.getByText("Syncing")).toBeTruthy(); + const initialRows = within(screen.getByRole("table")).getAllByRole("row"); + expect(within(initialRows[1]).getByText("acme/first")).toBeTruthy(); + expect(within(initialRows[2]).getByText("acme/second")).toBeTruthy(); + + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + const response: RepoIndexingStatusesResponse = { + repositories: [{ + repoId: 1, + indexedAt: "2026-08-17T12:00:00.000Z", + indexedCommitHash: "1111111111111111111111111111111111111111", + latestJob: { + id: "job-1", + data: { repoId: 1, type: "INDEX" }, + status: "COMPLETED", + errorMessage: null, + }, + }], + }; + await act(async () => { + resolveRequest?.(Response.json(response)); + }); + + await waitFor(() => expect(screen.getByText("Completed")).toBeTruthy()); + expect(screen.queryByText("Syncing")).toBeNull(); + const updatedRows = within(screen.getByRole("table")).getAllByRole("row"); + expect(within(updatedRows[1]).getByText("acme/first")).toBeTruthy(); + expect( + within(updatedRows[1]) + .getByRole("link", { name: "acme/first" }) + .getAttribute("href"), + ).toBe("/browse/github.com/acme/first/-/tree"); + expect( + within(updatedRows[1]) + .getByRole("link", { name: "1111111" }) + .getAttribute("href"), + ).toBe( + "/browse/github.com/acme/first/-/commit/1111111111111111111111111111111111111111", + ); + expect(within(updatedRows[1]).queryByText("-")).toBeNull(); + expect(within(updatedRows[2]).getByText("acme/second")).toBeTruthy(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("polls a syncing repository whose latest job is missing", async () => { + const response: RepoIndexingStatusesResponse = { + repositories: [{ + repoId: 1, + indexedAt: null, + indexedCommitHash: null, + latestJob: { + id: "job-1", + data: { repoId: 1, type: "INDEX" }, + status: "FAILED", + errorMessage: "Indexing failed", + }, + }], + }; + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json(response))); + + renderTable([{ ...repos[0], latestJob: null }]); + + expect(screen.getByText("Syncing")).toBeTruthy(); + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + await waitFor(() => expect(screen.getByText("Failed")).toBeTruthy()); + expect(screen.queryByText("Syncing")).toBeNull(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/web/src/app/(app)/reposv2/components/reposTable.tsx b/packages/web/src/app/(app)/reposv2/components/reposTable.tsx new file mode 100644 index 000000000..0182f859b --- /dev/null +++ b/packages/web/src/app/(app)/reposv2/components/reposTable.tsx @@ -0,0 +1,900 @@ +"use client"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, +} from "@/components/ui/input-group"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn, getCodeHostIcon, getRepoImageSrc } from "@/lib/utils"; +import type { CodeHostType } from "@sourcebot/db"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { useDebounce } from "@uidotdev/usehooks"; +import { + type ColumnDef, + flexRender, + getCoreRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, Check, Loader2, Search } from "lucide-react"; +import Image from "next/image"; +import Link from "next/link"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import { DisplayDate } from "../../components/DisplayDate"; +import { getBrowsePath } from "../../browse/hooks/utils"; +import type { RepoIndexingStatusesResponse } from "../types"; +import { RepoActionsMenu } from "./repoActionsMenu"; +import { SyncIssuePopover } from "./syncIssuePopover"; + +const POLL_INTERVAL_MS = 5_000; +const COMPLETED_BADGE_VISIBLE_MS = 5_000; + +export type Repo = { + id: number; + name: string; + displayName: string | null; + indexedAt: Date | null; + indexedCommitHash: string | null; + latestJob: WorkloadJob<"repo-index"> | null; + imageUrl: string | null; + webUrl: string | null; + codeHostType: CodeHostType; +}; + +type DisplayedRepo = Repo & { + showCompleted: boolean; + showExplicitSyncing: boolean; +}; + +type SortOrder = "asc" | "desc"; +type SortBy = "name" | "indexedAt"; +type StatusFilter = "all" | "failed" | "warning"; +type SyncAnnotation = "SYNCING" | "WARNING" | "FAILED" | null; + +const getStatusFilter = (value: string | null): StatusFilter => { + if (value === "failed" || value === "warning") { + return value; + } + + return "all"; +}; + +const getRepoName = (repo: Repo) => repo.displayName ?? repo.name; + +const getRepoIndexingStatuses = async ( + repoIds: number[], + signal: AbortSignal, +): Promise => { + const response = await fetch("/api/repo-index-status", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ repoIds }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load repository indexing statuses"); + } + + return response.json() as Promise; +}; + +const getSyncAnnotation = (repo: Repo): SyncAnnotation => { + const latestJob = repo.latestJob; + const isLatestIndexJob = latestJob?.data.repoId === repo.id + && latestJob.data.type === "INDEX"; + + if (isLatestIndexJob && latestJob.status === "FAILED") { + return repo.indexedAt ? "WARNING" : "FAILED"; + } + + if ( + !repo.indexedAt + && ( + !isLatestIndexJob + || latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ) + ) { + return "SYNCING"; + } + + return null; +}; + +const SyncAnnotationBadge = ({ + repo, + canRetry, + onRetryScheduled, + showCompleted, + showExplicitSyncing, +}: { + repo: Repo; + canRetry: boolean; + onRetryScheduled: (repoId: number, jobId: string) => void; + showCompleted: boolean; + showExplicitSyncing: boolean; +}) => { + const prefersReducedMotion = useReducedMotion(); + const completionKey = showCompleted + ? repo.latestJob?.id ?? repo.indexedAt?.toISOString() ?? `repo:${repo.id}` + : null; + const [expiredCompletionKey, setExpiredCompletionKey] = useState< + string | null + >(null); + useEffect(() => { + if (!completionKey || expiredCompletionKey === completionKey) { + return; + } + + const timeout = window.setTimeout(() => { + setExpiredCompletionKey(completionKey); + }, COMPLETED_BADGE_VISIBLE_MS); + return () => window.clearTimeout(timeout); + }, [completionKey, expiredCompletionKey]); + const annotation = completionKey !== null + && expiredCompletionKey !== completionKey + ? "COMPLETED" + : showExplicitSyncing + ? "SYNCING" + : getSyncAnnotation(repo); + const badge = (() => { + switch (annotation) { + case "COMPLETED": + return ( + + + Completed + + ); + case "SYNCING": + return ( + + + Syncing + + ); + case "WARNING": + return ( + + ); + case "FAILED": + return ( + + ); + default: + return null; + } + })(); + + const isCompleted = annotation === "COMPLETED"; + + return ( + + {annotation && badge && ( + + {badge} + + )} + + ); +}; + +const SortableHeader = ({ + label, + column, + sortBy, + sortOrder, + onSortChange, + tooltip, +}: { + label: string; + column: SortBy; + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + tooltip?: string; +}) => { + const isActive = sortBy === column; + const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; + + const button = ( + + ); + + if (!tooltip) { + return button; + } + + return ( + + {button} + +

{tooltip}

+ + + ); +}; + +const getColumns = ({ + sortBy, + sortOrder, + onSortChange, + canRetry, + onRetryScheduled, +}: { + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + canRetry: boolean; + onRetryScheduled: (repoId: number, jobId: string) => void; +}): ColumnDef[] => [ + { + id: "name", + accessorFn: getRepoName, + header: () => ( + + ), + cell: ({ row }) => { + const repo = row.original; + const displayName = getRepoName(repo); + const codeHostIcon = getCodeHostIcon(repo.codeHostType); + const repoImageSrc = repo.imageUrl + ? getRepoImageSrc(repo.imageUrl, repo.id) + : undefined; + const isInternalApiImage = repoImageSrc?.startsWith("/api/"); + const repoBrowseUrl = repo.indexedAt + ? getBrowsePath({ + repoName: repo.name, + path: "", + pathType: "tree", + }) + : null; + + return ( +
+ {repoImageSrc ? ( + {`${displayName} + ) : ( + {`${displayName} + )} + {repoBrowseUrl ? ( + + {displayName} + + ) : ( + + {displayName} + + )} + +
+ ); + }, + }, + { + accessorKey: "indexedAt", + header: () => ( + + ), + cell: ({ row }) => { + const indexedAt = row.original.indexedAt; + return indexedAt ? : "-"; + }, + }, + { + accessorKey: "indexedCommitHash", + header: "Synced commit", + cell: ({ row }) => { + const hash = row.original.indexedCommitHash; + if (!hash) { + return "-"; + } + + const repo = row.original; + const shortHash = hash.slice(0, 7); + const commitUrl = getBrowsePath({ + repoName: repo.name, + path: "", + pathType: "commit", + commitSha: hash, + }); + const hashElement = ( + + {shortHash} + + ); + + return ( + + {hashElement} + + {hash} + + + ); + }, + }, + { + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ +
+ ), + }, + ]; + +type ReposTableProps = { + data: Repo[]; + currentPage: number; + pageSize: number; + totalCount: number; + canRetry: boolean; + sortBy: SortBy; + sortOrder: SortOrder; +}; + +export const ReposTable = ({ + data, + currentPage, + pageSize, + totalCount, + canRetry, + sortBy, + sortOrder, +}: ReposTableProps) => { + const pathname = usePathname(); + const router = useRouter(); + const searchParams = useSearchParams(); + const searchParamsString = searchParams.toString(); + const urlSearchValue = searchParams.get("search") ?? ""; + const statusFilter = getStatusFilter(searchParams.get("status")); + const [searchValue, setSearchValue] = useState(urlSearchValue); + const [scheduledRetryJobs, setScheduledRetryJobs] = useState< + Map> + >(() => new Map()); + const debouncedSearchValue = useDebounce(searchValue, 300); + const [isSearchNavigationPending, startSearchTransition] = useTransition(); + const pendingSearchValuesRef = useRef>(new Set()); + const searchInputRef = useRef(null); + const isSearchPending = searchValue !== debouncedSearchValue + || isSearchNavigationPending; + + useHotkeys("/", (event) => { + event.preventDefault(); + searchInputRef.current?.focus(); + }); + + useEffect(() => { + if (pendingSearchValuesRef.current.delete(urlSearchValue)) { + return; + } + + setSearchValue(urlSearchValue); + }, [urlSearchValue]); + + useEffect(() => { + if (debouncedSearchValue !== searchValue) { + return; + } + + const nextSearchValue = debouncedSearchValue.trim(); + if (nextSearchValue === urlSearchValue) { + return; + } + + const params = new URLSearchParams(searchParamsString); + if (nextSearchValue) { + params.set("search", nextSearchValue); + } else { + params.delete("search"); + } + params.delete("page"); + + const nextSearchParamsString = params.toString(); + if (nextSearchParamsString === searchParamsString) { + return; + } + + pendingSearchValuesRef.current.add(nextSearchValue); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }, [ + debouncedSearchValue, + pathname, + router, + searchParamsString, + searchValue, + startSearchTransition, + urlSearchValue, + ]); + const onRetryScheduled = useCallback((repoId: number, jobId: string) => { + setScheduledRetryJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + nextJobs.set(repoId, { + id: jobId, + data: { repoId, type: "INDEX" }, + status: "PENDING", + errorMessage: null, + }); + return nextJobs; + }); + }, []); + const retryAwareData = useMemo( + () => data.map((repo) => { + const scheduledJob = scheduledRetryJobs.get(repo.id); + return scheduledJob + ? { ...repo, latestJob: scheduledJob } + : repo; + }), + [data, scheduledRetryJobs], + ); + const pollingTargets = useMemo( + () => retryAwareData.flatMap((repo) => { + const scheduledJob = scheduledRetryJobs.get(repo.id); + return scheduledJob || getSyncAnnotation(repo) === "SYNCING" + ? [{ repoId: repo.id, jobId: repo.latestJob?.id ?? null }] + : []; + }), + [retryAwareData, scheduledRetryJobs], + ); + const pollingRepoIds = useMemo( + () => pollingTargets.map(({ repoId }) => repoId), + [pollingTargets], + ); + const pollingKey = useMemo( + () => pollingTargets.map(({ repoId, jobId }) => `${repoId}:${jobId}`), + [pollingTargets], + ); + const { data: polledStatuses } = useQuery({ + queryKey: ["reposv2-indexing-status", pollingRepoIds, pollingKey], + queryFn: ({ signal }) => getRepoIndexingStatuses(pollingRepoIds, signal), + enabled: pollingRepoIds.length > 0, + placeholderData: (previousData) => previousData, + refetchInterval: (query) => { + const statuses = query.state.data?.repositories; + if (!statuses) { + return POLL_INTERVAL_MS; + } + + return pollingTargets.some((target) => { + const status = statuses.find( + ({ repoId }) => repoId === target.repoId, + ); + if (!status) { + return true; + } + + const latestJob = status.latestJob; + const isLatestIndexJob = latestJob?.data.repoId === status.repoId + && latestJob.data.type === "INDEX"; + const expectedJobId = target.jobId; + + if (expectedJobId && latestJob?.id !== expectedJobId) { + return true; + } + + if (expectedJobId) { + return latestJob?.status === "PENDING" + || latestJob?.status === "IN_PROGRESS"; + } + + return !status.indexedAt + && ( + !isLatestIndexJob + || latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ); + }) + ? POLL_INTERVAL_MS + : false; + }, + }); + const completedDuringPollingRepoIds = useMemo(() => { + const pollingTargetsByRepoId = new Map( + pollingTargets.map((target) => [target.repoId, target]), + ); + return new Set( + polledStatuses?.repositories.flatMap((status) => { + const target = pollingTargetsByRepoId.get(status.repoId); + const isExpectedJob = !target?.jobId + || status.latestJob?.id === target.jobId; + return target + && isExpectedJob + && status.latestJob?.status === "COMPLETED" + && status.indexedAt + ? [status.repoId] + : []; + }) ?? [], + ); + }, [polledStatuses, pollingTargets]); + const displayedData = useMemo(() => { + const statusesByRepoId = new Map( + polledStatuses?.repositories.map((status) => [ + status.repoId, + status, + ]) ?? [], + ); + return retryAwareData.map((repo): DisplayedRepo => { + const showCompleted = completedDuringPollingRepoIds.has(repo.id); + const status = statusesByRepoId.get(repo.id); + const scheduledJob = scheduledRetryJobs.get(repo.id); + const showExplicitSyncing = Boolean( + scheduledJob + && ( + !status + || status.latestJob?.id !== scheduledJob.id + || status.latestJob.status === "PENDING" + || status.latestJob.status === "IN_PROGRESS" + ), + ); + if (!status) { + return { ...repo, showCompleted, showExplicitSyncing }; + } + if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { + return { ...repo, showCompleted, showExplicitSyncing }; + } + + return { + ...repo, + indexedAt: status.indexedAt + ? new Date(status.indexedAt) + : null, + indexedCommitHash: status.indexedCommitHash, + latestJob: status.latestJob, + showCompleted, + showExplicitSyncing, + }; + }); + }, [ + completedDuringPollingRepoIds, + polledStatuses, + retryAwareData, + scheduledRetryJobs, + ]); + + const onSortChange = useCallback((column: SortBy) => { + const params = new URLSearchParams(searchParams.toString()); + const nextSortOrder = sortBy === column && sortOrder === "asc" + ? "desc" + : "asc"; + params.delete("page"); + if (column === "name") { + params.delete("sortBy"); + } else { + params.set("sortBy", column); + } + if (nextSortOrder === "asc") { + params.delete("sortOrder"); + } else { + params.set("sortOrder", nextSortOrder); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }, [pathname, router, searchParams, sortBy, sortOrder]); + const columns = useMemo( + () => getColumns({ + sortBy, + sortOrder, + onSortChange, + canRetry, + onRetryScheduled, + }), + [ + canRetry, + onRetryScheduled, + onSortChange, + sortBy, + sortOrder, + ], + ); + const table = useReactTable({ + data: displayedData, + columns, + getCoreRowModel: getCoreRowModel(), + manualPagination: true, + manualSorting: true, + rowCount: totalCount, + state: { + pagination: { + pageIndex: currentPage - 1, + pageSize, + }, + sorting: [{ id: sortBy, desc: sortOrder === "desc" }], + }, + }); + const totalPages = Math.max(1, table.getPageCount()); + const firstVisibleRepo = totalCount === 0 + ? 0 + : (currentPage - 1) * pageSize + 1; + const lastVisibleRepo = Math.min(currentPage * pageSize, totalCount); + + const goToPage = (page: number) => { + const params = new URLSearchParams(searchParams.toString()); + if (page === 1) { + params.delete("page"); + } else { + params.set("page", page.toString()); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }; + + const onStatusFilterChange = (status: StatusFilter) => { + const params = new URLSearchParams(searchParams.toString()); + if (status === "all") { + params.delete("status"); + } else { + params.set("status", status); + } + params.delete("page"); + + const query = params.toString(); + router.replace(query ? `${pathname}?${query}` : pathname, { + scroll: false, + }); + }; + + const emptyMessage = statusFilter === "failed" + ? "No failed repositories." + : statusFilter === "warning" + ? "No repositories with warnings." + : "No repositories found."; + + return ( +
+
+ + + + + setSearchValue(event.target.value)} + placeholder="Search repositories..." + /> + {isSearchPending && ( + + + + )} + + +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} + + ))} + + + {table.getRowModel().rows.length > 0 ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + ))} + + )) + ) : ( + + + {emptyMessage} + + + )} + +
+
+ {totalCount > 0 && ( +
+

+ Showing {firstVisibleRepo}-{lastVisibleRepo} of {totalCount} +

+
+

+ Page {currentPage} of {totalPages} +

+
+ + +
+
+
+ )} +
+ ); +}; diff --git a/packages/web/src/app/(app)/reposv2/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/reposv2/components/syncIssuePopover.tsx new file mode 100644 index 000000000..39e5ecec3 --- /dev/null +++ b/packages/web/src/app/(app)/reposv2/components/syncIssuePopover.tsx @@ -0,0 +1,184 @@ +"use client"; + +import { badgeVariants } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { useToast } from "@/components/hooks/use-toast"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { indexRepo } from "@/features/repos/actions"; +import { cn, isServiceError } from "@/lib/utils"; +import { ChevronDown, CircleX, Loader2, RotateCw, TriangleAlert } from "lucide-react"; +import { useState } from "react"; +import { DisplayDate } from "../../components/DisplayDate"; +import { LightweightCodeHighlighter } from "../../components/lightweightCodeHighlighter"; +import type { Repo } from "./reposTable"; + +type SyncIssuePopoverProps = { + repo: Repo; + annotation: "WARNING" | "FAILED"; + canRetry: boolean; + onRetryScheduled: (repoId: number, jobId: string) => void; +}; + +export const SyncIssuePopover = ({ + repo, + annotation, + canRetry, + onRetryScheduled, +}: SyncIssuePopoverProps) => { + const [isOpen, setIsOpen] = useState(false); + const [isRetrying, setIsRetrying] = useState(false); + const { toast } = useToast(); + const latestJob = repo.latestJob; + if (!latestJob || latestJob.status !== "FAILED") { + return null; + } + + const isWarning = annotation === "WARNING"; + const label = isWarning ? "Warning" : "Failed"; + const Icon = isWarning ? TriangleAlert : CircleX; + const repoDisplayName = repo.displayName ?? repo.name; + + const retrySync = async () => { + setIsRetrying(true); + + try { + const response = await indexRepo(repo.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: response.message, + }); + return; + } + + onRetryScheduled(repo.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${repoDisplayName} was queued for indexing.`, + }); + setIsOpen(false); + } catch { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsRetrying(false); + } + }; + + return ( + + + + + + + + +

View failure details

+ + + +
+ +
+
+

+ {isWarning + ? "Latest sync failed" + : "Repository sync failed"} +

+

+ {isWarning + ? "Search remains available using the last successful sync, but results may be stale." + : "This repository has not synced successfully, so its contents are not available in search."} +

+
+
+

+ Error +

+
+ + {latestJob.errorMessage + ?? "No error details were reported."} + +
+
+
+ {isWarning && repo.indexedAt && ( + <> +
+ Last successful sync +
+
+ +
+ + )} +
Job ID
+
+ {latestJob.id} +
+
+ {canRetry && ( +
+ +
+ )} +
+
+ + + ); +}; diff --git a/packages/web/src/app/(app)/reposv2/page.tsx b/packages/web/src/app/(app)/reposv2/page.tsx new file mode 100644 index 000000000..dd5488824 --- /dev/null +++ b/packages/web/src/app/(app)/reposv2/page.tsx @@ -0,0 +1,130 @@ +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + authenticatedPage, + type OptionalAuthOptions, +} from "@/middleware/authenticatedPage"; +import { + REPO_INDEX_QUEUE, + type WorkloadJob, +} from "@sourcebot/shared"; +import { OrgRole, type Prisma } from "@sourcebot/db"; +import { z } from "zod"; +import { ReposTable } from "./components/reposTable"; + +const DEFAULT_PAGE_SIZE = 20; +const pageSchema = z.coerce.number().int().positive(); +const sortBySchema = z.enum(["name", "indexedAt"]); +const sortOrderSchema = z.enum(["asc", "desc"]); +const statusSchema = z.enum(["failed", "warning"]); + +type ReposV2PageProps = { + searchParams: Promise<{ + page?: string; + search?: string; + status?: string; + sortBy?: string; + sortOrder?: string; + }>; +}; + +export default authenticatedPage< + ReposV2PageProps, + OptionalAuthOptions +>(async ({ org, prisma, role }, { searchParams }) => { + const params = await searchParams; + const page = pageSchema.safeParse(params.page).data ?? 1; + const search = z.string().optional().safeParse(params.search).data?.trim() ?? ""; + const status = statusSchema.safeParse(params.status).data ?? "all"; + const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; + const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const skip = (page - 1) * DEFAULT_PAGE_SIZE; + const orderBy = sortBy === "indexedAt" + ? [{ indexedAt: sortOrder }, { id: "asc" as const }] + : [{ displayName: sortOrder }, { id: "asc" as const }]; + const failedJobIds = status === "all" + ? [] + : await getBullMQClient().getFailedJobIds(REPO_INDEX_QUEUE); + const where: Prisma.RepoWhereInput = { + orgId: org.id, + ...(search + ? { + displayName: { + contains: search, + mode: "insensitive" as const, + }, + } + : {}), + ...(status === "all" + ? {} + : { + latestIndexingJobId: { in: failedJobIds }, + indexedAt: status === "failed" ? null : { not: null }, + }), + }; + + const [repos, totalCount] = await Promise.all([ + prisma.repo.findMany({ + where, + orderBy, + skip, + take: DEFAULT_PAGE_SIZE, + select: { + id: true, + name: true, + displayName: true, + indexedAt: true, + indexedCommitHash: true, + latestIndexingJobId: true, + imageUrl: true, + webUrl: true, + external_codeHostType: true, + }, + }), + prisma.repo.count({ + where, + }), + ]); + const latestJobIds = repos.flatMap((repo) => + repo.latestIndexingJobId ? [repo.latestIndexingJobId] : [], + ); + let latestJobs = new Map | null>(); + try { + latestJobs = await getBullMQClient().getJobs( + REPO_INDEX_QUEUE, + latestJobIds, + ); + } catch (error) { + console.error("Failed to load latest repository indexing jobs", error); + } + + return ( +
+
+

Repositories

+
+
+ ({ + id: repo.id, + name: repo.name, + displayName: repo.displayName, + indexedAt: repo.indexedAt, + indexedCommitHash: repo.indexedCommitHash, + latestJob: repo.latestIndexingJobId + ? latestJobs.get(repo.latestIndexingJobId) ?? null + : null, + imageUrl: repo.imageUrl, + webUrl: repo.webUrl, + codeHostType: repo.external_codeHostType, + }))} + currentPage={page} + pageSize={DEFAULT_PAGE_SIZE} + totalCount={totalCount} + canRetry={role === OrgRole.OWNER} + sortBy={sortBy} + sortOrder={sortOrder} + /> +
+
+ ); +}, { allowAnonymous: true }); diff --git a/packages/web/src/app/(app)/reposv2/types.ts b/packages/web/src/app/(app)/reposv2/types.ts new file mode 100644 index 000000000..499781029 --- /dev/null +++ b/packages/web/src/app/(app)/reposv2/types.ts @@ -0,0 +1,12 @@ +import type { WorkloadJob } from "@sourcebot/shared"; + +export type RepoIndexingStatus = { + repoId: number; + indexedAt: string | null; + indexedCommitHash: string | null; + latestJob: WorkloadJob<"repo-index"> | null; +}; + +export type RepoIndexingStatusesResponse = { + repositories: RepoIndexingStatus[]; +}; diff --git a/packages/web/src/app/api/(server)/repo-index-status/route.ts b/packages/web/src/app/api/(server)/repo-index-status/route.ts new file mode 100644 index 000000000..a68a95cbc --- /dev/null +++ b/packages/web/src/app/api/(server)/repo-index-status/route.ts @@ -0,0 +1,65 @@ +import type { RepoIndexingStatusesResponse } from "@/app/(app)/reposv2/types"; +import { apiHandler } from "@/lib/apiHandler"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + requestBodySchemaValidationError, + serviceErrorResponse, +} from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withOptionalAuth } from "@/middleware/withAuth"; +import { REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { z } from "zod"; + +const requestSchema = z.object({ + repoIds: z.array(z.number().int().positive()).min(1).max(100), +}); + +export const POST = apiHandler(async (request) => { + const parsed = requestSchema.safeParse( + await request.json().catch(() => null), + ); + if (!parsed.success) { + return serviceErrorResponse( + requestBodySchemaValidationError(parsed.error), + ); + } + + const result = await withOptionalAuth(async ({ org, prisma }) => { + const repositories = await prisma.repo.findMany({ + where: { + orgId: org.id, + id: { in: parsed.data.repoIds }, + }, + select: { + id: true, + indexedAt: true, + indexedCommitHash: true, + latestIndexingJobId: true, + }, + }); + const jobIds = repositories.flatMap((repo) => + repo.latestIndexingJobId ? [repo.latestIndexingJobId] : [], + ); + const jobs = await getBullMQClient().getJobs( + REPO_INDEX_QUEUE, + jobIds, + ); + + return { + repositories: repositories.map((repo) => ({ + repoId: repo.id, + indexedAt: repo.indexedAt?.toISOString() ?? null, + indexedCommitHash: repo.indexedCommitHash, + latestJob: repo.latestIndexingJobId + ? jobs.get(repo.latestIndexingJobId) ?? null + : null, + })), + } satisfies RepoIndexingStatusesResponse; + }); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result); +}); From 38fe67b66258073c6053d7c35f169ec7e5b3bcc0 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 21:22:29 -0700 Subject: [PATCH 02/34] remove old repos table --- .../web/src/app/(app)/repos/[id]/page.tsx | 198 --- .../repos/components/repoActionsDropdown.tsx | 82 -- .../components/repoActionsMenu.tsx | 0 .../repos/components/repoBranchesTable.tsx | 143 -- .../(app)/repos/components/repoJobsTable.tsx | 363 ----- .../components/reposTable.test.tsx | 0 .../app/(app)/repos/components/reposTable.tsx | 1203 +++++++++++------ .../components/syncIssuePopover.tsx | 0 packages/web/src/app/(app)/repos/layout.tsx | 30 - packages/web/src/app/(app)/repos/page.tsx | 285 ++-- .../src/app/(app)/{reposv2 => repos}/types.ts | 0 .../(app)/reposv2/components/reposTable.tsx | 900 ------------ packages/web/src/app/(app)/reposv2/page.tsx | 130 -- .../api/(server)/repo-index-status/route.ts | 2 +- 14 files changed, 912 insertions(+), 2424 deletions(-) delete mode 100644 packages/web/src/app/(app)/repos/[id]/page.tsx delete mode 100644 packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx rename packages/web/src/app/(app)/{reposv2 => repos}/components/repoActionsMenu.tsx (100%) delete mode 100644 packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx delete mode 100644 packages/web/src/app/(app)/repos/components/repoJobsTable.tsx rename packages/web/src/app/(app)/{reposv2 => repos}/components/reposTable.test.tsx (100%) rename packages/web/src/app/(app)/{reposv2 => repos}/components/syncIssuePopover.tsx (100%) delete mode 100644 packages/web/src/app/(app)/repos/layout.tsx rename packages/web/src/app/(app)/{reposv2 => repos}/types.ts (100%) delete mode 100644 packages/web/src/app/(app)/reposv2/components/reposTable.tsx delete mode 100644 packages/web/src/app/(app)/reposv2/page.tsx diff --git a/packages/web/src/app/(app)/repos/[id]/page.tsx b/packages/web/src/app/(app)/repos/[id]/page.tsx deleted file mode 100644 index 8145a535a..000000000 --- a/packages/web/src/app/(app)/repos/[id]/page.tsx +++ /dev/null @@ -1,198 +0,0 @@ -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" -import { Skeleton } from "@/components/ui/skeleton" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { env } from "@sourcebot/shared" -import { cn, getCodeHostInfoForRepo } from "@/lib/utils" -import { authenticatedPage, type OptionalAuthOptions } from "@/middleware/authenticatedPage" -import { getConfigSettings, repoMetadataSchema } from "@sourcebot/shared" -import { ExternalLink, Info } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { notFound } from "next/navigation" -import { Suspense } from "react" -import { BackButton } from "../../components/backButton" -import { DisplayDate } from "../../components/DisplayDate" -import { RepoBranchesTable } from "../components/repoBranchesTable" -import { RepoJobsTable } from "../components/repoJobsTable" -import { OrgRole } from "@sourcebot/db" - -type RepoDetailPageProps = { - params: Promise<{ id: string }> -} - -export default authenticatedPage(async ({ org, role, prisma }, props) => { - const { id } = await props.params; - const repo = await prisma.repo.findUnique({ - where: { - id: Number.parseInt(id), - orgId: org.id, - }, - include: { - jobs: { - orderBy: { - createdAt: 'desc', - }, - }, - }, - }); - if (!repo) { - notFound(); - } - - const codeHostInfo = getCodeHostInfoForRepo({ - codeHostType: repo.external_codeHostType, - name: repo.name, - displayName: repo.displayName ?? undefined, - externalWebUrl: repo.webUrl ?? undefined, - }); - - const configSettings = await getConfigSettings(env.CONFIG_PATH); - - const nextIndexAttempt = (() => { - const latestJob = repo.jobs.length > 0 ? repo.jobs[0] : null; - if (!latestJob) { - return undefined; - } - - if (latestJob.completedAt) { - return new Date(latestJob.completedAt.getTime() + configSettings.reindexIntervalMs); - } - - return undefined; - })(); - - const repoMetadata = repoMetadataSchema.parse(repo.metadata); - - return ( - <> -
- - -
-
-

{repo.displayName || repo.name}

-
- {codeHostInfo.externalWebUrl && ( - - )} -
- -
- {repo.isArchived && Archived} - {repo.isPublic && Public} -
-
- -
- - - - Created - - - - - -

When this repository was first added to Sourcebot

- - - - - - - - - - - - - Last indexed - - - - - -

The last time this repository was successfully indexed

- - - - - - {repo.indexedAt ? : "Never"} - - - - - - - Scheduled - - - - - -

When the next indexing job is scheduled to run

- - - - - - {nextIndexAttempt ? : "-"} - - -
- - {repoMetadata.indexedRevisions && ( - - -
- Indexed Branches -
- Branches that have been indexed for this repository. Docs - - - }> - - - - - )} - - - - Indexing History - History of all indexing and cleanup jobs for this repository. - - - }> - - - - - - ); -}, { allowAnonymous: true }); \ No newline at end of file diff --git a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx b/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx deleted file mode 100644 index 878ba3f2b..000000000 --- a/packages/web/src/app/(app)/repos/components/repoActionsDropdown.tsx +++ /dev/null @@ -1,82 +0,0 @@ -"use client" - -import { Button } from "@/components/ui/button" -import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuLabel, DropdownMenuSeparator, DropdownMenuTrigger } from "@/components/ui/dropdown-menu" -import { getCodeHostInfoForRepo, isServiceError } from "@/lib/utils" -import { ExternalLink, MoreHorizontal } from "lucide-react" -import Link from "next/link" -import { useState } from "react" -import { indexRepo } from "@/features/repos/actions" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import type { Repo } from "./reposTable" - -interface RepoActionsDropdownProps { - repo: Repo -} - -export const RepoActionsDropdown = ({ repo }: RepoActionsDropdownProps) => { - const [isSyncing, setIsSyncing] = useState(false) - const router = useRouter() - const { toast } = useToast() - - const codeHostInfo = getCodeHostInfoForRepo({ - codeHostType: repo.codeHostType, - name: repo.name, - displayName: repo.displayName ?? undefined, - externalWebUrl: repo.webUrl ?? undefined, - }) - - const handleTriggerSync = async () => { - setIsSyncing(true) - const response = await indexRepo(repo.id) - - if (!isServiceError(response)) { - const { jobId } = response - toast({ - description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, - }) - router.refresh() - } else { - toast({ - description: `❌ Failed to sync repository. ${response.message}`, - }) - } - - setIsSyncing(false) - } - - return ( - - - - - - Actions - - View details - - - Trigger sync - - {repo.webUrl && ( - <> - - - - Open in {codeHostInfo.codeHostName} - - - - - )} - - - ) -} diff --git a/packages/web/src/app/(app)/reposv2/components/repoActionsMenu.tsx b/packages/web/src/app/(app)/repos/components/repoActionsMenu.tsx similarity index 100% rename from packages/web/src/app/(app)/reposv2/components/repoActionsMenu.tsx rename to packages/web/src/app/(app)/repos/components/repoActionsMenu.tsx diff --git a/packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx b/packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx deleted file mode 100644 index dcd043a55..000000000 --- a/packages/web/src/app/(app)/repos/components/repoBranchesTable.tsx +++ /dev/null @@ -1,143 +0,0 @@ -"use client" - -import * as React from "react" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { getCodeHostBrowseAtBranchUrl } from "@/lib/utils" -import Link from "next/link" -import { CodeHostType } from "@sourcebot/db"; - -type RepoBranchesTableProps = { - indexRevisions: string[]; - repoWebUrl: string | null; - repoCodeHostType: CodeHostType; -} - -export const RepoBranchesTable = ({ indexRevisions, repoWebUrl, repoCodeHostType }: RepoBranchesTableProps) => { - const [sorting, setSorting] = React.useState([]) - const [columnFilters, setColumnFilters] = React.useState([]) - - const columns = React.useMemo[]>(() => { - return [ - { - id: "refName", - accessorFn: (row) => row, - header: "Revision", - cell: ({ row }) => { - const refName = row.original; - const shortRefName = refName.replace(/^refs\/(heads|tags)\//, ""); - - const branchUrl = getCodeHostBrowseAtBranchUrl({ - webUrl: repoWebUrl, - codeHostType: repoCodeHostType, - branchName: refName, - }); - - return branchUrl ? ( - - {shortRefName} - - ) : ( - - {shortRefName} - - ) - }, - } - ] - }, [repoCodeHostType, repoWebUrl]); - - const table = useReactTable({ - data: indexRevisions, - columns, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - state: { - sorting, - columnFilters, - }, - initialState: { - pagination: { - pageSize: 5, - }, - }, - }) - - return ( -
-
- table.getColumn("refName")?.setFilterValue(event.target.value)} - className="max-w-sm" - /> -
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ))} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No branches found. - - - )} - -
-
- -
- - -
-
- ) -} diff --git a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx b/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx deleted file mode 100644 index 7f82ff796..000000000 --- a/packages/web/src/app/(app)/repos/components/repoJobsTable.tsx +++ /dev/null @@ -1,363 +0,0 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { AlertCircle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" -import * as React from "react" -import { CopyIconButton } from "../../components/copyIconButton" -import { useMemo } from "react" -import { LightweightCodeHighlighter } from "../../components/lightweightCodeHighlighter" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import { DisplayDate } from "../../components/DisplayDate" -import { LoadingButton } from "@/components/ui/loading-button" -import { indexRepo } from "@/features/repos/actions" -import { isServiceError } from "@/lib/utils" - -// @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS - -export type RepoIndexingJob = { - id: string - type: "INDEX" | "CLEANUP" - status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" - createdAt: Date - updatedAt: Date - completedAt: Date | null - errorMessage: string | null -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: RepoIndexingJob["status"]) => { - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -const getTypeBadge = (type: RepoIndexingJob["type"]) => { - return ( - - {type} - - ) -} - -const getDuration = (start: Date, end: Date | null) => { - if (!end) return "-" - const diff = end.getTime() - start.getTime() - const minutes = Math.floor(diff / 60000) - const seconds = Math.floor((diff % 60000) / 1000) - return `${minutes}m ${seconds}s` -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "type", - header: "Type", - cell: ({ row }) => getTypeBadge(row.getValue("type")), - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "status", - header: "Status", - cell: ({ row }) => { - const job = row.original - return ( -
- {getStatusBadge(row.getValue("status"))} - {job.errorMessage && ( - - - - - - - - {job.errorMessage} - - - - - )} -
- ) - }, - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "createdAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => , - }, - { - accessorKey: "completedAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const completedAt = row.getValue("completedAt") as Date | null; - if (!completedAt) { - return "-"; - } - - return - }, - }, - { - id: "duration", - header: "Duration", - cell: ({ row }) => { - const job = row.original - return getDuration(job.createdAt, job.completedAt) - }, - }, - { - accessorKey: "id", - header: "Job ID", - cell: ({ row }) => { - const id = row.getValue("id") as string - return ( -
- {id} - { - navigator.clipboard.writeText(id); - return true; - }} /> -
- ) - }, - }, -] - -export const RepoJobsTable = ({ - data, - repoId, - isIndexButtonVisible, -}: { - data: RepoIndexingJob[], - repoId: number, - isIndexButtonVisible: boolean, -}) => { - const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) - const [columnFilters, setColumnFilters] = React.useState([]) - const [columnVisibility, setColumnVisibility] = React.useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const [isIndexSubmitting, setIsIndexSubmitting] = React.useState(false); - const onIndexButtonClick = React.useCallback(async () => { - setIsIndexSubmitting(true); - const response = await indexRepo(repoId); - - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, - }) - router.refresh(); - } else { - toast({ - description: `❌ Failed to index repository. ${response.message}`, - }); - } - - setIsIndexSubmitting(false); - }, [repoId, router, toast]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - state: { - sorting, - columnFilters, - columnVisibility, - }, - }) - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - } = useMemo(() => { - return { - numCompleted: data.filter((job) => job.status === "COMPLETED").length, - numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, - numPending: data.filter((job) => job.status === "PENDING").length, - numFailed: data.filter((job) => job.status === "FAILED").length, - }; - }, [data]); - - return ( -
-
- - - - -
- - - {isIndexButtonVisible && ( - - - Trigger sync - - )} -
-
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No indexing jobs found. - - - )} - -
-
- -
-
- {table.getFilteredRowModel().rows.length} job(s) total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/reposv2/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx similarity index 100% rename from packages/web/src/app/(app)/reposv2/components/reposTable.test.tsx rename to packages/web/src/app/(app)/repos/components/reposTable.test.tsx diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index 865f19402..0182f859b 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -1,511 +1,900 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { cn, getCodeHostCommitUrl, getCodeHostIcon, getRepoImageSrc, isServiceError } from "@/lib/utils" +"use client"; + +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, +} from "@/components/ui/input-group"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn, getCodeHostIcon, getRepoImageSrc } from "@/lib/utils"; +import type { CodeHostType } from "@sourcebot/db"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { useDebounce } from "@uidotdev/usehooks"; import { type ColumnDef, - type VisibilityState, flexRender, getCoreRowModel, useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { ArrowDown, ArrowUp, ArrowUpDown, Loader2, RefreshCwIcon } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { useEffect, useRef, useState } from "react" -import { getBrowsePath } from "../../browse/hooks/utils" -import { useRouter, useSearchParams, usePathname } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast"; -import { DisplayDate } from "../../components/DisplayDate" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { NotificationDot } from "../../components/notificationDot" -import { CodeHostType } from "@sourcebot/db" -import { useHotkeys } from "react-hotkeys-hook" -import { indexRepo } from "@/features/repos/actions" -import { RepoActionsDropdown } from "./repoActionsDropdown" - -// @see: https://v0.app/chat/repo-indexing-status-uhjdDim8OUS +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, Check, Loader2, Search } from "lucide-react"; +import Image from "next/image"; +import Link from "next/link"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import { DisplayDate } from "../../components/DisplayDate"; +import { getBrowsePath } from "../../browse/hooks/utils"; +import type { RepoIndexingStatusesResponse } from "../types"; +import { RepoActionsMenu } from "./repoActionsMenu"; +import { SyncIssuePopover } from "./syncIssuePopover"; + +const POLL_INTERVAL_MS = 5_000; +const COMPLETED_BADGE_VISIBLE_MS = 5_000; export type Repo = { - id: number - name: string - displayName: string | null - isArchived: boolean - isPublic: boolean - indexedAt: Date | null - createdAt: Date - webUrl: string | null - codeHostType: CodeHostType - imageUrl: string | null - indexedCommitHash: string | null - latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null - isFirstTimeIndex: boolean -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", + id: number; + name: string; + displayName: string | null; + indexedAt: Date | null; + indexedCommitHash: string | null; + latestJob: WorkloadJob<"repo-index"> | null; + imageUrl: string | null; + webUrl: string | null; + codeHostType: CodeHostType; +}; + +type DisplayedRepo = Repo & { + showCompleted: boolean; + showExplicitSyncing: boolean; +}; + +type SortOrder = "asc" | "desc"; +type SortBy = "name" | "indexedAt"; +type StatusFilter = "all" | "failed" | "warning"; +type SyncAnnotation = "SYNCING" | "WARNING" | "FAILED" | null; + +const getStatusFilter = (value: string | null): StatusFilter => { + if (value === "failed" || value === "warning") { + return value; + } + + return "all"; +}; + +const getRepoName = (repo: Repo) => repo.displayName ?? repo.name; + +const getRepoIndexingStatuses = async ( + repoIds: number[], + signal: AbortSignal, +): Promise => { + const response = await fetch("/api/repo-index-status", { + method: "POST", + headers: { + "Content-Type": "application/json", }, - }, -}) + body: JSON.stringify({ repoIds }), + signal, + }); -const getStatusBadge = (status: Repo["latestJobStatus"]) => { - if (!status) { - return "-"; + if (!response.ok) { + throw new Error("Failed to load repository indexing statuses"); } - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", + return response.json() as Promise; +}; + +const getSyncAnnotation = (repo: Repo): SyncAnnotation => { + const latestJob = repo.latestJob; + const isLatestIndexJob = latestJob?.data.repoId === repo.id + && latestJob.data.type === "INDEX"; + + if (isLatestIndexJob && latestJob.status === "FAILED") { + return repo.indexedAt ? "WARNING" : "FAILED"; } - return {labels[status]} -} - -interface ColumnsContext { - onSortChange: (sortBy: string) => void; - currentSortBy?: string; - currentSortOrder: string; - onTriggerSync: (repoId: number) => void; -} - -export const getColumns = (context: ColumnsContext): ColumnDef[] => [ - { - accessorKey: "displayName", - size: 400, - header: () => { - const isActive = context.currentSortBy === 'displayName'; - const Icon = isActive - ? (context.currentSortOrder === 'asc' ? ArrowUp : ArrowDown) - : ArrowUpDown; - - return ( - - ) - }, - cell: ({ row }) => { - const repo = row.original; - const codeHostIcon = getCodeHostIcon(repo.codeHostType); - const repoImageSrc = repo.imageUrl ? getRepoImageSrc(repo.imageUrl, repo.id) : undefined; - // Internal API routes require authentication headers (cookies/API keys) to be passed through. - // Next.js Image Optimization doesn't forward these headers, so we use unoptimized=true - // to bypass the optimization and make direct requests that include auth headers. - const isInternalApiImage = repoImageSrc?.startsWith('/api/'); - - return ( -
- { - repoImageSrc ? ( + {badge} + + )} + + ); +}; + +const SortableHeader = ({ + label, + column, + sortBy, + sortOrder, + onSortChange, + tooltip, +}: { + label: string; + column: SortBy; + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + tooltip?: string; +}) => { + const isActive = sortBy === column; + const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; + + const button = ( + + ); + + if (!tooltip) { + return button; + } + + return ( + + {button} + +

{tooltip}

+ + + ); +}; + +const getColumns = ({ + sortBy, + sortOrder, + onSortChange, + canRetry, + onRetryScheduled, +}: { + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + canRetry: boolean; + onRetryScheduled: (repoId: number, jobId: string) => void; +}): ColumnDef[] => [ + { + id: "name", + accessorFn: getRepoName, + header: () => ( + + ), + cell: ({ row }) => { + const repo = row.original; + const displayName = getRepoName(repo); + const codeHostIcon = getCodeHostIcon(repo.codeHostType); + const repoImageSrc = repo.imageUrl + ? getRepoImageSrc(repo.imageUrl, repo.id) + : undefined; + const isInternalApiImage = repoImageSrc?.startsWith("/api/"); + const repoBrowseUrl = repo.indexedAt + ? getBrowsePath({ + repoName: repo.name, + path: "", + pathType: "tree", + }) + : null; + + return ( +
+ {repoImageSrc ? ( {`${repo.displayName} - ) : {`${repo.displayName} + )} + {repoBrowseUrl ? ( + + {displayName} + + ) : ( + + {displayName} + + )} + - } +
+ ); + }, + }, + { + accessorKey: "indexedAt", + header: () => ( + + ), + cell: ({ row }) => { + const indexedAt = row.original.indexedAt; + return indexedAt ? : "-"; + }, + }, + { + accessorKey: "indexedCommitHash", + header: "Synced commit", + cell: ({ row }) => { + const hash = row.original.indexedCommitHash; + if (!hash) { + return "-"; + } - {/* Link to the details page (instead of browse) when the repo is indexing - as the code will not be available yet */} + const repo = row.original; + const shortHash = hash.slice(0, 7); + const commitUrl = getBrowsePath({ + repoName: repo.name, + path: "", + pathType: "commit", + commitSha: hash, + }); + const hashElement = ( - {repo.displayName || repo.name} + {shortHash} - {repo.isFirstTimeIndex && ( - - - - - - - - This is the first time Sourcebot is indexing this repository. It may take a few minutes to complete. - - - )} -
- ) - }, - }, - { - accessorKey: "latestJobStatus", - size: 150, - header: "Lastest status", - cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), - }, - { - accessorKey: "indexedAt", - size: 200, - header: () => { - const isActive = context.currentSortBy === 'indexedAt'; - const Icon = isActive - ? (context.currentSortOrder === 'asc' ? ArrowUp : ArrowDown) - : ArrowUpDown; - - return ( - - ) - }, - cell: ({ row }) => { - const indexedAt = row.getValue("indexedAt") as Date | null; - if (!indexedAt) { - return "-"; - } + ); - return ( - - ) - } - }, - { - accessorKey: "indexedCommitHash", - size: 150, - header: "Synced commit", - cell: ({ row }) => { - const hash = row.getValue("indexedCommitHash") as string | null; - if (!hash) { - return "-"; - } - - const smallHash = hash.slice(0, 7); - const repo = row.original; - const codeHostType = repo.codeHostType; - const webUrl = repo.webUrl; - - const commitUrl = getCodeHostCommitUrl({ - webUrl, - codeHostType, - commitHash: hash, - }); - - const HashComponent = commitUrl ? ( - - {smallHash} - - ) : ( - - {smallHash} - - ) - - return ( - - - {HashComponent} - - - {hash} - - - ); + return ( + + {hashElement} + + {hash} + + + ); + }, }, - }, - { - id: "actions", - size: 80, - enableHiding: false, - cell: ({ row }) => { - const repo = row.original - return + { + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ +
+ ), }, - }, -] + ]; -interface ReposTableProps { +type ReposTableProps = { data: Repo[]; currentPage: number; pageSize: number; totalCount: number; - initialSearch: string; - initialStatus: string; - initialSortBy?: string; - initialSortOrder: string; - stats: { - numCompleted: number - numFailed: number - numPending: number - numInProgress: number - numNoJobs: number - } -} - -export const ReposTable = ({ - data, - currentPage, - pageSize, - totalCount, - initialSearch, - initialStatus, - initialSortBy, - initialSortOrder, - stats, + canRetry: boolean; + sortBy: SortBy; + sortOrder: SortOrder; +}; + +export const ReposTable = ({ + data, + currentPage, + pageSize, + totalCount, + canRetry, + sortBy, + sortOrder, }: ReposTableProps) => { - const [columnVisibility, setColumnVisibility] = useState({}) - const [rowSelection, setRowSelection] = useState({}) - const [searchValue, setSearchValue] = useState(initialSearch) - const [isPendingSearch, setIsPendingSearch] = useState(false) - const searchInputRef = useRef(null) + const pathname = usePathname(); const router = useRouter(); const searchParams = useSearchParams(); - const pathname = usePathname(); - const { toast } = useToast(); + const searchParamsString = searchParams.toString(); + const urlSearchValue = searchParams.get("search") ?? ""; + const statusFilter = getStatusFilter(searchParams.get("status")); + const [searchValue, setSearchValue] = useState(urlSearchValue); + const [scheduledRetryJobs, setScheduledRetryJobs] = useState< + Map> + >(() => new Map()); + const debouncedSearchValue = useDebounce(searchValue, 300); + const [isSearchNavigationPending, startSearchTransition] = useTransition(); + const pendingSearchValuesRef = useRef>(new Set()); + const searchInputRef = useRef(null); + const isSearchPending = searchValue !== debouncedSearchValue + || isSearchNavigationPending; - // Focus search box when '/' is pressed - useHotkeys('/', (event) => { + useHotkeys("/", (event) => { event.preventDefault(); searchInputRef.current?.focus(); }); - // Debounced search effect - only runs when searchValue changes useEffect(() => { - setIsPendingSearch(true); - const timer = setTimeout(() => { - const params = new URLSearchParams(searchParams.toString()); - if (searchValue) { - params.set('search', searchValue); - } else { - params.delete('search'); - } - params.set('page', '1'); // Reset to page 1 on search - router.replace(`${pathname}?${params.toString()}`); - setIsPendingSearch(false); - }, 300); - - return () => { - clearTimeout(timer); - setIsPendingSearch(false); - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [searchValue]); - - const updateStatusFilter = (value: string) => { - const params = new URLSearchParams(searchParams.toString()); - if (value === 'all') { - params.delete('status'); - } else { - params.set('status', value); + if (pendingSearchValuesRef.current.delete(urlSearchValue)) { + return; } - params.set('page', '1'); // Reset to page 1 on filter change - router.replace(`${pathname}?${params.toString()}`); - }; - const handleSortChange = (sortBy: string) => { - const params = new URLSearchParams(searchParams.toString()); - - // Toggle sort order if clicking the same column - if (initialSortBy === sortBy) { - const newOrder = initialSortOrder === 'asc' ? 'desc' : 'asc'; - params.set('sortOrder', newOrder); - } else { - // Default to ascending when changing columns - params.set('sortBy', sortBy); - params.set('sortOrder', 'asc'); + setSearchValue(urlSearchValue); + }, [urlSearchValue]); + + useEffect(() => { + if (debouncedSearchValue !== searchValue) { + return; } - - params.set('page', '1'); // Reset to page 1 on sort change - router.replace(`${pathname}?${params.toString()}`); - }; - const handleTriggerSync = async (repoId: number) => { - const response = await indexRepo(repoId); + const nextSearchValue = debouncedSearchValue.trim(); + if (nextSearchValue === urlSearchValue) { + return; + } - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Repository indexing scheduled. Job ID: ${jobId}`, - }); - router.refresh(); + const params = new URLSearchParams(searchParamsString); + if (nextSearchValue) { + params.set("search", nextSearchValue); } else { - toast({ - description: `❌ Failed to sync repository. ${response.message}`, - }); + params.delete("search"); } - }; + params.delete("page"); - const totalPages = Math.ceil(totalCount / pageSize); + const nextSearchParamsString = params.toString(); + if (nextSearchParamsString === searchParamsString) { + return; + } - const columns = getColumns({ - onSortChange: handleSortChange, - currentSortBy: initialSortBy, - currentSortOrder: initialSortOrder, - onTriggerSync: handleTriggerSync + pendingSearchValuesRef.current.add(nextSearchValue); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }, [ + debouncedSearchValue, + pathname, + router, + searchParamsString, + searchValue, + startSearchTransition, + urlSearchValue, + ]); + const onRetryScheduled = useCallback((repoId: number, jobId: string) => { + setScheduledRetryJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + nextJobs.set(repoId, { + id: jobId, + data: { repoId, type: "INDEX" }, + status: "PENDING", + errorMessage: null, + }); + return nextJobs; + }); + }, []); + const retryAwareData = useMemo( + () => data.map((repo) => { + const scheduledJob = scheduledRetryJobs.get(repo.id); + return scheduledJob + ? { ...repo, latestJob: scheduledJob } + : repo; + }), + [data, scheduledRetryJobs], + ); + const pollingTargets = useMemo( + () => retryAwareData.flatMap((repo) => { + const scheduledJob = scheduledRetryJobs.get(repo.id); + return scheduledJob || getSyncAnnotation(repo) === "SYNCING" + ? [{ repoId: repo.id, jobId: repo.latestJob?.id ?? null }] + : []; + }), + [retryAwareData, scheduledRetryJobs], + ); + const pollingRepoIds = useMemo( + () => pollingTargets.map(({ repoId }) => repoId), + [pollingTargets], + ); + const pollingKey = useMemo( + () => pollingTargets.map(({ repoId, jobId }) => `${repoId}:${jobId}`), + [pollingTargets], + ); + const { data: polledStatuses } = useQuery({ + queryKey: ["reposv2-indexing-status", pollingRepoIds, pollingKey], + queryFn: ({ signal }) => getRepoIndexingStatuses(pollingRepoIds, signal), + enabled: pollingRepoIds.length > 0, + placeholderData: (previousData) => previousData, + refetchInterval: (query) => { + const statuses = query.state.data?.repositories; + if (!statuses) { + return POLL_INTERVAL_MS; + } + + return pollingTargets.some((target) => { + const status = statuses.find( + ({ repoId }) => repoId === target.repoId, + ); + if (!status) { + return true; + } + + const latestJob = status.latestJob; + const isLatestIndexJob = latestJob?.data.repoId === status.repoId + && latestJob.data.type === "INDEX"; + const expectedJobId = target.jobId; + + if (expectedJobId && latestJob?.id !== expectedJobId) { + return true; + } + + if (expectedJobId) { + return latestJob?.status === "PENDING" + || latestJob?.status === "IN_PROGRESS"; + } + + return !status.indexedAt + && ( + !isLatestIndexJob + || latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ); + }) + ? POLL_INTERVAL_MS + : false; + }, }); + const completedDuringPollingRepoIds = useMemo(() => { + const pollingTargetsByRepoId = new Map( + pollingTargets.map((target) => [target.repoId, target]), + ); + return new Set( + polledStatuses?.repositories.flatMap((status) => { + const target = pollingTargetsByRepoId.get(status.repoId); + const isExpectedJob = !target?.jobId + || status.latestJob?.id === target.jobId; + return target + && isExpectedJob + && status.latestJob?.status === "COMPLETED" + && status.indexedAt + ? [status.repoId] + : []; + }) ?? [], + ); + }, [polledStatuses, pollingTargets]); + const displayedData = useMemo(() => { + const statusesByRepoId = new Map( + polledStatuses?.repositories.map((status) => [ + status.repoId, + status, + ]) ?? [], + ); + return retryAwareData.map((repo): DisplayedRepo => { + const showCompleted = completedDuringPollingRepoIds.has(repo.id); + const status = statusesByRepoId.get(repo.id); + const scheduledJob = scheduledRetryJobs.get(repo.id); + const showExplicitSyncing = Boolean( + scheduledJob + && ( + !status + || status.latestJob?.id !== scheduledJob.id + || status.latestJob.status === "PENDING" + || status.latestJob.status === "IN_PROGRESS" + ), + ); + if (!status) { + return { ...repo, showCompleted, showExplicitSyncing }; + } + if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { + return { ...repo, showCompleted, showExplicitSyncing }; + } + return { + ...repo, + indexedAt: status.indexedAt + ? new Date(status.indexedAt) + : null, + indexedCommitHash: status.indexedCommitHash, + latestJob: status.latestJob, + showCompleted, + showExplicitSyncing, + }; + }); + }, [ + completedDuringPollingRepoIds, + polledStatuses, + retryAwareData, + scheduledRetryJobs, + ]); + + const onSortChange = useCallback((column: SortBy) => { + const params = new URLSearchParams(searchParams.toString()); + const nextSortOrder = sortBy === column && sortOrder === "asc" + ? "desc" + : "asc"; + params.delete("page"); + if (column === "name") { + params.delete("sortBy"); + } else { + params.set("sortBy", column); + } + if (nextSortOrder === "asc") { + params.delete("sortOrder"); + } else { + params.set("sortOrder", nextSortOrder); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }, [pathname, router, searchParams, sortBy, sortOrder]); + const columns = useMemo( + () => getColumns({ + sortBy, + sortOrder, + onSortChange, + canRetry, + onRetryScheduled, + }), + [ + canRetry, + onRetryScheduled, + onSortChange, + sortBy, + sortOrder, + ], + ); const table = useReactTable({ - data, + data: displayedData, columns, getCoreRowModel: getCoreRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - columnResizeMode: 'onChange', - enableColumnResizing: false, + manualPagination: true, + manualSorting: true, + rowCount: totalCount, state: { - columnVisibility, - rowSelection, + pagination: { + pageIndex: currentPage - 1, + pageSize, + }, + sorting: [{ id: sortBy, desc: sortOrder === "desc" }], }, - }) + }); + const totalPages = Math.max(1, table.getPageCount()); + const firstVisibleRepo = totalCount === 0 + ? 0 + : (currentPage - 1) * pageSize + 1; + const lastVisibleRepo = Math.min(currentPage * pageSize, totalCount); + + const goToPage = (page: number) => { + const params = new URLSearchParams(searchParams.toString()); + if (page === 1) { + params.delete("page"); + } else { + params.set("page", page.toString()); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }; + + const onStatusFilterChange = (status: StatusFilter) => { + const params = new URLSearchParams(searchParams.toString()); + if (status === "all") { + params.delete("status"); + } else { + params.set("status", status); + } + params.delete("page"); + + const query = params.toString(); + router.replace(query ? `${pathname}?${query}` : pathname, { + scroll: false, + }); + }; + + const emptyMessage = statusFilter === "failed" + ? "No failed repositories." + : statusFilter === "warning" + ? "No repositories with warnings." + : "No repositories found."; return ( -
-
- +
+
+ + + + setSearchValue(event.target.value)} - className="ring-0" + placeholder="Search repositories..." /> - {isPendingSearch && ( + {isSearchPending && ( - + )} -
- +
{table.getHeaderGroups().map((headerGroup) => ( - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} ))} - {table.getRowModel().rows?.length ? ( + {table.getRowModel().rows.length > 0 ? ( table.getRowModel().rows.map((row) => ( - + {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} ))} )) ) : ( - - No results. + + {emptyMessage} )}
-
-
- {totalCount} {totalCount !== 1 ? 'repositories' : 'repository'} total - {totalPages > 1 && ` • Page ${currentPage} of ${totalPages}`} + {totalCount > 0 && ( +
+

+ Showing {firstVisibleRepo}-{lastVisibleRepo} of {totalCount} +

+
+

+ Page {currentPage} of {totalPages} +

+
+ + +
+
-
- - -
-
+ )}
- ) -} + ); +}; diff --git a/packages/web/src/app/(app)/reposv2/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx similarity index 100% rename from packages/web/src/app/(app)/reposv2/components/syncIssuePopover.tsx rename to packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx diff --git a/packages/web/src/app/(app)/repos/layout.tsx b/packages/web/src/app/(app)/repos/layout.tsx deleted file mode 100644 index 88738d22e..000000000 --- a/packages/web/src/app/(app)/repos/layout.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { getReposStats } from "@/actions"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; - -interface LayoutProps { - children: React.ReactNode; -} - -export default async function Layout( - props: LayoutProps -) { - const { children } = props; - - const repoStats = await getReposStats(); - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - - return ( -
-
-
-
- {children} -
-
-
-
- ) -} \ No newline at end of file diff --git a/packages/web/src/app/(app)/repos/page.tsx b/packages/web/src/app/(app)/repos/page.tsx index 5e123b735..dd5488824 100644 --- a/packages/web/src/app/(app)/repos/page.tsx +++ b/packages/web/src/app/(app)/repos/page.tsx @@ -1,185 +1,130 @@ -import { sew } from "@/middleware/sew"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; -import { withOptionalAuth } from "@/middleware/withAuth"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + authenticatedPage, + type OptionalAuthOptions, +} from "@/middleware/authenticatedPage"; +import { + REPO_INDEX_QUEUE, + type WorkloadJob, +} from "@sourcebot/shared"; +import { OrgRole, type Prisma } from "@sourcebot/db"; +import { z } from "zod"; import { ReposTable } from "./components/reposTable"; -import { RepoIndexingJobStatus, Prisma } from "@sourcebot/db"; -import z from "zod"; - -const numberSchema = z.coerce.number().int().positive(); const DEFAULT_PAGE_SIZE = 20; +const pageSchema = z.coerce.number().int().positive(); +const sortBySchema = z.enum(["name", "indexedAt"]); +const sortOrderSchema = z.enum(["asc", "desc"]); +const statusSchema = z.enum(["failed", "warning"]); -interface ReposPageProps { +type ReposV2PageProps = { searchParams: Promise<{ page?: string; - pageSize?: string; search?: string; status?: string; sortBy?: string; sortOrder?: string; }>; -} - -export default async function ReposPage(props: ReposPageProps) { - const params = await props.searchParams; - - // Parse pagination parameters with defaults - const page = numberSchema.safeParse(params.page).data ?? 1; - const pageSize = numberSchema.safeParse(params.pageSize).data ?? DEFAULT_PAGE_SIZE; - - // Parse filter parameters - const search = z.string().optional().safeParse(params.search).data ?? ''; - const status = z.enum(['all', 'none', 'COMPLETED', 'IN_PROGRESS', 'PENDING', 'FAILED']).safeParse(params.status).data ?? 'all'; - const sortBy = z.enum(['displayName', 'indexedAt']).safeParse(params.sortBy).data ?? undefined; - const sortOrder = z.enum(['asc', 'desc']).safeParse(params.sortOrder).data ?? 'asc'; - - // Calculate skip for pagination - const skip = (page - 1) * pageSize; - - const _result = await getRepos({ - skip, - take: pageSize, - search, - status, - sortBy, - sortOrder, - }); - if (isServiceError(_result)) { - throw new ServiceErrorException(_result); +}; + +export default authenticatedPage< + ReposV2PageProps, + OptionalAuthOptions +>(async ({ org, prisma, role }, { searchParams }) => { + const params = await searchParams; + const page = pageSchema.safeParse(params.page).data ?? 1; + const search = z.string().optional().safeParse(params.search).data?.trim() ?? ""; + const status = statusSchema.safeParse(params.status).data ?? "all"; + const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; + const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const skip = (page - 1) * DEFAULT_PAGE_SIZE; + const orderBy = sortBy === "indexedAt" + ? [{ indexedAt: sortOrder }, { id: "asc" as const }] + : [{ displayName: sortOrder }, { id: "asc" as const }]; + const failedJobIds = status === "all" + ? [] + : await getBullMQClient().getFailedJobIds(REPO_INDEX_QUEUE); + const where: Prisma.RepoWhereInput = { + orgId: org.id, + ...(search + ? { + displayName: { + contains: search, + mode: "insensitive" as const, + }, + } + : {}), + ...(status === "all" + ? {} + : { + latestIndexingJobId: { in: failedJobIds }, + indexedAt: status === "failed" ? null : { not: null }, + }), + }; + + const [repos, totalCount] = await Promise.all([ + prisma.repo.findMany({ + where, + orderBy, + skip, + take: DEFAULT_PAGE_SIZE, + select: { + id: true, + name: true, + displayName: true, + indexedAt: true, + indexedCommitHash: true, + latestIndexingJobId: true, + imageUrl: true, + webUrl: true, + external_codeHostType: true, + }, + }), + prisma.repo.count({ + where, + }), + ]); + const latestJobIds = repos.flatMap((repo) => + repo.latestIndexingJobId ? [repo.latestIndexingJobId] : [], + ); + let latestJobs = new Map | null>(); + try { + latestJobs = await getBullMQClient().getJobs( + REPO_INDEX_QUEUE, + latestJobIds, + ); + } catch (error) { + console.error("Failed to load latest repository indexing jobs", error); } - const { repos, totalCount, stats } = _result; - return ( - <> -
-

Repositories

-

View and manage your code repositories and their indexing status.

+
+
+

Repositories

- ({ - id: repo.id, - name: repo.name, - displayName: repo.displayName ?? repo.name, - isArchived: repo.isArchived, - isPublic: repo.isPublic, - indexedAt: repo.indexedAt, - createdAt: repo.createdAt, - webUrl: repo.webUrl, - imageUrl: repo.imageUrl, - latestJobStatus: repo.latestIndexingJobStatus, - isFirstTimeIndex: repo.indexedAt === null, - codeHostType: repo.external_codeHostType, - indexedCommitHash: repo.indexedCommitHash, - }))} - currentPage={page} - pageSize={pageSize} - totalCount={totalCount} - initialSearch={search} - initialStatus={status} - initialSortBy={sortBy} - initialSortOrder={sortOrder} - stats={stats} - /> - - ) -} - -interface GetReposParams { - skip: number; - take: number; - search: string; - status: 'all' | 'none' | 'COMPLETED' | 'IN_PROGRESS' | 'PENDING' | 'FAILED'; - sortBy?: 'displayName' | 'indexedAt'; - sortOrder: 'asc' | 'desc'; -} - -const getRepos = async ({ skip, take, search, status, sortBy, sortOrder }: GetReposParams) => sew(() => - withOptionalAuth(async ({ prisma }) => { - const whereClause: Prisma.RepoWhereInput = { - ...(search ? { - displayName: { contains: search, mode: 'insensitive' }, - } : {}), - latestIndexingJobStatus: - status === 'all' ? undefined : - status === 'none' ? null : - status - }; - - // Build orderBy clause based on sortBy and sortOrder - const orderByClause: Prisma.RepoOrderByWithRelationInput = {}; - - if (sortBy === 'displayName') { - orderByClause.displayName = sortOrder === 'asc' ? 'asc' : 'desc'; - } else if (sortBy === 'indexedAt') { - orderByClause.indexedAt = sortOrder === 'asc' ? 'asc' : 'desc'; - } else { - // Default to displayName asc - orderByClause.displayName = 'asc'; - } - - const repos = await prisma.repo.findMany({ - skip, - take, - where: whereClause, - orderBy: orderByClause, - }); - - // Calculate total count using the filtered where clause - const totalCount = await prisma.repo.count({ - where: whereClause - }); - - // Status stats - const [ - numCompleted, - numFailed, - numPending, - numInProgress, - numNoJobs - ] = await Promise.all([ - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.PENDING, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, - } - }), - prisma.repo.count({ - where: { - ...whereClause, - latestIndexingJobStatus: null, - } - }), - ]) - - return { - repos, - totalCount, - stats: { - numCompleted, - numFailed, - numPending, - numInProgress, - numNoJobs, - } - }; - })); \ No newline at end of file +
+ ({ + id: repo.id, + name: repo.name, + displayName: repo.displayName, + indexedAt: repo.indexedAt, + indexedCommitHash: repo.indexedCommitHash, + latestJob: repo.latestIndexingJobId + ? latestJobs.get(repo.latestIndexingJobId) ?? null + : null, + imageUrl: repo.imageUrl, + webUrl: repo.webUrl, + codeHostType: repo.external_codeHostType, + }))} + currentPage={page} + pageSize={DEFAULT_PAGE_SIZE} + totalCount={totalCount} + canRetry={role === OrgRole.OWNER} + sortBy={sortBy} + sortOrder={sortOrder} + /> +
+
+ ); +}, { allowAnonymous: true }); diff --git a/packages/web/src/app/(app)/reposv2/types.ts b/packages/web/src/app/(app)/repos/types.ts similarity index 100% rename from packages/web/src/app/(app)/reposv2/types.ts rename to packages/web/src/app/(app)/repos/types.ts diff --git a/packages/web/src/app/(app)/reposv2/components/reposTable.tsx b/packages/web/src/app/(app)/reposv2/components/reposTable.tsx deleted file mode 100644 index 0182f859b..000000000 --- a/packages/web/src/app/(app)/reposv2/components/reposTable.tsx +++ /dev/null @@ -1,900 +0,0 @@ -"use client"; - -import { Badge } from "@/components/ui/badge"; -import { Button } from "@/components/ui/button"; -import { - InputGroup, - InputGroupAddon, - InputGroupInput, -} from "@/components/ui/input-group"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { cn, getCodeHostIcon, getRepoImageSrc } from "@/lib/utils"; -import type { CodeHostType } from "@sourcebot/db"; -import type { WorkloadJob } from "@sourcebot/shared"; -import { useQuery } from "@tanstack/react-query"; -import { useDebounce } from "@uidotdev/usehooks"; -import { - type ColumnDef, - flexRender, - getCoreRowModel, - useReactTable, -} from "@tanstack/react-table"; -import { ArrowDown, ArrowUp, Check, Loader2, Search } from "lucide-react"; -import Image from "next/image"; -import Link from "next/link"; -import { AnimatePresence, motion, useReducedMotion } from "motion/react"; -import { usePathname, useRouter, useSearchParams } from "next/navigation"; -import { - useCallback, - useEffect, - useMemo, - useRef, - useState, - useTransition, -} from "react"; -import { useHotkeys } from "react-hotkeys-hook"; -import { DisplayDate } from "../../components/DisplayDate"; -import { getBrowsePath } from "../../browse/hooks/utils"; -import type { RepoIndexingStatusesResponse } from "../types"; -import { RepoActionsMenu } from "./repoActionsMenu"; -import { SyncIssuePopover } from "./syncIssuePopover"; - -const POLL_INTERVAL_MS = 5_000; -const COMPLETED_BADGE_VISIBLE_MS = 5_000; - -export type Repo = { - id: number; - name: string; - displayName: string | null; - indexedAt: Date | null; - indexedCommitHash: string | null; - latestJob: WorkloadJob<"repo-index"> | null; - imageUrl: string | null; - webUrl: string | null; - codeHostType: CodeHostType; -}; - -type DisplayedRepo = Repo & { - showCompleted: boolean; - showExplicitSyncing: boolean; -}; - -type SortOrder = "asc" | "desc"; -type SortBy = "name" | "indexedAt"; -type StatusFilter = "all" | "failed" | "warning"; -type SyncAnnotation = "SYNCING" | "WARNING" | "FAILED" | null; - -const getStatusFilter = (value: string | null): StatusFilter => { - if (value === "failed" || value === "warning") { - return value; - } - - return "all"; -}; - -const getRepoName = (repo: Repo) => repo.displayName ?? repo.name; - -const getRepoIndexingStatuses = async ( - repoIds: number[], - signal: AbortSignal, -): Promise => { - const response = await fetch("/api/repo-index-status", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ repoIds }), - signal, - }); - - if (!response.ok) { - throw new Error("Failed to load repository indexing statuses"); - } - - return response.json() as Promise; -}; - -const getSyncAnnotation = (repo: Repo): SyncAnnotation => { - const latestJob = repo.latestJob; - const isLatestIndexJob = latestJob?.data.repoId === repo.id - && latestJob.data.type === "INDEX"; - - if (isLatestIndexJob && latestJob.status === "FAILED") { - return repo.indexedAt ? "WARNING" : "FAILED"; - } - - if ( - !repo.indexedAt - && ( - !isLatestIndexJob - || latestJob.status === "PENDING" - || latestJob.status === "IN_PROGRESS" - ) - ) { - return "SYNCING"; - } - - return null; -}; - -const SyncAnnotationBadge = ({ - repo, - canRetry, - onRetryScheduled, - showCompleted, - showExplicitSyncing, -}: { - repo: Repo; - canRetry: boolean; - onRetryScheduled: (repoId: number, jobId: string) => void; - showCompleted: boolean; - showExplicitSyncing: boolean; -}) => { - const prefersReducedMotion = useReducedMotion(); - const completionKey = showCompleted - ? repo.latestJob?.id ?? repo.indexedAt?.toISOString() ?? `repo:${repo.id}` - : null; - const [expiredCompletionKey, setExpiredCompletionKey] = useState< - string | null - >(null); - useEffect(() => { - if (!completionKey || expiredCompletionKey === completionKey) { - return; - } - - const timeout = window.setTimeout(() => { - setExpiredCompletionKey(completionKey); - }, COMPLETED_BADGE_VISIBLE_MS); - return () => window.clearTimeout(timeout); - }, [completionKey, expiredCompletionKey]); - const annotation = completionKey !== null - && expiredCompletionKey !== completionKey - ? "COMPLETED" - : showExplicitSyncing - ? "SYNCING" - : getSyncAnnotation(repo); - const badge = (() => { - switch (annotation) { - case "COMPLETED": - return ( - - - Completed - - ); - case "SYNCING": - return ( - - - Syncing - - ); - case "WARNING": - return ( - - ); - case "FAILED": - return ( - - ); - default: - return null; - } - })(); - - const isCompleted = annotation === "COMPLETED"; - - return ( - - {annotation && badge && ( - - {badge} - - )} - - ); -}; - -const SortableHeader = ({ - label, - column, - sortBy, - sortOrder, - onSortChange, - tooltip, -}: { - label: string; - column: SortBy; - sortBy: SortBy; - sortOrder: SortOrder; - onSortChange: (column: SortBy) => void; - tooltip?: string; -}) => { - const isActive = sortBy === column; - const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; - - const button = ( - - ); - - if (!tooltip) { - return button; - } - - return ( - - {button} - -

{tooltip}

- - - ); -}; - -const getColumns = ({ - sortBy, - sortOrder, - onSortChange, - canRetry, - onRetryScheduled, -}: { - sortBy: SortBy; - sortOrder: SortOrder; - onSortChange: (column: SortBy) => void; - canRetry: boolean; - onRetryScheduled: (repoId: number, jobId: string) => void; -}): ColumnDef[] => [ - { - id: "name", - accessorFn: getRepoName, - header: () => ( - - ), - cell: ({ row }) => { - const repo = row.original; - const displayName = getRepoName(repo); - const codeHostIcon = getCodeHostIcon(repo.codeHostType); - const repoImageSrc = repo.imageUrl - ? getRepoImageSrc(repo.imageUrl, repo.id) - : undefined; - const isInternalApiImage = repoImageSrc?.startsWith("/api/"); - const repoBrowseUrl = repo.indexedAt - ? getBrowsePath({ - repoName: repo.name, - path: "", - pathType: "tree", - }) - : null; - - return ( -
- {repoImageSrc ? ( - {`${displayName} - ) : ( - {`${displayName} - )} - {repoBrowseUrl ? ( - - {displayName} - - ) : ( - - {displayName} - - )} - -
- ); - }, - }, - { - accessorKey: "indexedAt", - header: () => ( - - ), - cell: ({ row }) => { - const indexedAt = row.original.indexedAt; - return indexedAt ? : "-"; - }, - }, - { - accessorKey: "indexedCommitHash", - header: "Synced commit", - cell: ({ row }) => { - const hash = row.original.indexedCommitHash; - if (!hash) { - return "-"; - } - - const repo = row.original; - const shortHash = hash.slice(0, 7); - const commitUrl = getBrowsePath({ - repoName: repo.name, - path: "", - pathType: "commit", - commitSha: hash, - }); - const hashElement = ( - - {shortHash} - - ); - - return ( - - {hashElement} - - {hash} - - - ); - }, - }, - { - id: "actions", - header: () => Actions, - cell: ({ row }) => ( -
- -
- ), - }, - ]; - -type ReposTableProps = { - data: Repo[]; - currentPage: number; - pageSize: number; - totalCount: number; - canRetry: boolean; - sortBy: SortBy; - sortOrder: SortOrder; -}; - -export const ReposTable = ({ - data, - currentPage, - pageSize, - totalCount, - canRetry, - sortBy, - sortOrder, -}: ReposTableProps) => { - const pathname = usePathname(); - const router = useRouter(); - const searchParams = useSearchParams(); - const searchParamsString = searchParams.toString(); - const urlSearchValue = searchParams.get("search") ?? ""; - const statusFilter = getStatusFilter(searchParams.get("status")); - const [searchValue, setSearchValue] = useState(urlSearchValue); - const [scheduledRetryJobs, setScheduledRetryJobs] = useState< - Map> - >(() => new Map()); - const debouncedSearchValue = useDebounce(searchValue, 300); - const [isSearchNavigationPending, startSearchTransition] = useTransition(); - const pendingSearchValuesRef = useRef>(new Set()); - const searchInputRef = useRef(null); - const isSearchPending = searchValue !== debouncedSearchValue - || isSearchNavigationPending; - - useHotkeys("/", (event) => { - event.preventDefault(); - searchInputRef.current?.focus(); - }); - - useEffect(() => { - if (pendingSearchValuesRef.current.delete(urlSearchValue)) { - return; - } - - setSearchValue(urlSearchValue); - }, [urlSearchValue]); - - useEffect(() => { - if (debouncedSearchValue !== searchValue) { - return; - } - - const nextSearchValue = debouncedSearchValue.trim(); - if (nextSearchValue === urlSearchValue) { - return; - } - - const params = new URLSearchParams(searchParamsString); - if (nextSearchValue) { - params.set("search", nextSearchValue); - } else { - params.delete("search"); - } - params.delete("page"); - - const nextSearchParamsString = params.toString(); - if (nextSearchParamsString === searchParamsString) { - return; - } - - pendingSearchValuesRef.current.add(nextSearchValue); - startSearchTransition(() => { - router.replace( - `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, - { scroll: false }, - ); - }); - }, [ - debouncedSearchValue, - pathname, - router, - searchParamsString, - searchValue, - startSearchTransition, - urlSearchValue, - ]); - const onRetryScheduled = useCallback((repoId: number, jobId: string) => { - setScheduledRetryJobs((currentJobs) => { - const nextJobs = new Map(currentJobs); - nextJobs.set(repoId, { - id: jobId, - data: { repoId, type: "INDEX" }, - status: "PENDING", - errorMessage: null, - }); - return nextJobs; - }); - }, []); - const retryAwareData = useMemo( - () => data.map((repo) => { - const scheduledJob = scheduledRetryJobs.get(repo.id); - return scheduledJob - ? { ...repo, latestJob: scheduledJob } - : repo; - }), - [data, scheduledRetryJobs], - ); - const pollingTargets = useMemo( - () => retryAwareData.flatMap((repo) => { - const scheduledJob = scheduledRetryJobs.get(repo.id); - return scheduledJob || getSyncAnnotation(repo) === "SYNCING" - ? [{ repoId: repo.id, jobId: repo.latestJob?.id ?? null }] - : []; - }), - [retryAwareData, scheduledRetryJobs], - ); - const pollingRepoIds = useMemo( - () => pollingTargets.map(({ repoId }) => repoId), - [pollingTargets], - ); - const pollingKey = useMemo( - () => pollingTargets.map(({ repoId, jobId }) => `${repoId}:${jobId}`), - [pollingTargets], - ); - const { data: polledStatuses } = useQuery({ - queryKey: ["reposv2-indexing-status", pollingRepoIds, pollingKey], - queryFn: ({ signal }) => getRepoIndexingStatuses(pollingRepoIds, signal), - enabled: pollingRepoIds.length > 0, - placeholderData: (previousData) => previousData, - refetchInterval: (query) => { - const statuses = query.state.data?.repositories; - if (!statuses) { - return POLL_INTERVAL_MS; - } - - return pollingTargets.some((target) => { - const status = statuses.find( - ({ repoId }) => repoId === target.repoId, - ); - if (!status) { - return true; - } - - const latestJob = status.latestJob; - const isLatestIndexJob = latestJob?.data.repoId === status.repoId - && latestJob.data.type === "INDEX"; - const expectedJobId = target.jobId; - - if (expectedJobId && latestJob?.id !== expectedJobId) { - return true; - } - - if (expectedJobId) { - return latestJob?.status === "PENDING" - || latestJob?.status === "IN_PROGRESS"; - } - - return !status.indexedAt - && ( - !isLatestIndexJob - || latestJob.status === "PENDING" - || latestJob.status === "IN_PROGRESS" - ); - }) - ? POLL_INTERVAL_MS - : false; - }, - }); - const completedDuringPollingRepoIds = useMemo(() => { - const pollingTargetsByRepoId = new Map( - pollingTargets.map((target) => [target.repoId, target]), - ); - return new Set( - polledStatuses?.repositories.flatMap((status) => { - const target = pollingTargetsByRepoId.get(status.repoId); - const isExpectedJob = !target?.jobId - || status.latestJob?.id === target.jobId; - return target - && isExpectedJob - && status.latestJob?.status === "COMPLETED" - && status.indexedAt - ? [status.repoId] - : []; - }) ?? [], - ); - }, [polledStatuses, pollingTargets]); - const displayedData = useMemo(() => { - const statusesByRepoId = new Map( - polledStatuses?.repositories.map((status) => [ - status.repoId, - status, - ]) ?? [], - ); - return retryAwareData.map((repo): DisplayedRepo => { - const showCompleted = completedDuringPollingRepoIds.has(repo.id); - const status = statusesByRepoId.get(repo.id); - const scheduledJob = scheduledRetryJobs.get(repo.id); - const showExplicitSyncing = Boolean( - scheduledJob - && ( - !status - || status.latestJob?.id !== scheduledJob.id - || status.latestJob.status === "PENDING" - || status.latestJob.status === "IN_PROGRESS" - ), - ); - if (!status) { - return { ...repo, showCompleted, showExplicitSyncing }; - } - if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { - return { ...repo, showCompleted, showExplicitSyncing }; - } - - return { - ...repo, - indexedAt: status.indexedAt - ? new Date(status.indexedAt) - : null, - indexedCommitHash: status.indexedCommitHash, - latestJob: status.latestJob, - showCompleted, - showExplicitSyncing, - }; - }); - }, [ - completedDuringPollingRepoIds, - polledStatuses, - retryAwareData, - scheduledRetryJobs, - ]); - - const onSortChange = useCallback((column: SortBy) => { - const params = new URLSearchParams(searchParams.toString()); - const nextSortOrder = sortBy === column && sortOrder === "asc" - ? "desc" - : "asc"; - params.delete("page"); - if (column === "name") { - params.delete("sortBy"); - } else { - params.set("sortBy", column); - } - if (nextSortOrder === "asc") { - params.delete("sortOrder"); - } else { - params.set("sortOrder", nextSortOrder); - } - const query = params.toString(); - router.push(query ? `${pathname}?${query}` : pathname); - }, [pathname, router, searchParams, sortBy, sortOrder]); - const columns = useMemo( - () => getColumns({ - sortBy, - sortOrder, - onSortChange, - canRetry, - onRetryScheduled, - }), - [ - canRetry, - onRetryScheduled, - onSortChange, - sortBy, - sortOrder, - ], - ); - const table = useReactTable({ - data: displayedData, - columns, - getCoreRowModel: getCoreRowModel(), - manualPagination: true, - manualSorting: true, - rowCount: totalCount, - state: { - pagination: { - pageIndex: currentPage - 1, - pageSize, - }, - sorting: [{ id: sortBy, desc: sortOrder === "desc" }], - }, - }); - const totalPages = Math.max(1, table.getPageCount()); - const firstVisibleRepo = totalCount === 0 - ? 0 - : (currentPage - 1) * pageSize + 1; - const lastVisibleRepo = Math.min(currentPage * pageSize, totalCount); - - const goToPage = (page: number) => { - const params = new URLSearchParams(searchParams.toString()); - if (page === 1) { - params.delete("page"); - } else { - params.set("page", page.toString()); - } - const query = params.toString(); - router.push(query ? `${pathname}?${query}` : pathname); - }; - - const onStatusFilterChange = (status: StatusFilter) => { - const params = new URLSearchParams(searchParams.toString()); - if (status === "all") { - params.delete("status"); - } else { - params.set("status", status); - } - params.delete("page"); - - const query = params.toString(); - router.replace(query ? `${pathname}?${query}` : pathname, { - scroll: false, - }); - }; - - const emptyMessage = statusFilter === "failed" - ? "No failed repositories." - : statusFilter === "warning" - ? "No repositories with warnings." - : "No repositories found."; - - return ( -
-
- - - - - setSearchValue(event.target.value)} - placeholder="Search repositories..." - /> - {isSearchPending && ( - - - - )} - - -
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - {header.isPlaceholder - ? null - : flexRender( - header.column.columnDef.header, - header.getContext(), - )} - - ))} - - ))} - - - {table.getRowModel().rows.length > 0 ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - {flexRender( - cell.column.columnDef.cell, - cell.getContext(), - )} - - ))} - - )) - ) : ( - - - {emptyMessage} - - - )} - -
-
- {totalCount > 0 && ( -
-

- Showing {firstVisibleRepo}-{lastVisibleRepo} of {totalCount} -

-
-

- Page {currentPage} of {totalPages} -

-
- - -
-
-
- )} -
- ); -}; diff --git a/packages/web/src/app/(app)/reposv2/page.tsx b/packages/web/src/app/(app)/reposv2/page.tsx deleted file mode 100644 index dd5488824..000000000 --- a/packages/web/src/app/(app)/reposv2/page.tsx +++ /dev/null @@ -1,130 +0,0 @@ -import { getBullMQClient } from "@/lib/bullmqClient"; -import { - authenticatedPage, - type OptionalAuthOptions, -} from "@/middleware/authenticatedPage"; -import { - REPO_INDEX_QUEUE, - type WorkloadJob, -} from "@sourcebot/shared"; -import { OrgRole, type Prisma } from "@sourcebot/db"; -import { z } from "zod"; -import { ReposTable } from "./components/reposTable"; - -const DEFAULT_PAGE_SIZE = 20; -const pageSchema = z.coerce.number().int().positive(); -const sortBySchema = z.enum(["name", "indexedAt"]); -const sortOrderSchema = z.enum(["asc", "desc"]); -const statusSchema = z.enum(["failed", "warning"]); - -type ReposV2PageProps = { - searchParams: Promise<{ - page?: string; - search?: string; - status?: string; - sortBy?: string; - sortOrder?: string; - }>; -}; - -export default authenticatedPage< - ReposV2PageProps, - OptionalAuthOptions ->(async ({ org, prisma, role }, { searchParams }) => { - const params = await searchParams; - const page = pageSchema.safeParse(params.page).data ?? 1; - const search = z.string().optional().safeParse(params.search).data?.trim() ?? ""; - const status = statusSchema.safeParse(params.status).data ?? "all"; - const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; - const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; - const skip = (page - 1) * DEFAULT_PAGE_SIZE; - const orderBy = sortBy === "indexedAt" - ? [{ indexedAt: sortOrder }, { id: "asc" as const }] - : [{ displayName: sortOrder }, { id: "asc" as const }]; - const failedJobIds = status === "all" - ? [] - : await getBullMQClient().getFailedJobIds(REPO_INDEX_QUEUE); - const where: Prisma.RepoWhereInput = { - orgId: org.id, - ...(search - ? { - displayName: { - contains: search, - mode: "insensitive" as const, - }, - } - : {}), - ...(status === "all" - ? {} - : { - latestIndexingJobId: { in: failedJobIds }, - indexedAt: status === "failed" ? null : { not: null }, - }), - }; - - const [repos, totalCount] = await Promise.all([ - prisma.repo.findMany({ - where, - orderBy, - skip, - take: DEFAULT_PAGE_SIZE, - select: { - id: true, - name: true, - displayName: true, - indexedAt: true, - indexedCommitHash: true, - latestIndexingJobId: true, - imageUrl: true, - webUrl: true, - external_codeHostType: true, - }, - }), - prisma.repo.count({ - where, - }), - ]); - const latestJobIds = repos.flatMap((repo) => - repo.latestIndexingJobId ? [repo.latestIndexingJobId] : [], - ); - let latestJobs = new Map | null>(); - try { - latestJobs = await getBullMQClient().getJobs( - REPO_INDEX_QUEUE, - latestJobIds, - ); - } catch (error) { - console.error("Failed to load latest repository indexing jobs", error); - } - - return ( -
-
-

Repositories

-
-
- ({ - id: repo.id, - name: repo.name, - displayName: repo.displayName, - indexedAt: repo.indexedAt, - indexedCommitHash: repo.indexedCommitHash, - latestJob: repo.latestIndexingJobId - ? latestJobs.get(repo.latestIndexingJobId) ?? null - : null, - imageUrl: repo.imageUrl, - webUrl: repo.webUrl, - codeHostType: repo.external_codeHostType, - }))} - currentPage={page} - pageSize={DEFAULT_PAGE_SIZE} - totalCount={totalCount} - canRetry={role === OrgRole.OWNER} - sortBy={sortBy} - sortOrder={sortOrder} - /> -
-
- ); -}, { allowAnonymous: true }); diff --git a/packages/web/src/app/api/(server)/repo-index-status/route.ts b/packages/web/src/app/api/(server)/repo-index-status/route.ts index a68a95cbc..5f5b29830 100644 --- a/packages/web/src/app/api/(server)/repo-index-status/route.ts +++ b/packages/web/src/app/api/(server)/repo-index-status/route.ts @@ -1,4 +1,4 @@ -import type { RepoIndexingStatusesResponse } from "@/app/(app)/reposv2/types"; +import type { RepoIndexingStatusesResponse } from "@/app/(app)/repos/types"; import { apiHandler } from "@/lib/apiHandler"; import { getBullMQClient } from "@/lib/bullmqClient"; import { From e0b027a0f021e7678eabbfb8849f390800fff1a3 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 21:24:35 -0700 Subject: [PATCH 03/34] wip --- .../app/(app)/repos/components/reposTable.test.tsx | 12 ++++++++++++ .../src/app/(app)/repos/components/reposTable.tsx | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index e2c61d00f..ce4763844 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -169,6 +169,18 @@ describe("ReposTable", () => { ).toBe("/browse/github.com/acme/second/-/tree"); }); + test("does not link a repository name without a synced commit", () => { + renderTable([{ + ...repos[1], + indexedCommitHash: null, + }]); + + expect( + screen.queryByRole("link", { name: "acme/second" }), + ).toBeNull(); + expect(screen.getByText("acme/second")).toBeTruthy(); + }); + test("shows repository actions to everyone and sync only to owners", () => { const { unmount } = renderTable([repos[1]], false); diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index 0182f859b..dd27ade30 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -323,7 +323,7 @@ const getColumns = ({ ? getRepoImageSrc(repo.imageUrl, repo.id) : undefined; const isInternalApiImage = repoImageSrc?.startsWith("/api/"); - const repoBrowseUrl = repo.indexedAt + const repoBrowseUrl = repo.indexedCommitHash ? getBrowsePath({ repoName: repo.name, path: "", From d7de6d14f92d39b1b66d8b46b2739b07a5d7e1e4 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 21:40:34 -0700 Subject: [PATCH 04/34] add banner --- .../components/banners/bannerResolver.test.ts | 76 +++++++++++++++++++ .../components/banners/bannerResolver.tsx | 39 ++++++++++ .../(app)/components/banners/bannerSlot.tsx | 2 + .../repositorySyncIssuesBanner.test.tsx | 73 ++++++++++++++++++ .../banners/repositorySyncIssuesBanner.tsx | 62 +++++++++++++++ .../src/app/(app)/components/banners/types.ts | 4 + packages/web/src/app/(app)/layout.tsx | 40 +++++++++- packages/web/src/app/(app)/repos/page.tsx | 4 +- 8 files changed, 297 insertions(+), 3 deletions(-) create mode 100644 packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx create mode 100644 packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts index 2cc92230d..b70284f45 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts @@ -21,6 +21,7 @@ vi.mock('./invoicePastDueBanner', () => ({ InvoicePastDueBanner: () => null })); vi.mock('./servicePingFailedBanner', () => ({ ServicePingFailedBanner: () => null })); vi.mock('./trialBanner', () => ({ TrialBanner: () => null })); vi.mock('./upgradeAvailableBanner', () => ({ UpgradeAvailableBanner: () => null })); +vi.mock('./repositorySyncIssuesBanner', () => ({ RepositorySyncIssuesBanner: () => null })); import { resolveActiveBanner, type BannerContext } from './bannerResolver'; @@ -79,6 +80,7 @@ const makeContext = (overrides: Partial = {}): BannerContext => ( hasPermissionSyncEntitlement: false, hasPendingFirstSync: false, permissionSyncIssues: [], + repositorySyncIssueCounts: { failedCount: 0, warningCount: 0 }, dismissals: {}, today: TODAY, now: NOW, @@ -134,6 +136,80 @@ describe('resolveActiveBanner', () => { })); expect(result?.id).toBe('permissionSync'); }); + + test('permission sync outranks repository sync failures', () => { + const result = resolveActiveBanner(makeContext({ + hasPermissionSyncEntitlement: true, + hasPendingFirstSync: true, + repositorySyncIssueCounts: { failedCount: 1, warningCount: 0 }, + })); + expect(result?.id).toBe('permissionSync'); + }); + + test('repository sync failures outrank trial notices', () => { + const result = resolveActiveBanner(makeContext({ + license: makeLicense({ + status: 'trialing', + trialEnd: daysFromNow(7), + }), + repositorySyncIssueCounts: { failedCount: 1, warningCount: 0 }, + })); + expect(result?.id).toBe('repositorySyncFailed'); + }); + + test('trial notices outrank repository sync warnings', () => { + const result = resolveActiveBanner(makeContext({ + license: makeLicense({ + status: 'trialing', + trialEnd: daysFromNow(7), + }), + repositorySyncIssueCounts: { failedCount: 0, warningCount: 1 }, + })); + expect(result?.id).toBe('trial'); + }); + }); + + describe('repository sync issues', () => { + test('shows failures to owners', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncIssueCounts: { failedCount: 2, warningCount: 1 }, + })); + expect(result?.id).toBe('repositorySyncFailed'); + expect(result?.dismissible).toBe(true); + expect(result?.audience).toBe('owner'); + }); + + test('shows warnings when there are no failures', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncIssueCounts: { failedCount: 0, warningCount: 2 }, + })); + expect(result?.id).toBe('repositorySyncWarning'); + expect(result?.dismissible).toBe(true); + }); + + test('hides issues from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + repositorySyncIssueCounts: { failedCount: 1, warningCount: 1 }, + })); + expect(result).toBeNull(); + }); + + test('does not let a warning dismissal suppress a later failure', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncIssueCounts: { failedCount: 1, warningCount: 1 }, + dismissals: { repositorySyncWarning: TODAY }, + })); + expect(result?.id).toBe('repositorySyncFailed'); + }); + + test('hides failures dismissed today', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncIssueCounts: { failedCount: 1, warningCount: 1 }, + dismissals: { repositorySyncFailed: TODAY }, + })); + expect(result).toBeNull(); + }); }); describe('audience filtering', () => { diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx index 879abe251..768b6df7d 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx @@ -15,6 +15,7 @@ import { InvoicePastDueBanner } from "./invoicePastDueBanner"; import { ServicePingFailedBanner } from "./servicePingFailedBanner"; import { TrialBanner } from "./trialBanner"; import { UpgradeAvailableBanner } from "./upgradeAvailableBanner"; +import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; import type { PermissionSyncStatusResponse } from "@/app/api/(server)/ee/permissionSyncStatus/api"; // Mirrors the value in `lighthouse: lambda/serviceError.ts` and the gating @@ -30,6 +31,10 @@ export interface BannerContext { hasPermissionSyncEntitlement: boolean; hasPendingFirstSync: boolean; permissionSyncIssues: PermissionSyncStatusResponse['issues']; + repositorySyncIssueCounts: { + failedCount: number; + warningCount: number; + }; dismissals: Partial>; today: string; now: Date; @@ -178,6 +183,40 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { }); } + const { + failedCount: repositorySyncFailedCount, + warningCount: repositorySyncWarningCount, + } = ctx.repositorySyncIssueCounts; + if (repositorySyncFailedCount > 0) { + banners.push({ + id: 'repositorySyncFailed', + priority: BannerPriority.REPOSITORY_SYNC_FAILED, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } else if (repositorySyncWarningCount > 0) { + banners.push({ + id: 'repositorySyncWarning', + priority: BannerPriority.REPOSITORY_SYNC_WARNING, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + const upgrade = getUpgradeAvailability(ctx); if (upgrade) { banners.push({ diff --git a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx index 3c2b93d5d..c84aeaeea 100644 --- a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx @@ -9,6 +9,8 @@ const KNOWN_BANNER_IDS: BannerId[] = [ 'licenseReboundElsewhere', 'invoicePastDue', 'permissionSync', + 'repositorySyncFailed', + 'repositorySyncWarning', 'licenseExpiryHeadsUp', 'trial', 'servicePingFailed', diff --git a/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx new file mode 100644 index 000000000..5a8ada7e4 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.test.tsx @@ -0,0 +1,73 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ + icon, + title, + description, + action, + }: { + icon?: ReactNode; + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+ {icon} +
{title}
+
{description}
+
{action}
+
+ ), +})); + +afterEach(cleanup); + +describe("RepositorySyncIssuesBanner", () => { + test("summarizes failures and warnings with links to both filters", () => { + render( + , + ); + + expect(screen.getByText("3 repositories need attention")).toBeTruthy(); + expect(screen.getByText( + "1 repository failed to sync and is unavailable. 2 repositories have warnings and may contain stale results.", + )).toBeTruthy(); + expect( + screen.getByRole("link", { name: "View failed" }).getAttribute("href"), + ).toBe("/repos?status=failed"); + expect( + screen.getByRole("link", { name: "View warnings" }).getAttribute("href"), + ).toBe("/repos?status=warning"); + }); + + test("uses singular copy and only renders the relevant action", () => { + render( + , + ); + + expect(screen.getByText("1 repository needs attention")).toBeTruthy(); + expect(screen.getByText( + "1 repository has a warning and may contain stale results.", + )).toBeTruthy(); + expect(screen.queryByRole("link", { name: "View failed" })).toBeNull(); + expect(screen.getByRole("link", { name: "View warnings" })).toBeTruthy(); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx new file mode 100644 index 000000000..d8a270c89 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositorySyncIssuesBanner.tsx @@ -0,0 +1,62 @@ +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; +import { AlertTriangle } from "lucide-react"; +import Link from "next/link"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +interface RepositorySyncIssuesBannerProps extends BannerProps { + failedCount: number; + warningCount: number; +} + +const pluralizeRepository = (count: number) => + count === 1 ? "repository" : "repositories"; + +export function RepositorySyncIssuesBanner({ + id, + dismissible, + failedCount, + warningCount, +}: RepositorySyncIssuesBannerProps) { + const totalCount = failedCount + warningCount; + const descriptions = [ + failedCount > 0 + ? `${failedCount} ${pluralizeRepository(failedCount)} failed to sync and ${failedCount === 1 ? "is" : "are"} unavailable.` + : null, + warningCount > 0 + ? `${warningCount} ${pluralizeRepository(warningCount)} ${warningCount === 1 ? "has a warning" : "have warnings"} and may contain stale results.` + : null, + ].filter((description): description is string => description !== null); + + return ( + 0 && "text-destructive", + )} + /> + )} + title={`${totalCount} ${pluralizeRepository(totalCount)} ${totalCount === 1 ? "needs" : "need"} attention`} + description={descriptions.join(" ")} + action={( + <> + {failedCount > 0 && ( + + )} + {warningCount > 0 && ( + + )} + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/types.ts b/packages/web/src/app/(app)/components/banners/types.ts index 1edbcd186..b195c7ee7 100644 --- a/packages/web/src/app/(app)/components/banners/types.ts +++ b/packages/web/src/app/(app)/components/banners/types.ts @@ -7,8 +7,10 @@ export const BannerPriority = { SERVICE_PING_ENFORCED: 95, INVOICE_PAST_DUE: 90, PERMISSION_SYNC: 50, + REPOSITORY_SYNC_FAILED: 45, TRIAL: 25, LICENSE_EXPIRY_HEADS_UP: 20, + REPOSITORY_SYNC_WARNING: 15, SERVICE_PING_FAILED: 10, UPGRADE_AVAILABLE: 5, } as const; @@ -18,6 +20,8 @@ export type BannerId = | 'licenseReboundElsewhere' | 'invoicePastDue' | 'permissionSync' + | 'repositorySyncFailed' + | 'repositorySyncWarning' | 'licenseExpiryHeadsUp' | 'trial' | 'servicePingFailed' diff --git a/packages/web/src/app/(app)/layout.tsx b/packages/web/src/app/(app)/layout.tsx index 7f5975c5d..ad52a4b84 100644 --- a/packages/web/src/app/(app)/layout.tsx +++ b/packages/web/src/app/(app)/layout.tsx @@ -14,7 +14,7 @@ import { PendingApprovalCard } from "../../features/membership/components/pendin import { SubmitJoinRequestCard } from "../../features/membership/components/submitJoinRequestCard"; import { NotProvisionedCard } from "@/features/membership/components/notProvisionedCard"; import { isScimEnabled } from "@/features/scim/utils"; -import { env, getOfflineLicenseMetadata, SOURCEBOT_VERSION, isMemberApprovalRequired } from "@sourcebot/shared"; +import { env, getOfflineLicenseMetadata, REPO_INDEX_QUEUE, SOURCEBOT_VERSION, isMemberApprovalRequired } from "@sourcebot/shared"; import { hasEntitlement, isAnonymousAccessEnabled } from "@/lib/entitlements"; import { GcpIapAuth } from "./components/gcpIapAuth"; import { JoinOrganizationCard } from "@/features/membership/components/joinOrganizationCard"; @@ -36,12 +36,46 @@ import { tryGetLatestSourcebotTag } from "./components/banners/actions"; import { LanguageModelProvider } from "@/features/chat/languageModelContext"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { NavigationGuardProvider } from "next-navigation-guard"; +import { getBullMQClient } from "@/lib/bullmqClient"; interface LayoutProps { children: React.ReactNode; sidebar: React.ReactNode; } +const getRepositorySyncIssueCounts = async (orgId: number) => { + try { + const failedJobIds = await getBullMQClient().getFailedJobIds( + REPO_INDEX_QUEUE, + ); + if (failedJobIds.length === 0) { + return { failedCount: 0, warningCount: 0 }; + } + + const [failedCount, warningCount] = await Promise.all([ + __unsafePrisma.repo.count({ + where: { + orgId, + latestIndexingJobId: { in: failedJobIds }, + indexedAt: null, + }, + }), + __unsafePrisma.repo.count({ + where: { + orgId, + latestIndexingJobId: { in: failedJobIds }, + indexedAt: { not: null }, + }, + }), + ]); + + return { failedCount, warningCount }; + } catch (error) { + console.error("Failed to load repository sync issue counts", error); + return { failedCount: 0, warningCount: 0 }; + } +}; + export default async function Layout(props: LayoutProps) { const { children, @@ -169,6 +203,9 @@ export default async function Layout(props: LayoutProps) { permissionSyncStatus !== null && !isServiceError(permissionSyncStatus) ? permissionSyncStatus.issues : []; + const repositorySyncIssueCounts = role === OrgRole.OWNER + ? await getRepositorySyncIssueCounts(org.id) + : { failedCount: 0, warningCount: 0 }; const offlineLicense = getOfflineLicenseMetadata(); const license = offlineLicense @@ -203,6 +240,7 @@ export default async function Layout(props: LayoutProps) { hasPermissionSyncEntitlement={hasPermissionSyncEntitlement} hasPendingFirstSync={hasPendingFirstSync} permissionSyncIssues={permissionSyncIssues} + repositorySyncIssueCounts={repositorySyncIssueCounts} currentVersion={SOURCEBOT_VERSION} latestVersion={latestVersion} /> diff --git a/packages/web/src/app/(app)/repos/page.tsx b/packages/web/src/app/(app)/repos/page.tsx index dd5488824..b4e1a65d5 100644 --- a/packages/web/src/app/(app)/repos/page.tsx +++ b/packages/web/src/app/(app)/repos/page.tsx @@ -17,7 +17,7 @@ const sortBySchema = z.enum(["name", "indexedAt"]); const sortOrderSchema = z.enum(["asc", "desc"]); const statusSchema = z.enum(["failed", "warning"]); -type ReposV2PageProps = { +type ReposPageProps = { searchParams: Promise<{ page?: string; search?: string; @@ -28,7 +28,7 @@ type ReposV2PageProps = { }; export default authenticatedPage< - ReposV2PageProps, + ReposPageProps, OptionalAuthOptions >(async ({ org, prisma, role }, { searchParams }) => { const params = await searchParams; From 1fd72914594cfa2e5f67a28a0fb99be5ec93771d Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 21:57:15 -0700 Subject: [PATCH 05/34] remove repository carousel --- packages/shared/src/env.server.ts | 2 - packages/web/src/actions.ts | 56 +------ .../src/app/(app)/chat/chatLandingPage.tsx | 56 +------ .../app/(app)/chat/components/demoCards.tsx | 157 ------------------ .../(app)/components/repositoryCarousel.tsx | 124 -------------- .../search/components/searchLandingPage.tsx | 18 +- packages/web/src/types.ts | 25 --- 7 files changed, 6 insertions(+), 432 deletions(-) delete mode 100644 packages/web/src/app/(app)/chat/components/demoCards.tsx delete mode 100644 packages/web/src/app/(app)/components/repositoryCarousel.tsx delete mode 100644 packages/web/src/types.ts diff --git a/packages/shared/src/env.server.ts b/packages/shared/src/env.server.ts index 2199df140..618e1a857 100644 --- a/packages/shared/src/env.server.ts +++ b/packages/shared/src/env.server.ts @@ -348,8 +348,6 @@ const options = { DEBUG_ENABLE_REACT_SCAN: booleanSchema.default('false'), DEBUG_ENABLE_REACT_GRAB: booleanSchema.default('false'), - SOURCEBOT_DEMO_EXAMPLES_PATH: z.string().optional(), - DISABLE_API_KEY_USAGE_FOR_NON_OWNER_USERS: booleanSchema.default('false'), DISABLE_API_KEY_CREATION_FOR_NON_OWNER_USERS: booleanSchema diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index c64979173..2c9e2128f 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { ConnectionSyncJobStatus, OrgRole, Prisma } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -220,60 +220,6 @@ export const getRepos = async ({ } satisfies RepositoryQuery)) })); -/** - * Returns a set of aggregated stats about the repos in the org - */ -export const getReposStats = async () => sew(() => - withOptionalAuth(async ({ org, prisma }) => { - const [ - // Total number of repos. - numberOfRepos, - // Number of repos with their first time indexing jobs either - // pending or in progress. - numberOfReposWithFirstTimeIndexingJobsInProgress, - // Number of repos that have been indexed at least once. - numberOfReposWithIndex, - ] = await Promise.all([ - prisma.repo.count({ - where: { - orgId: org.id, - } - }), - prisma.repo.count({ - where: { - orgId: org.id, - indexedAt: null, - jobs: { - some: { - type: RepoIndexingJobType.INDEX, - status: { - in: [ - RepoIndexingJobStatus.PENDING, - RepoIndexingJobStatus.IN_PROGRESS, - ] - } - }, - }, - } - }), - prisma.repo.count({ - where: { - orgId: org.id, - NOT: { - indexedAt: null, - } - } - }) - ]); - - return { - numberOfRepos, - numberOfReposWithFirstTimeIndexingJobsInProgress, - numberOfReposWithIndex, - }; - }) -) - export const getConnectionStats = async () => sew(() => withAuth(async ({ org, prisma }) => { const [ diff --git a/packages/web/src/app/(app)/chat/chatLandingPage.tsx b/packages/web/src/app/(app)/chat/chatLandingPage.tsx index b78e28e10..e0658e889 100644 --- a/packages/web/src/app/(app)/chat/chatLandingPage.tsx +++ b/packages/web/src/app/(app)/chat/chatLandingPage.tsx @@ -1,17 +1,12 @@ -import { getRepos, getReposStats, getSearchContexts } from "@/actions"; +import { getRepos, getSearchContexts } from "@/actions"; import { SourcebotLogo } from "@/app/components/sourcebotLogo"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError, measure } from "@/lib/utils"; +import { isServiceError, } from "@/lib/utils"; import { LandingPageChatBox } from "./components/landingPageChatBox"; import { ChatLandingDropzone } from "./components/chatLandingDropzone"; -import { RepositoryCarousel } from "../components/repositoryCarousel"; -import { Separator } from "@/components/ui/separator"; -import { DemoCards } from "./components/demoCards"; import { env } from "@sourcebot/shared"; -import { loadJsonFile } from "@sourcebot/shared"; -import { DemoExamples, demoExamplesSchema } from "@/types"; import { auth } from "@/auth"; import { hasEntitlement } from "@/lib/entitlements"; import { listAgentSkillCommandsOrEmpty } from "@/ee/features/chat/skills/skillCommands.server"; @@ -26,17 +21,6 @@ export async function ChatLandingPage() { ? await listAgentSkillCommandsOrEmpty() : []; - const carouselRepos = await getRepos({ - where: { - indexedAt: { - not: null, - }, - }, - take: 10, - }); - - const repoStats = await getReposStats(); - if (isServiceError(allRepos)) { throw new ServiceErrorException(allRepos); } @@ -45,23 +29,6 @@ export async function ChatLandingPage() { throw new ServiceErrorException(searchContexts); } - if (isServiceError(carouselRepos)) { - throw new ServiceErrorException(carouselRepos); - } - - if (isServiceError(repoStats)) { - throw new ServiceErrorException(repoStats); - } - - const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => { - try { - return (await measure(() => loadJsonFile(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data; - } catch (error) { - console.error('Failed to load demo examples:', error); - return undefined; - } - })() : undefined; - return (
@@ -81,25 +48,6 @@ export async function ChatLandingPage() { maxImageBytes={env.SOURCEBOT_CHAT_ATTACHMENT_MAX_IMAGE_BYTES} /> - -
- -
- - {demoExamples && ( - <> -
- -
- - - - )}
) diff --git a/packages/web/src/app/(app)/chat/components/demoCards.tsx b/packages/web/src/app/(app)/chat/components/demoCards.tsx deleted file mode 100644 index 7c4fe316b..000000000 --- a/packages/web/src/app/(app)/chat/components/demoCards.tsx +++ /dev/null @@ -1,157 +0,0 @@ -'use client'; - -import { useState } from "react"; -import Image from "next/image"; -import { Search, LibraryBigIcon, Code, Info } from "lucide-react"; -import { Badge } from "@/components/ui/badge"; -import { Card } from "@/components/ui/card"; -import { CardContent } from "@/components/ui/card"; -import { DemoExamples, DemoSearchExample, DemoSearchScope } from "@/types"; -import { cn, getCodeHostIcon } from "@/lib/utils"; -import useCaptureEvent from "@/hooks/useCaptureEvent"; -import { SearchScopeInfoCard } from "@/features/chat/components/chatBox/searchScopeInfoCard"; -import { CodeHostType } from "@sourcebot/db"; - -interface DemoCards { - demoExamples: DemoExamples; -} - -export const DemoCards = ({ - demoExamples, -}: DemoCards) => { - const captureEvent = useCaptureEvent(); - const [selectedFilterSearchScope, setSelectedFilterSearchScope] = useState(null); - - const handleExampleClick = (example: DemoSearchExample) => { - captureEvent('wa_demo_search_example_card_pressed', { - exampleTitle: example.title, - exampleUrl: example.url || '', - }); - - if (example.url) { - window.open(example.url, '_blank'); - } - } - - const getSearchScopeIcon = (searchScope: DemoSearchScope, size: number = 20, isSelected: boolean = false) => { - const sizeClass = size === 12 ? "h-3 w-3" : "h-5 w-5"; - const colorClass = isSelected ? "text-primary-foreground" : "text-muted-foreground"; - - if (searchScope.type === "reposet") { - return ; - } - - if (searchScope.codeHostType) { - const codeHostIcon = getCodeHostIcon(searchScope.codeHostType as CodeHostType); - // When selected, icons need to match the inverted badge colors - // In light mode selected: light icon on dark bg (invert) - // In dark mode selected: dark icon on light bg (no invert, override dark:invert) - const selectedIconClass = isSelected - ? "invert dark:invert-0" - : codeHostIcon.className; - - return ( - {`${searchScope.codeHostType} - ); - } - - return ; - } - - return ( -
- {/* Example Searches Row */} -
-
-
- -

Community Ask Results

-
-
- - {/* Search Scope Filter */} -
-
-
- -
- -
-
-
- Search Scope: -
- { - setSelectedFilterSearchScope(null); - }} - > - All - - {demoExamples.searchScopes.map((searchScope) => ( - { - setSelectedFilterSearchScope(searchScope.id); - }} - > - {getSearchScopeIcon(searchScope, 12, selectedFilterSearchScope === searchScope.id)} - {searchScope.displayName} - - ))} -
- -
- {demoExamples.searchExamples - .filter((example) => { - if (selectedFilterSearchScope === null) return true; - return example.searchScopes.includes(selectedFilterSearchScope); - }) - .map((example) => { - const searchScopes = demoExamples.searchScopes.filter((searchScope) => example.searchScopes.includes(searchScope.id)) - return ( - handleExampleClick(example)} - > - -
-
- {searchScopes.map((searchScope) => ( - - {getSearchScopeIcon(searchScope, 12)} - {searchScope.displayName} - - ))} -
-
-

- {example.title} -

-

- {example.description} -

-
-
- - - ) - })} -
-
-
- ); -}; \ No newline at end of file diff --git a/packages/web/src/app/(app)/components/repositoryCarousel.tsx b/packages/web/src/app/(app)/components/repositoryCarousel.tsx deleted file mode 100644 index 26d92b5b0..000000000 --- a/packages/web/src/app/(app)/components/repositoryCarousel.tsx +++ /dev/null @@ -1,124 +0,0 @@ -'use client'; - -import { - Carousel, - CarouselContent, - CarouselItem, -} from "@/components/ui/carousel"; -import { RepositoryQuery } from "@/lib/types"; -import { getCodeHostInfoForRepo } from "@/lib/utils"; -import clsx from "clsx"; -import Autoscroll from "embla-carousel-auto-scroll"; -import Image from "next/image"; -import Link from "next/link"; - -interface RepositoryCarouselProps { - displayRepos: RepositoryQuery[]; - numberOfReposWithIndex: number; -} - -export function RepositoryCarousel({ - displayRepos, - numberOfReposWithIndex, -}: RepositoryCarouselProps) { - if (numberOfReposWithIndex === 0) { - return ( -
- No repositories found - -
-
- - <> - Create a{" "} - - connection - {" "} - to start indexing repositories - - -
-
-
- ) - } - - return ( -
- - {`${numberOfReposWithIndex} `} - - {numberOfReposWithIndex > 1 ? 'repositories' : 'repository'} - - {` indexed`} - - - - {displayRepos.map((repo, index) => ( - - - - ))} - - -
- ) -} - -interface RepositoryBadgeProps { - repo: RepositoryQuery; -} - -const RepositoryBadge = ({ - repo -}: RepositoryBadgeProps) => { - const { repoIcon, displayName } = (() => { - const info = getCodeHostInfoForRepo({ - codeHostType: repo.codeHostType, - name: repo.repoName, - displayName: repo.repoDisplayName, - externalWebUrl: repo.externalWebUrl, - }); - - return { - repoIcon: {info.codeHostName}, - displayName: info.displayName, - } - })(); - - return ( - - {repoIcon} - - {displayName} - - - ) -} diff --git a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx index e9236607e..f83d2aed3 100644 --- a/packages/web/src/app/(app)/search/components/searchLandingPage.tsx +++ b/packages/web/src/app/(app)/search/components/searchLandingPage.tsx @@ -1,11 +1,10 @@ import { SourcebotLogo } from "@/app/components/sourcebotLogo" -import { RepositoryCarousel } from "../../components/repositoryCarousel" import { Separator } from "@/components/ui/separator" import { SyntaxReferenceGuideHint } from "../../components/syntaxReferenceGuideHint" import Link from "next/link" import { SearchBar } from "../../components/searchBar" import { SearchModeSelector } from "../../components/searchModeSelector" -import { getRepos, getReposStats } from "@/actions" +import { getRepos } from "@/actions" import { ServiceErrorException } from "@/lib/serviceError" import { isServiceError } from "@/lib/utils" @@ -25,10 +24,7 @@ export const SearchLandingPage = async ({ take: 10, }); - const repoStats = await getReposStats(); - if (isServiceError(carouselRepos)) throw new ServiceErrorException(carouselRepos); - if (isServiceError(repoStats)) throw new ServiceErrorException(repoStats); return (
@@ -53,15 +49,7 @@ export const SearchLandingPage = async ({
-
- -
- -
- +
How to search
) -} \ No newline at end of file +} diff --git a/packages/web/src/types.ts b/packages/web/src/types.ts deleted file mode 100644 index 73ff02391..000000000 --- a/packages/web/src/types.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { z } from "zod"; - -export const demoSearchScopeSchema = z.object({ - id: z.number(), - displayName: z.string(), - value: z.string(), - type: z.enum(["repo", "reposet"]), - codeHostType: z.string().optional(), -}) - -export const demoSearchExampleSchema = z.object({ - title: z.string(), - description: z.string(), - url: z.string(), - searchScopes: z.array(z.number()) -}) - -export const demoExamplesSchema = z.object({ - searchScopes: demoSearchScopeSchema.array(), - searchExamples: demoSearchExampleSchema.array(), -}) - -export type DemoExamples = z.infer; -export type DemoSearchScope = z.infer; -export type DemoSearchExample = z.infer; \ No newline at end of file From 127c1e7d15b5dbcc8b19419d1c18c101a61b7a1e Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 22:04:09 -0700 Subject: [PATCH 06/34] add example questions to chat page --- .../src/app/(app)/chat/chatLandingPage.tsx | 7 ++ .../components/exampleQuestionBadges.test.tsx | 78 +++++++++++++++++ .../chat/components/exampleQuestionBadges.tsx | 76 ++++++++++++++++ .../chat/components/exampleQuestions.test.ts | 18 ++++ .../(app)/chat/components/exampleQuestions.ts | 86 +++++++++++++++++++ 5 files changed, 265 insertions(+) create mode 100644 packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx create mode 100644 packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx create mode 100644 packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts create mode 100644 packages/web/src/app/(app)/chat/components/exampleQuestions.ts diff --git a/packages/web/src/app/(app)/chat/chatLandingPage.tsx b/packages/web/src/app/(app)/chat/chatLandingPage.tsx index e0658e889..3b0ff0342 100644 --- a/packages/web/src/app/(app)/chat/chatLandingPage.tsx +++ b/packages/web/src/app/(app)/chat/chatLandingPage.tsx @@ -10,6 +10,8 @@ import { env } from "@sourcebot/shared"; import { auth } from "@/auth"; import { hasEntitlement } from "@/lib/entitlements"; import { listAgentSkillCommandsOrEmpty } from "@/ee/features/chat/skills/skillCommands.server"; +import { ExampleQuestionBadges } from "./components/exampleQuestionBadges"; +import { selectRandomExampleQuestions } from "./components/exampleQuestions"; export async function ChatLandingPage() { const languageModels = await getConfiguredLanguageModelsInfo(); @@ -20,6 +22,7 @@ export async function ChatLandingPage() { const askCommands = session?.user && hasAskEntitlement ? await listAgentSkillCommandsOrEmpty() : []; + const exampleQuestions = selectRandomExampleQuestions(3); if (isServiceError(allRepos)) { throw new ServiceErrorException(allRepos); @@ -47,6 +50,10 @@ export async function ChatLandingPage() { isLoginWallEnabled={env.EXPERIMENT_ASK_GH_ENABLED === 'true'} maxImageBytes={env.SOURCEBOT_CHAT_ATTACHMENT_MAX_IMAGE_BYTES} /> +
diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx new file mode 100644 index 000000000..dc0a0ee7c --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.test.tsx @@ -0,0 +1,78 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { useEffect } from "react"; +import { Node } from "slate"; +import { ReactEditor, useSlate } from "slate-react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { CustomSlateEditor } from "@/features/chat/customSlateEditor"; +import type { CustomEditor } from "@/features/chat/types"; +import { ExampleQuestionBadges } from "./exampleQuestionBadges"; +import type { ExampleQuestion } from "./exampleQuestions"; + +const questions = [ + { + label: "Entry points", + question: "Find the main entry points across the indexed repositories.", + icon: "search", + }, + { + label: "Configuration", + question: "Where is configuration defined and loaded?", + icon: "settings", + }, + { + label: "Testing", + question: "Find examples of how this code is tested.", + icon: "flask", + }, +] as const satisfies readonly ExampleQuestion[]; + +const EditorValue = ({ + onEditor, +}: { + onEditor: (editor: CustomEditor) => void; +}) => { + const editor = useSlate(); + + useEffect(() => { + onEditor(editor); + }, [editor, onEditor]); + + return null; +}; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +describe("ExampleQuestionBadges", () => { + test("inserts the selected question and replaces the previous prompt", () => { + vi.spyOn(ReactEditor, "focus").mockImplementation(() => undefined); + const onEditor = vi.fn<(editor: CustomEditor) => void>(); + render( + + + + , + ); + const editor = onEditor.mock.calls[0]?.[0]; + expect(editor).toBeDefined(); + + expect(screen.getByText("Ask questions about:")).toBeTruthy(); + expect(screen.getByRole("button", { name: "Ask about entry points" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Ask about configuration" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Ask about testing" })).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { + name: "Ask about entry points", + })); + expect(Node.string(editor!)).toBe( + "Find the main entry points across the indexed repositories.", + ); + + fireEvent.click(screen.getByRole("button", { + name: "Ask about testing", + })); + expect(Node.string(editor!)).toBe("Find examples of how this code is tested."); + }); +}); diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx new file mode 100644 index 000000000..f3236d0fe --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestionBadges.tsx @@ -0,0 +1,76 @@ +'use client'; + +import { Button } from "@/components/ui/button"; +import { + BookOpen, + CircleAlert, + Database, + FlaskConical, + ListTodo, + Logs, + Package, + Search, + Settings, + Variable, + type LucideIcon, +} from "lucide-react"; +import { Editor, Transforms } from "slate"; +import { ReactEditor, useSlate } from "slate-react"; +import type { ExampleQuestion, ExampleQuestionIcon } from "./exampleQuestions"; + +const icons: Record = { + book: BookOpen, + database: Database, + error: CircleAlert, + flask: FlaskConical, + list: ListTodo, + logs: Logs, + package: Package, + search: Search, + settings: Settings, + variable: Variable, +}; + +interface ExampleQuestionBadgesProps { + questions: readonly ExampleQuestion[]; + disabled?: boolean; +} + +export function ExampleQuestionBadges({ + questions, + disabled = false, +}: ExampleQuestionBadgesProps) { + const editor = useSlate(); + + const insertQuestion = (question: string) => { + Transforms.select(editor, Editor.range(editor, [])); + Transforms.insertText(editor, question); + ReactEditor.focus(editor); + }; + + return ( +
+ + Ask questions about: + + {questions.map(({ label, question, icon }) => { + const Icon = icons[icon]; + return ( + + ); + })} +
+ ); +} diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts b/packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts new file mode 100644 index 000000000..9ce7a03d8 --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestions.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from "vitest"; +import { + exampleQuestions, + selectRandomExampleQuestions, +} from "./exampleQuestions"; + +describe("selectRandomExampleQuestions", () => { + test("selects three unique questions from the larger pool", () => { + const selectedQuestions = selectRandomExampleQuestions(3, () => 0); + + expect(exampleQuestions.length).toBeGreaterThan(3); + expect(selectedQuestions).toHaveLength(3); + expect(new Set(selectedQuestions.map(({ label }) => label)).size).toBe(3); + expect(selectedQuestions.every((question) => + exampleQuestions.some(({ label }) => label === question.label) + )).toBe(true); + }); +}); diff --git a/packages/web/src/app/(app)/chat/components/exampleQuestions.ts b/packages/web/src/app/(app)/chat/components/exampleQuestions.ts new file mode 100644 index 000000000..3cd4a17ad --- /dev/null +++ b/packages/web/src/app/(app)/chat/components/exampleQuestions.ts @@ -0,0 +1,86 @@ +export type ExampleQuestionIcon = + | "book" + | "database" + | "error" + | "flask" + | "list" + | "logs" + | "package" + | "search" + | "settings" + | "variable"; + +export interface ExampleQuestion { + label: string; + question: string; + icon: ExampleQuestionIcon; +} + +export const exampleQuestions = [ + { + label: "Entry points", + question: "Find the main entry points across all repositories.", + icon: "search", + }, + { + label: "Configuration", + question: "Where is configuration defined and loaded?", + icon: "settings", + }, + { + label: "Testing", + question: "Find examples of how this code is tested.", + icon: "flask", + }, + { + label: "Error handling", + question: "Find common error-handling patterns across the codebase.", + icon: "error", + }, + { + label: "Data models", + question: "Where are the main data models defined?", + icon: "database", + }, + { + label: "TODOs", + question: "Find TODOs and summarize the unfinished work.", + icon: "list", + }, + { + label: "Dependencies", + question: "Where are project dependencies declared and configured?", + icon: "package", + }, + { + label: "Environment", + question: "Where are environment variables read and used?", + icon: "variable", + }, + { + label: "Logging", + question: "Where and how is logging used?", + icon: "logs", + }, + { + label: "Documentation", + question: "Find the most useful developer documentation.", + icon: "book", + }, +] as const satisfies readonly ExampleQuestion[]; + +export const selectRandomExampleQuestions = ( + count: number, + random: () => number = Math.random, +): ExampleQuestion[] => { + const shuffled: ExampleQuestion[] = [...exampleQuestions]; + for (let index = shuffled.length - 1; index > 0; index -= 1) { + const swapIndex = Math.floor(random() * (index + 1)); + [shuffled[index], shuffled[swapIndex]] = [ + shuffled[swapIndex]!, + shuffled[index]!, + ]; + } + + return shuffled.slice(0, Math.max(0, count)); +}; From f3c4f3bf96f724bd0592612c55f63708e604fc15 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Mon, 17 Aug 2026 22:14:06 -0700 Subject: [PATCH 07/34] remove repo indexing job table --- CLAUDE.md | 5 +- packages/backend/src/api.ts | 6 +- .../backend/src/repoIndexWorkload.test.ts | 92 ++----------------- packages/backend/src/repoIndexWorkload.ts | 65 +------------ .../migration.sql | 11 +++ packages/db/prisma/schema.prisma | 39 +------- packages/db/tools/scripts/inject-repo-data.ts | 22 +---- .../src/app/(app)/askgh/[owner]/[repo]/api.ts | 10 +- 8 files changed, 32 insertions(+), 218 deletions(-) create mode 100644 packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql diff --git a/CLAUDE.md b/CLAUDE.md index db943715c..9f2c8623a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,8 +32,9 @@ Use the workload system in `packages/backend` for background work. Define the qu ### Lifecycle state -- In `onStarted`, upsert the workload-specific job row as `IN_PROGRESS`. If the parent resource tracks a `latest...JobId`, update that pointer in the same database transaction. -- Completion and terminal-failure hooks must always update their own historical job row by job ID. Do not condition that update on the job still being latest. Every job row should record its actual outcome. +- BullMQ is the default source of truth for workload lifecycle state. Only persist a separate database job row when the product explicitly requires durable history beyond the queue's retention policy. +- If a parent resource tracks a `latest...JobId`, update that pointer when the job starts so consumers can resolve its state from BullMQ. +- For workloads that persist historical job rows, completion and terminal-failure hooks must always update their own row by job ID. Do not condition that update on the job still being latest. Every persisted job row should record its actual outcome. - `onTerminalFailure` only runs after the job exhausts all retry attempts. Intermediate failures are retried without marking the lifecycle row as terminally failed. - If a completion or failure hook publishes state onto the parent resource, use a conditional `updateMany` keyed by both the resource ID and its `latest...JobId`. This prevents an older hook from overwriting state belonging to a newer job after the execution lock has been released. - Parent-resource state written inside `process` is already serialized by the execution lock. It does not need a latest-job conditional merely because the resource tracks the latest job ID. diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 3cb61ebb8..7e5fdc908 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -7,7 +7,7 @@ import { } from '@bull-board/metrics'; import { Octokit } from '@octokit/rest'; import * as Sentry from "@sentry/node"; -import { PrismaClient, RepoIndexingJobType } from '@sourcebot/db'; +import { PrismaClient } from '@sourcebot/db'; import { createLogger, env, JOB_PRIORITIES } from '@sourcebot/shared'; import express, { NextFunction, Request, Response } from 'express'; import 'express-async-errors'; @@ -176,7 +176,7 @@ const scheduleAndTriggerRepoIndexing = async ({ reindexIntervalMs, { repoId, - type: RepoIndexingJobType.INDEX, + type: "INDEX", }, { priority: JOB_PRIORITIES.SCHEDULED }, ); @@ -185,7 +185,7 @@ const scheduleAndTriggerRepoIndexing = async ({ "repo-index", { repoId, - type: RepoIndexingJobType.INDEX, + type: "INDEX", }, { priority: JOB_PRIORITIES.INTERACTIVE }, ); diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 85b0e9a63..46c290e74 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -32,21 +32,13 @@ import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; const repoFindUnique = vi.fn(); const repoDeleteMany = vi.fn(); -const repoIndexingJobUpsert = vi.fn(); -const repoIndexingJobUpdateMany = vi.fn(); const repoUpdate = vi.fn(); -const repoUpdateMany = vi.fn(); const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ - repoIndexingJob: { - upsert: repoIndexingJobUpsert, - updateMany: repoIndexingJobUpdateMany, - }, repo: { findUnique: repoFindUnique, update: repoUpdate, - updateMany: repoUpdateMany, }, }), ); @@ -102,10 +94,7 @@ describe("repoIndexWorkload", () => { fsMocks.rm.mockResolvedValue(undefined); repoFindUnique.mockResolvedValue(eligibleRepo); repoDeleteMany.mockResolvedValue({ count: 1 }); - repoIndexingJobUpsert.mockResolvedValue(undefined); - repoIndexingJobUpdateMany.mockResolvedValue({ count: 1 }); repoUpdate.mockResolvedValue(undefined); - repoUpdateMany.mockResolvedValue({ count: 1 }); }); test("uses the same repository execution lock for INDEX and CLEANUP", () => { @@ -119,11 +108,11 @@ describe("repoIndexWorkload", () => { expect(workload.executionLock?.durationMs).toBe(60_000); expect(workload.queueSpec.dedupKey).toBeUndefined(); expect(workload.onStarted).toBeUndefined(); - expect(workload.onCompleted).toBeTypeOf("function"); - expect(workload.onTerminalFailure).toBeTypeOf("function"); + expect(workload.onCompleted).toBeUndefined(); + expect(workload.onTerminalFailure).toBeUndefined(); }); - test("validates state and marks an eligible job in progress inside process", async () => { + test("validates state and records the latest job inside process", async () => { await workload.process({ ...processContext, data: { repoId: 42, type: "CLEANUP" }, @@ -139,29 +128,12 @@ describe("repoIndexWorkload", () => { }, }, }); - expect(repoIndexingJobUpsert).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - }, - create: { - id: "job-1", - repoId: 42, - type: "CLEANUP", - status: "IN_PROGRESS", - }, - }); expect(repoUpdate).toHaveBeenCalledWith({ where: { id: 42, }, data: { latestIndexingJobId: "job-1", - latestIndexingJobStatus: "IN_PROGRESS", }, }); expect(repoDeleteMany).toHaveBeenCalledWith({ @@ -201,7 +173,7 @@ describe("repoIndexWorkload", () => { await workload.process(processContext); - expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoUpdate).not.toHaveBeenCalled(); expect(repoDeleteMany).not.toHaveBeenCalled(); expect(fsMocks.readdir).not.toHaveBeenCalled(); expect(lifecycleLogger.debug).toHaveBeenCalledWith( @@ -222,7 +194,7 @@ describe("repoIndexWorkload", () => { data: { repoId: 42, type: "CLEANUP" }, }); - expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoUpdate).not.toHaveBeenCalled(); expect(fsMocks.rm).toHaveBeenCalledWith( expect.stringMatching(/repos\/42$/), { recursive: true, force: true }, @@ -256,7 +228,7 @@ describe("repoIndexWorkload", () => { data: { repoId: 42, type: "CLEANUP" }, }); - expect(repoIndexingJobUpsert).not.toHaveBeenCalled(); + expect(repoUpdate).not.toHaveBeenCalled(); expect(repoDeleteMany).not.toHaveBeenCalled(); expect(fsMocks.readdir).not.toHaveBeenCalled(); expect(lifecycleLogger.debug).toHaveBeenCalledWith( @@ -272,7 +244,7 @@ describe("repoIndexWorkload", () => { data: { repoId: 42, type: "CLEANUP" }, }); - expect(repoIndexingJobUpsert).toHaveBeenCalled(); + expect(repoUpdate).toHaveBeenCalled(); expect(repoDeleteMany).toHaveBeenCalled(); expect(fsMocks.readdir).not.toHaveBeenCalled(); expect(lifecycleLogger.debug).toHaveBeenCalledWith( @@ -280,54 +252,4 @@ describe("repoIndexWorkload", () => { ); }); - test("marks a completed job and fences the repository summary by job id", async () => { - await workload.onCompleted?.(lifecycleContext, undefined); - - expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); - expect(repoUpdateMany).toHaveBeenCalledWith({ - where: { - id: 42, - latestIndexingJobId: "job-1", - }, - data: { - latestIndexingJobStatus: "COMPLETED", - }, - }); - }); - - test("marks a terminal failure and fences the repository summary by job id", async () => { - await workload.onTerminalFailure?.( - lifecycleContext, - new Error("Unable to clone repository"), - ); - - expect(repoIndexingJobUpdateMany).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Unable to clone repository", - }, - }); - expect(repoUpdateMany).toHaveBeenCalledWith({ - where: { - id: 42, - latestIndexingJobId: "job-1", - }, - data: { - latestIndexingJobStatus: "FAILED", - }, - }); - }); }); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 7633e5a4c..258b6ff38 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -1,4 +1,4 @@ -import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db"; +import { PrismaClient, Repo } from "@sourcebot/db"; import { createLogger, getRepoPath, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; import { existsSync } from 'fs'; import { readdir, rm } from 'fs/promises'; @@ -117,52 +117,6 @@ export const createRepoIndexWorkload = ({ } } }, - onCompleted: async ({ data: { repoId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.repoIndexingJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoIndexingJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); - await tx.repo.updateMany({ - where: { - id: repoId, - latestIndexingJobId: jobId, - }, - data: { - latestIndexingJobStatus: RepoIndexingJobStatus.COMPLETED, - }, - }); - }); - }, - onTerminalFailure: async ({ data: { repoId }, jobId }, error) => { - await db.$transaction(async (tx) => { - await tx.repoIndexingJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoIndexingJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); - await tx.repo.updateMany({ - where: { - id: repoId, - latestIndexingJobId: jobId, - }, - data: { - latestIndexingJobStatus: RepoIndexingJobStatus.FAILED, - }, - }); - }); - }, }); type RepoIndexStartDecision = @@ -223,29 +177,12 @@ const prepareRepoIndexJob = async ({ }; } - await tx.repoIndexingJob.upsert({ - where: { - id: jobId, - }, - update: { - status: RepoIndexingJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - repoId, - type: RepoIndexingJobType[type], - status: RepoIndexingJobStatus.IN_PROGRESS, - }, - }); await tx.repo.update({ where: { id: repoId, }, data: { latestIndexingJobId: jobId, - latestIndexingJobStatus: RepoIndexingJobStatus.IN_PROGRESS, }, }); diff --git a/packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql b/packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql new file mode 100644 index 000000000..f735104ea --- /dev/null +++ b/packages/db/prisma/migrations/20260817220832_drop_repo_indexing_job/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "Repo" DROP COLUMN "latestIndexingJobStatus"; + +-- DropTable +DROP TABLE "RepoIndexingJob"; + +-- DropEnum +DROP TYPE "RepoIndexingJobStatus"; + +-- DropEnum +DROP TYPE "RepoIndexingJobType"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 7ebf93f14..9b689603f 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -76,12 +76,10 @@ model Repo { permissionSyncedAt DateTime? /// When the permissions were last synced successfully. latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest repo state. - jobs RepoIndexingJob[] - indexedAt DateTime? /// When the repo was last indexed successfully. - indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). - latestIndexingJobId String? /// The job allowed to publish the latest indexing status. - latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job. - pushedAt DateTime? /// The timestamp of the most recent commit across all branches. + indexedAt DateTime? /// When the repo was last indexed successfully. + indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). + latestIndexingJobId String? /// The latest indexing job in the workload queue. + pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service external_codeHostType CodeHostType /// The type of the external service (e.g., github, gitlab, etc.) @@ -98,35 +96,6 @@ model Repo { @@index([indexedAt]) } -enum RepoIndexingJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -enum RepoIndexingJobType { - INDEX - CLEANUP -} - -model RepoIndexingJob { - id String @id @default(cuid()) - type RepoIndexingJobType - status RepoIndexingJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - metadata Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts - - errorMessage String? - - repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) - repoId Int - - @@index([repoId, type, status]) -} - enum RepoPermissionSyncJobStatus { PENDING IN_PROGRESS diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 609bcdcc7..75e7d1a75 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,7 +2,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; -const NUM_INDEXING_JOBS_PER_REPO = 10000; const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { @@ -37,8 +36,6 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; - const indexingJobTypes = ['INDEX', 'CLEANUP'] as const; - for (let i = 0; i < NUM_REPOS; i++) { const repo = await prisma.repo.create({ data: { @@ -71,23 +68,8 @@ export const injectRepoData: Script = { } }); } - - for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) { - const status = statuses[Math.floor(Math.random() * statuses.length)]; - const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)]; - await prisma.repoIndexingJob.create({ - data: { - repoId: repo.id, - type, - status, - completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, - errorMessage: status === 'FAILED' ? 'Mock indexing error' : null, - metadata: {} - } - }); - } } - console.log(`Created ${NUM_REPOS} repos with associated jobs.`); + console.log(`Created ${NUM_REPOS} repos with associated permission sync jobs.`); } -}; \ No newline at end of file +}; diff --git a/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts b/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts index 64d37f2e6..647f63911 100644 --- a/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts +++ b/packages/web/src/app/(app)/askgh/[owner]/[repo]/api.ts @@ -9,14 +9,6 @@ export const getRepoInfo = async (repoId: number): Promise { const repo = await prisma.repo.findUnique({ where: { id: repoId }, - include: { - jobs: { - orderBy: { - createdAt: 'desc', - }, - take: 1, - }, - }, }); if (!repo) { @@ -31,4 +23,4 @@ export const getRepoInfo = async (repoId: number): Promise Date: Mon, 17 Aug 2026 22:16:56 -0700 Subject: [PATCH 08/34] add clear filter button --- .../repos/components/reposTable.test.tsx | 25 +++++++++++++++ .../app/(app)/repos/components/reposTable.tsx | 32 ++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index ce4763844..a2dd801af 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -114,6 +114,31 @@ describe("ReposTable", () => { expect(screen.queryByRole("button", { name: "Next" })).toBeNull(); }); + test("clears search and status filters from the empty state", () => { + navigation.searchParams = "search=missing&status=failed&page=2&sortBy=indexedAt"; + + renderTable([]); + + fireEvent.click(screen.getByRole("button", { name: "Clear filters" })); + + expect( + (screen.getByPlaceholderText("Search repositories...") as HTMLInputElement) + .value, + ).toBe(""); + expect(navigation.replace).toHaveBeenCalledWith( + "/reposv2?sortBy=indexedAt", + { scroll: false }, + ); + }); + + test("does not offer to clear filters in the unfiltered empty state", () => { + renderTable([]); + + expect( + screen.queryByRole("button", { name: "Clear filters" }), + ).toBeNull(); + }); + test("does not reset pagination when the search value is unchanged", () => { navigation.searchParams = "page=2"; diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index dd27ade30..a61d9a357 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -758,6 +758,25 @@ export const ReposTable = ({ }); }; + const hasActiveFilters = statusFilter !== "all" + || urlSearchValue.trim().length > 0; + const clearFilters = () => { + const params = new URLSearchParams(searchParamsString); + params.delete("search"); + params.delete("status"); + params.delete("page"); + + const nextSearchParamsString = params.toString(); + setSearchValue(""); + pendingSearchValuesRef.current.add(""); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }; + const emptyMessage = statusFilter === "failed" ? "No failed repositories." : statusFilter === "warning" @@ -858,7 +877,18 @@ export const ReposTable = ({ colSpan={columns.length} className="h-28 text-center text-sm text-muted-foreground" > - {emptyMessage} +
+

{emptyMessage}

+ {hasActiveFilters && ( + + )} +
)} From 18d1ff3897d2b5e02050b4a5458b7da6f01f7af1 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 10:51:41 -0700 Subject: [PATCH 09/34] add first sync banner --- .../components/banners/bannerResolver.test.ts | 60 ++++++++++++++++--- .../components/banners/bannerResolver.tsx | 21 ++++++- .../(app)/components/banners/bannerSlot.tsx | 1 + .../banners/repositoryFirstSyncBanner.tsx | 34 +++++++++++ .../src/app/(app)/components/banners/types.ts | 2 + packages/web/src/app/(app)/layout.tsx | 31 ++++++---- 6 files changed, 126 insertions(+), 23 deletions(-) create mode 100644 packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts index b70284f45..b38503475 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts @@ -22,6 +22,7 @@ vi.mock('./servicePingFailedBanner', () => ({ ServicePingFailedBanner: () => nul vi.mock('./trialBanner', () => ({ TrialBanner: () => null })); vi.mock('./upgradeAvailableBanner', () => ({ UpgradeAvailableBanner: () => null })); vi.mock('./repositorySyncIssuesBanner', () => ({ RepositorySyncIssuesBanner: () => null })); +vi.mock('./repositoryFirstSyncBanner', () => ({ RepositoryFirstSyncBanner: () => null })); import { resolveActiveBanner, type BannerContext } from './bannerResolver'; @@ -80,7 +81,7 @@ const makeContext = (overrides: Partial = {}): BannerContext => ( hasPermissionSyncEntitlement: false, hasPendingFirstSync: false, permissionSyncIssues: [], - repositorySyncIssueCounts: { failedCount: 0, warningCount: 0 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 0, warningCount: 0 }, dismissals: {}, today: TODAY, now: NOW, @@ -141,7 +142,7 @@ describe('resolveActiveBanner', () => { const result = resolveActiveBanner(makeContext({ hasPermissionSyncEntitlement: true, hasPendingFirstSync: true, - repositorySyncIssueCounts: { failedCount: 1, warningCount: 0 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 0 }, })); expect(result?.id).toBe('permissionSync'); }); @@ -152,7 +153,7 @@ describe('resolveActiveBanner', () => { status: 'trialing', trialEnd: daysFromNow(7), }), - repositorySyncIssueCounts: { failedCount: 1, warningCount: 0 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 0 }, })); expect(result?.id).toBe('repositorySyncFailed'); }); @@ -163,7 +164,7 @@ describe('resolveActiveBanner', () => { status: 'trialing', trialEnd: daysFromNow(7), }), - repositorySyncIssueCounts: { failedCount: 0, warningCount: 1 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 0, warningCount: 1 }, })); expect(result?.id).toBe('trial'); }); @@ -172,7 +173,7 @@ describe('resolveActiveBanner', () => { describe('repository sync issues', () => { test('shows failures to owners', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncIssueCounts: { failedCount: 2, warningCount: 1 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 2, warningCount: 1 }, })); expect(result?.id).toBe('repositorySyncFailed'); expect(result?.dismissible).toBe(true); @@ -181,7 +182,7 @@ describe('resolveActiveBanner', () => { test('shows warnings when there are no failures', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncIssueCounts: { failedCount: 0, warningCount: 2 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 0, warningCount: 2 }, })); expect(result?.id).toBe('repositorySyncWarning'); expect(result?.dismissible).toBe(true); @@ -190,14 +191,14 @@ describe('resolveActiveBanner', () => { test('hides issues from members', () => { const result = resolveActiveBanner(makeContext({ role: OrgRole.MEMBER, - repositorySyncIssueCounts: { failedCount: 1, warningCount: 1 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 1 }, })); expect(result).toBeNull(); }); test('does not let a warning dismissal suppress a later failure', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncIssueCounts: { failedCount: 1, warningCount: 1 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 1 }, dismissals: { repositorySyncWarning: TODAY }, })); expect(result?.id).toBe('repositorySyncFailed'); @@ -205,13 +206,54 @@ describe('resolveActiveBanner', () => { test('hides failures dismissed today', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncIssueCounts: { failedCount: 1, warningCount: 1 }, + repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 1 }, dismissals: { repositorySyncFailed: TODAY }, })); expect(result).toBeNull(); }); }); + describe('repository first sync', () => { + test('shows first-time syncing repositories to owners', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { + syncingCount: 3, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result?.id).toBe('repositoryFirstSync'); + expect(result?.dismissible).toBe(true); + expect(result?.audience).toBe('owner'); + }); + + test('hides first-time syncing repositories from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + repositorySyncCounts: { + syncingCount: 3, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result).toBeNull(); + }); + + test('repository warnings take priority over first-time syncing', () => { + const result = resolveActiveBanner(makeContext({ + repositorySyncCounts: { + syncingCount: 3, + failedCount: 0, + warningCount: 1, + }, + })); + + expect(result?.id).toBe('repositorySyncWarning'); + }); + }); + describe('audience filtering', () => { test('hides owner-only banner from members', () => { const result = resolveActiveBanner(makeContext({ diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx index 768b6df7d..1ff5ba0ec 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx @@ -16,6 +16,7 @@ import { ServicePingFailedBanner } from "./servicePingFailedBanner"; import { TrialBanner } from "./trialBanner"; import { UpgradeAvailableBanner } from "./upgradeAvailableBanner"; import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; +import { RepositoryFirstSyncBanner } from "./repositoryFirstSyncBanner"; import type { PermissionSyncStatusResponse } from "@/app/api/(server)/ee/permissionSyncStatus/api"; // Mirrors the value in `lighthouse: lambda/serviceError.ts` and the gating @@ -31,7 +32,8 @@ export interface BannerContext { hasPermissionSyncEntitlement: boolean; hasPendingFirstSync: boolean; permissionSyncIssues: PermissionSyncStatusResponse['issues']; - repositorySyncIssueCounts: { + repositorySyncCounts: { + syncingCount: number; failedCount: number; warningCount: number; }; @@ -184,9 +186,10 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { } const { + syncingCount: repositorySyncingCount, failedCount: repositorySyncFailedCount, warningCount: repositorySyncWarningCount, - } = ctx.repositorySyncIssueCounts; + } = ctx.repositorySyncCounts; if (repositorySyncFailedCount > 0) { banners.push({ id: 'repositorySyncFailed', @@ -216,6 +219,20 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { ), }); } + if (repositorySyncingCount > 0) { + banners.push({ + id: 'repositoryFirstSync', + priority: BannerPriority.REPOSITORY_FIRST_SYNC, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } const upgrade = getUpgradeAvailability(ctx); if (upgrade) { diff --git a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx index c84aeaeea..9cb70337b 100644 --- a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx @@ -11,6 +11,7 @@ const KNOWN_BANNER_IDS: BannerId[] = [ 'permissionSync', 'repositorySyncFailed', 'repositorySyncWarning', + 'repositoryFirstSync', 'licenseExpiryHeadsUp', 'trial', 'servicePingFailed', diff --git a/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx new file mode 100644 index 000000000..6e772771e --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx @@ -0,0 +1,34 @@ +import { Button } from "@/components/ui/button"; +import { Loader2 } from "lucide-react"; +import Link from "next/link"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +interface RepositoryFirstSyncBannerProps extends BannerProps { + syncingCount: number; +} + +export function RepositoryFirstSyncBanner({ + id, + dismissible, + syncingCount, +}: RepositoryFirstSyncBannerProps) { + const isSingular = syncingCount === 1; + + return ( + } + title={`${syncingCount} ${isSingular ? "repository is" : "repositories are"} syncing for the first time`} + description={`${isSingular ? "It" : "They"} won't be available until syncing completes.`} + action={( + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/types.ts b/packages/web/src/app/(app)/components/banners/types.ts index b195c7ee7..caaab2627 100644 --- a/packages/web/src/app/(app)/components/banners/types.ts +++ b/packages/web/src/app/(app)/components/banners/types.ts @@ -11,6 +11,7 @@ export const BannerPriority = { TRIAL: 25, LICENSE_EXPIRY_HEADS_UP: 20, REPOSITORY_SYNC_WARNING: 15, + REPOSITORY_FIRST_SYNC: 12, SERVICE_PING_FAILED: 10, UPGRADE_AVAILABLE: 5, } as const; @@ -22,6 +23,7 @@ export type BannerId = | 'permissionSync' | 'repositorySyncFailed' | 'repositorySyncWarning' + | 'repositoryFirstSync' | 'licenseExpiryHeadsUp' | 'trial' | 'servicePingFailed' diff --git a/packages/web/src/app/(app)/layout.tsx b/packages/web/src/app/(app)/layout.tsx index ad52a4b84..58a5ecf2c 100644 --- a/packages/web/src/app/(app)/layout.tsx +++ b/packages/web/src/app/(app)/layout.tsx @@ -43,16 +43,19 @@ interface LayoutProps { sidebar: React.ReactNode; } -const getRepositorySyncIssueCounts = async (orgId: number) => { +const getRepositorySyncCounts = async (orgId: number) => { try { const failedJobIds = await getBullMQClient().getFailedJobIds( REPO_INDEX_QUEUE, ); - if (failedJobIds.length === 0) { - return { failedCount: 0, warningCount: 0 }; - } - const [failedCount, warningCount] = await Promise.all([ + const [unindexedCount, failedCount, warningCount] = await Promise.all([ + __unsafePrisma.repo.count({ + where: { + orgId, + indexedAt: null, + }, + }), __unsafePrisma.repo.count({ where: { orgId, @@ -69,10 +72,14 @@ const getRepositorySyncIssueCounts = async (orgId: number) => { }), ]); - return { failedCount, warningCount }; + return { + syncingCount: Math.max(0, unindexedCount - failedCount), + failedCount, + warningCount, + }; } catch (error) { - console.error("Failed to load repository sync issue counts", error); - return { failedCount: 0, warningCount: 0 }; + console.error("Failed to load repository sync counts", error); + return { syncingCount: 0, failedCount: 0, warningCount: 0 }; } }; @@ -203,9 +210,9 @@ export default async function Layout(props: LayoutProps) { permissionSyncStatus !== null && !isServiceError(permissionSyncStatus) ? permissionSyncStatus.issues : []; - const repositorySyncIssueCounts = role === OrgRole.OWNER - ? await getRepositorySyncIssueCounts(org.id) - : { failedCount: 0, warningCount: 0 }; + const repositorySyncCounts = role === OrgRole.OWNER + ? await getRepositorySyncCounts(org.id) + : { syncingCount: 0, failedCount: 0, warningCount: 0 }; const offlineLicense = getOfflineLicenseMetadata(); const license = offlineLicense @@ -240,7 +247,7 @@ export default async function Layout(props: LayoutProps) { hasPermissionSyncEntitlement={hasPermissionSyncEntitlement} hasPendingFirstSync={hasPendingFirstSync} permissionSyncIssues={permissionSyncIssues} - repositorySyncIssueCounts={repositorySyncIssueCounts} + repositorySyncCounts={repositorySyncCounts} currentVersion={SOURCEBOT_VERSION} latestVersion={latestVersion} /> From cd323f311140d997a6cc1de6613bda6dd5ebb2f8 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 13:59:14 -0700 Subject: [PATCH 10/34] remove permission job tables --- .../ee/accountPermissionSyncWorkload.test.ts | 56 ++--------- .../src/ee/accountPermissionSyncWorkload.ts | 52 ++--------- .../src/ee/repoPermissionSyncWorkload.test.ts | 53 ++--------- .../src/ee/repoPermissionSyncWorkload.ts | 76 ++++----------- .../migration.sql | 5 + .../migration.sql | 5 + packages/db/prisma/schema.prisma | 42 --------- packages/db/tools/scripts/inject-repo-data.ts | 17 +--- .../api.test.ts | 59 +++++++++++- .../ee/accountPermissionSyncJobStatus/api.ts | 28 ++++-- .../ee/permissionSyncStatus/api.test.ts | 61 +++++++++++- .../(server)/ee/permissionSyncStatus/api.ts | 92 ++++++++++++++----- 12 files changed, 248 insertions(+), 298 deletions(-) create mode 100644 packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql create mode 100644 packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts index befdb21af..5fcf9691d 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.test.ts @@ -102,18 +102,12 @@ const accountUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const repoFindMany = vi.fn().mockResolvedValue([]); const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); const permissionDeleteMany = vi.fn().mockResolvedValue({ count: 95 }); -const permissionSyncJobUpsert = vi.fn(); -const permissionSyncJobUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const transactionClient = { account: { findUnique: accountFindUnique, update: accountUpdate, updateMany: accountUpdateMany, }, - accountPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, }; const transaction = vi.fn( ( @@ -140,10 +134,6 @@ const db = { repo: { findMany: repoFindMany, }, - accountPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, $transaction: transaction, } as unknown as PrismaClient; @@ -189,7 +179,6 @@ beforeEach(() => { repoFindMany.mockResolvedValue([]); permissionCreateMany.mockResolvedValue({ count: 0 }); permissionDeleteMany.mockResolvedValue({ count: 95 }); - permissionSyncJobUpdateMany.mockResolvedValue({ count: 1 }); accountUpdateMany.mockResolvedValue({ count: 1 }); }); @@ -406,40 +395,19 @@ describe("accountPermissionSyncWorkload", () => { expect(transaction).not.toHaveBeenCalled(); }); - test("marks a job as in progress when started", async () => { + test("records the latest job when started", async () => { await createWorkload().onStarted?.(lifecycleContext); - expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ - where: { id: "job_1" }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - }, - create: { - id: "job_1", - accountId: "account_1", - status: "IN_PROGRESS", - }, - }); expect(accountUpdate).toHaveBeenCalledWith({ where: { id: "account_1" }, data: { latestPermissionSyncJobId: "job_1" }, }); - expect(transaction).toHaveBeenCalledOnce(); + expect(transaction).not.toHaveBeenCalled(); }); - test("marks a job completed and clears the account issue when it is still latest", async () => { + test("clears the account issue when the completed job is still latest", async () => { await createWorkload().onCompleted?.(lifecycleContext, undefined); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ - where: { id: "job_1" }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); expect(accountUpdateMany).toHaveBeenCalledWith({ where: { id: "account_1", @@ -458,21 +426,11 @@ describe("accountPermissionSyncWorkload", () => { expect(transaction).toHaveBeenCalledOnce(); }); - test("marks a job failed after terminal failure", async () => { + test("reports terminal failures", async () => { const error = new Error("Upstream unavailable"); await createWorkload().onTerminalFailure?.(lifecycleContext, error); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith( - expect.objectContaining({ - where: { id: "job_1" }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Upstream unavailable", - }, - }), - ); expect(mocks.captureException).toHaveBeenCalledWith(error, { tags: { jobId: "job_1", @@ -485,8 +443,7 @@ describe("accountPermissionSyncWorkload", () => { }); }); - test("completion tolerates a cascaded job and account row", async () => { - permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); + test("completion tolerates a deleted account", async () => { accountUpdateMany.mockResolvedValue({ count: 0 }); accountFindUnique.mockResolvedValue(null); @@ -495,8 +452,7 @@ describe("accountPermissionSyncWorkload", () => { ).resolves.toBeUndefined(); }); - test("terminal failure tolerates a cascaded job and account row", async () => { - permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); + test("terminal failure tolerates a deleted account", async () => { accountFindUnique.mockResolvedValue(null); await expect( diff --git a/packages/backend/src/ee/accountPermissionSyncWorkload.ts b/packages/backend/src/ee/accountPermissionSyncWorkload.ts index 2f042250c..1aecaa29f 100644 --- a/packages/backend/src/ee/accountPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/accountPermissionSyncWorkload.ts @@ -2,7 +2,6 @@ import * as Sentry from "@sentry/node"; import { Account, AccountPermissionSyncIssue, - AccountPermissionSyncJobStatus, PermissionSyncSource, PrismaClient, } from "@sourcebot/db"; @@ -247,30 +246,13 @@ export const createAccountPermissionSyncWorkload = ({ } }, onStarted: async ({ data: { accountId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.accountPermissionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - accountId, - status: AccountPermissionSyncJobStatus.IN_PROGRESS, - }, - }); - await tx.account.update({ - where: { - id: accountId, - }, - data: { - latestPermissionSyncJobId: jobId, - }, - }); + await db.account.update({ + where: { + id: accountId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, }); }, onCompleted: async ({ @@ -278,16 +260,6 @@ export const createAccountPermissionSyncWorkload = ({ jobId, }) => { const account = await db.$transaction(async (tx) => { - await tx.accountPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); await tx.account.updateMany({ where: { id: accountId, @@ -326,16 +298,6 @@ export const createAccountPermissionSyncWorkload = ({ }, }); - await db.accountPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: AccountPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); const account = await db.account.findUnique({ where: { id: accountId, diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts index a66bdf240..ce1394bd1 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.test.ts @@ -87,17 +87,11 @@ const repoUpdate = vi.fn().mockResolvedValue(repo); const repoUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const accountFindMany = vi.fn().mockResolvedValue([]); const permissionCreateMany = vi.fn().mockResolvedValue({ count: 0 }); -const permissionSyncJobUpsert = vi.fn(); -const permissionSyncJobUpdateMany = vi.fn().mockResolvedValue({ count: 1 }); const transactionClient = { repo: { update: repoUpdate, updateMany: repoUpdateMany, }, - repoPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, }; const transaction = vi.fn( ( @@ -122,10 +116,6 @@ const db = { accountToRepoPermission: { createMany: permissionCreateMany, }, - repoPermissionSyncJob: { - upsert: permissionSyncJobUpsert, - updateMany: permissionSyncJobUpdateMany, - }, $transaction: transaction, } as unknown as PrismaClient; @@ -164,7 +154,6 @@ beforeEach(() => { mocks.createOctokitFromToken.mockReset().mockResolvedValue({ octokit: {} }); mocks.getRepoCollaborators.mockReset().mockResolvedValue([]); repoFindUniqueOrThrow.mockResolvedValue(repo); - permissionSyncJobUpdateMany.mockResolvedValue({ count: 1 }); accountFindMany.mockResolvedValue([]); repoUpdate.mockResolvedValue(repo); repoUpdateMany.mockResolvedValue({ count: 1 }); @@ -172,7 +161,7 @@ beforeEach(() => { }); describe("repoPermissionSyncWorkload", () => { - test("uses the configured concurrency and database-backed lifecycle hooks", () => { + test("uses the configured concurrency and lifecycle hooks", () => { const workload = createWorkload(); expect(workload.queueSpec.name).toBe("repo-permission-sync"); @@ -350,42 +339,21 @@ describe("repoPermissionSyncWorkload", () => { }); }); - test("marks a job as in progress when started", async () => { + test("records the latest job when started", async () => { await createWorkload().onStarted?.(lifecycleContext); - expect(permissionSyncJobUpsert).toHaveBeenCalledWith({ - where: { id: "job_1" }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - }, - create: { - id: "job_1", - repoId: 42, - status: "IN_PROGRESS", - }, - }); expect(repoUpdate).toHaveBeenCalledWith({ where: { id: 42 }, data: { latestPermissionSyncJobId: "job_1" }, }); - expect(transaction).toHaveBeenCalledOnce(); + expect(transaction).not.toHaveBeenCalled(); }); - test("marks a job completed and updates the repo sync timestamp when it is still latest", async () => { + test("updates the repo sync timestamp when the completed job is still latest", async () => { await createWorkload().onCompleted?.(lifecycleContext, { repoName: "sourcebot-dev/sourcebot", }); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ - where: { id: "job_1" }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); expect(repoUpdateMany).toHaveBeenCalledWith({ where: { id: 42, @@ -395,11 +363,10 @@ describe("repoPermissionSyncWorkload", () => { permissionSyncedAt: expect.any(Date), }, }); - expect(transaction).toHaveBeenCalledOnce(); + expect(transaction).not.toHaveBeenCalled(); }); test("does not fail completion after the repo has been deleted", async () => { - permissionSyncJobUpdateMany.mockResolvedValue({ count: 0 }); repoUpdateMany.mockResolvedValue({ count: 0 }); await expect( @@ -409,19 +376,11 @@ describe("repoPermissionSyncWorkload", () => { ).resolves.toBeUndefined(); }); - test("marks a job failed after terminal failure", async () => { + test("reports terminal failures", async () => { const error = new Error("Upstream unavailable"); await createWorkload().onTerminalFailure?.(lifecycleContext, error); - expect(permissionSyncJobUpdateMany).toHaveBeenCalledWith({ - where: { id: "job_1" }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Upstream unavailable", - }, - }); expect(mocks.captureException).toHaveBeenCalledWith(error, { tags: { jobId: "job_1", diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts index 199f111ee..0e7d9dd4e 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -1,9 +1,5 @@ import * as Sentry from "@sentry/node"; -import { - PermissionSyncSource, - PrismaClient, - RepoPermissionSyncJobStatus, -} from "@sourcebot/db"; +import { PermissionSyncSource, PrismaClient } from "@sourcebot/db"; import { createLogger, REPO_PERMISSION_SYNC_QUEUE, @@ -133,56 +129,27 @@ export const createRepoPermissionSyncWorkload = ({ }; }, onStarted: async ({ data: { repoId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.repoPermissionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - }, - create: { - id: jobId, - repoId, - status: RepoPermissionSyncJobStatus.IN_PROGRESS, - }, - }); - await tx.repo.update({ - where: { - id: repoId, - }, - data: { - latestPermissionSyncJobId: jobId, - }, - }); + await db.repo.update({ + where: { + id: repoId, + }, + data: { + latestPermissionSyncJobId: jobId, + }, }); }, onCompleted: async ( { data: { repoId }, jobId }, { repoName }, ) => { - await db.$transaction(async (tx) => { - await tx.repoPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); - await tx.repo.updateMany({ - where: { - id: repoId, - latestPermissionSyncJobId: jobId, - }, - data: { - permissionSyncedAt: new Date(), - }, - }); + await db.repo.updateMany({ + where: { + id: repoId, + latestPermissionSyncJobId: jobId, + }, + data: { + permissionSyncedAt: new Date(), + }, }); logger.debug(`Permissions synced for repo ${repoName}`); @@ -195,17 +162,6 @@ export const createRepoPermissionSyncWorkload = ({ }, }); - await db.repoPermissionSyncJob.updateMany({ - where: { - id: jobId, - }, - data: { - status: RepoPermissionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, - }, - }); - logger.error( `Repo permission sync job failed for repo ${repoId}: ${error.message}`, ); diff --git a/packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql b/packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql new file mode 100644 index 000000000..3424e7740 --- /dev/null +++ b/packages/db/prisma/migrations/20260818133549_drop_repo_permission_sync_job/migration.sql @@ -0,0 +1,5 @@ +-- DropTable +DROP TABLE "RepoPermissionSyncJob"; + +-- DropEnum +DROP TYPE "RepoPermissionSyncJobStatus"; diff --git a/packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql b/packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql new file mode 100644 index 000000000..debcecf60 --- /dev/null +++ b/packages/db/prisma/migrations/20260818134450_drop_account_permission_sync_job/migration.sql @@ -0,0 +1,5 @@ +-- DropTable +DROP TABLE "AccountPermissionSyncJob"; + +-- DropEnum +DROP TYPE "AccountPermissionSyncJobStatus"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 9b689603f..9fb098353 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -72,7 +72,6 @@ model Repo { permittedAccounts AccountToRepoPermission[] scopedAccessTokens ScopedAccessTokenToRepo[] - permissionSyncJobs RepoPermissionSyncJob[] permissionSyncedAt DateTime? /// When the permissions were last synced successfully. latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest repo state. @@ -96,26 +95,6 @@ model Repo { @@index([indexedAt]) } -enum RepoPermissionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model RepoPermissionSyncJob { - id String @id @default(cuid()) - status RepoPermissionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - errorMessage String? - - repo Repo @relation(fields: [repoId], references: [id], onDelete: Cascade) - repoId Int -} - model SearchContext { id Int @id @default(autoincrement()) @@ -543,26 +522,6 @@ model User { } -enum AccountPermissionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model AccountPermissionSyncJob { - id String @id @default(cuid()) - status AccountPermissionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - errorMessage String? - - account Account @relation(fields: [accountId], references: [id], onDelete: Cascade) - accountId String -} - enum PermissionSyncSource { ACCOUNT_DRIVEN REPO_DRIVEN @@ -615,7 +574,6 @@ model Account { /// List of repos that this account has access to. accessibleRepos AccountToRepoPermission[] - permissionSyncJobs AccountPermissionSyncJob[] permissionSyncedAt DateTime? latestPermissionSyncJobId String? /// The permission sync job allowed to publish the latest account state. diff --git a/packages/db/tools/scripts/inject-repo-data.ts b/packages/db/tools/scripts/inject-repo-data.ts index 75e7d1a75..a5109ebe1 100644 --- a/packages/db/tools/scripts/inject-repo-data.ts +++ b/packages/db/tools/scripts/inject-repo-data.ts @@ -2,7 +2,6 @@ import { Script } from "../scriptRunner"; import { PrismaClient } from "../../dist"; const NUM_REPOS = 1000; -const NUM_PERMISSION_JOBS_PER_REPO = 10000; export const injectRepoData: Script = { run: async (prisma: PrismaClient) => { @@ -35,9 +34,8 @@ export const injectRepoData: Script = { console.log(`Creating ${NUM_REPOS} repos...`); - const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const; for (let i = 0; i < NUM_REPOS; i++) { - const repo = await prisma.repo.create({ + await prisma.repo.create({ data: { name: `test-repo-${i}`, isFork: false, @@ -57,19 +55,8 @@ export const injectRepoData: Script = { } }); - for (let j = 0; j < NUM_PERMISSION_JOBS_PER_REPO; j++) { - const status = statuses[Math.floor(Math.random() * statuses.length)]; - await prisma.repoPermissionSyncJob.create({ - data: { - repoId: repo.id, - status, - completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null, - errorMessage: status === 'FAILED' ? 'Mock error message' : null - } - }); - } } - console.log(`Created ${NUM_REPOS} repos with associated permission sync jobs.`); + console.log(`Created ${NUM_REPOS} repos.`); } }; diff --git a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts index 41715adef..1ed721985 100644 --- a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts +++ b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.test.ts @@ -2,13 +2,19 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ authContext: undefined as unknown, + getJob: vi.fn(), })); vi.mock('@/middleware/withAuth', () => ({ withAuth: vi.fn((callback: (context: unknown) => unknown) => callback(mocks.authContext)), })); +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ getJob: mocks.getJob }), +})); + vi.mock('@sourcebot/shared', () => ({ + ACCOUNT_PERMISSION_SYNC_QUEUE: { name: 'account-permission-sync' }, createLogger: () => ({ error: vi.fn() }), })); @@ -16,25 +22,70 @@ const { getAccountSyncStatus } = await import('./api'); beforeEach(() => { vi.clearAllMocks(); + mocks.getJob.mockReset(); }); describe('getAccountSyncStatus', () => { test.each(['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const)( 'returns the underlying %s job status', async (status) => { - const findFirst = vi.fn().mockResolvedValue({ status }); + const findFirst = vi.fn().mockResolvedValue({ id: 'account_1' }); + mocks.getJob.mockResolvedValue({ + id: 'job_1', + data: { accountId: 'account_1' }, + status, + errorMessage: null, + }); mocks.authContext = { user: { id: 'user_1' }, - prisma: { accountPermissionSyncJob: { findFirst } }, + prisma: { account: { findFirst } }, }; await expect(getAccountSyncStatus('job_1')).resolves.toEqual({ status }); + expect(mocks.getJob).toHaveBeenCalledWith( + { name: 'account-permission-sync' }, + 'job_1', + ); expect(findFirst).toHaveBeenCalledWith({ where: { - id: 'job_1', - account: { userId: 'user_1' }, + id: 'account_1', + userId: 'user_1', }, + select: { id: true }, }); }, ); + + test('does not expose a job belonging to another user', async () => { + mocks.getJob.mockResolvedValue({ + id: 'job_1', + data: { accountId: 'account_1' }, + status: 'IN_PROGRESS', + errorMessage: null, + }); + mocks.authContext = { + user: { id: 'user_1' }, + prisma: { + account: { findFirst: vi.fn().mockResolvedValue(null) }, + }, + }; + + await expect(getAccountSyncStatus('job_1')).resolves.toMatchObject({ + statusCode: 404, + }); + }); + + test('returns not found when Redis no longer has the job', async () => { + const findFirst = vi.fn(); + mocks.getJob.mockResolvedValue(null); + mocks.authContext = { + user: { id: 'user_1' }, + prisma: { account: { findFirst } }, + }; + + await expect(getAccountSyncStatus('missing')).resolves.toMatchObject({ + statusCode: 404, + }); + expect(findFirst).not.toHaveBeenCalled(); + }); }); diff --git a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts index e92c549d1..e3c034e16 100644 --- a/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts +++ b/packages/web/src/app/api/(server)/ee/accountPermissionSyncJobStatus/api.ts @@ -1,24 +1,38 @@ 'use server'; import { ServiceError, notFound } from "@/lib/serviceError"; +import { getBullMQClient } from "@/lib/bullmqClient"; import { withAuth } from "@/middleware/withAuth"; -import { AccountPermissionSyncJobStatus } from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + type WorkloadJobStatus, +} from "@sourcebot/shared"; import { sew } from "@/middleware/sew"; export interface AccountSyncStatusResponse { - status: AccountPermissionSyncJobStatus; + status: WorkloadJobStatus; } export const getAccountSyncStatus = async (jobId: string): Promise => sew(() => withAuth(async ({ prisma, user }) => { - const job = await prisma.accountPermissionSyncJob.findFirst({ + const job = await getBullMQClient().getJob( + ACCOUNT_PERMISSION_SYNC_QUEUE, + jobId, + ); + if (!job) { + return notFound(); + } + + const account = await prisma.account.findFirst({ where: { - id: jobId, - account: { userId: user.id }, + id: job.data.accountId, + userId: user.id, }, + select: { id: true }, }); - - if (!job) return notFound(); + if (!account) { + return notFound(); + } return { status: job.status } satisfies AccountSyncStatusResponse; })); diff --git a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts index 4cf5dbe8c..c28765c78 100644 --- a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts +++ b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ authContext: undefined as unknown, getEntitlements: vi.fn(), + getJobs: vi.fn(), })); vi.mock('@/middleware/withAuth', () => ({ @@ -13,8 +14,13 @@ vi.mock('@/lib/entitlements', () => ({ getEntitlements: mocks.getEntitlements, })); +vi.mock('@/lib/bullmqClient', () => ({ + getBullMQClient: () => ({ getJobs: mocks.getJobs }), +})); + vi.mock('@sourcebot/shared', () => ({ createLogger: () => ({ error: vi.fn() }), + ACCOUNT_PERMISSION_SYNC_QUEUE: { name: 'account-permission-sync' }, env: { PERMISSION_SYNC_ENABLED: 'true' }, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS: [ 'github', @@ -29,6 +35,7 @@ const { getPermissionSyncStatus } = await import('./api'); beforeEach(() => { vi.clearAllMocks(); mocks.getEntitlements.mockResolvedValue(['permission-syncing']); + mocks.getJobs.mockResolvedValue(new Map()); }); describe('getPermissionSyncStatus', () => { @@ -41,7 +48,7 @@ describe('getPermissionSyncStatus', () => { permissionSyncedAt: null, permissionSyncIssue: null, permissionSyncIssueAt: null, - permissionSyncJobs: [{ status: 'PENDING' }], + latestPermissionSyncJobId: 'job_pending', }, { id: 'account_action_required', @@ -50,9 +57,23 @@ describe('getPermissionSyncStatus', () => { permissionSyncedAt: new Date('2026-07-01T00:00:00Z'), permissionSyncIssue: 'REAUTHENTICATION_REQUIRED', permissionSyncIssueAt: new Date('2026-07-22T12:00:00Z'), - permissionSyncJobs: [{ status: 'PENDING' }], + latestPermissionSyncJobId: 'job_recovery', }, ]); + mocks.getJobs.mockResolvedValue(new Map([ + ['job_pending', { + id: 'job_pending', + data: { accountId: 'account_pending' }, + status: 'PENDING', + errorMessage: null, + }], + ['job_recovery', { + id: 'job_recovery', + data: { accountId: 'account_action_required' }, + status: 'IN_PROGRESS', + errorMessage: null, + }], + ])); mocks.authContext = { user: { id: 'user_1' }, prisma: { account: { findMany } }, @@ -72,6 +93,10 @@ describe('getPermissionSyncStatus', () => { expect(findMany).toHaveBeenCalledWith(expect.objectContaining({ where: expect.objectContaining({ userId: 'user_1' }), })); + expect(mocks.getJobs).toHaveBeenCalledWith( + { name: 'account-permission-sync' }, + ['job_pending', 'job_recovery'], + ); }); test('returns an issue even when the account has no issue timestamp', async () => { @@ -86,14 +111,44 @@ describe('getPermissionSyncStatus', () => { permissionSyncedAt: new Date('2026-07-01T00:00:00Z'), permissionSyncIssue: 'INSUFFICIENT_SCOPE', permissionSyncIssueAt: null, - permissionSyncJobs: [{ status: 'FAILED' }], + latestPermissionSyncJobId: 'job_failed', }]), }, }, }; + mocks.getJobs.mockResolvedValue(new Map([['job_failed', { + id: 'job_failed', + data: { accountId: 'account_1' }, + status: 'FAILED', + errorMessage: 'Insufficient scope', + }]])); await expect(getPermissionSyncStatus()).resolves.toMatchObject({ issues: [{ reason: 'INSUFFICIENT_SCOPE', occurredAt: null, isSyncing: false }], }); }); + + test('treats a missing first-sync job as pending', async () => { + mocks.authContext = { + user: { id: 'user_1' }, + prisma: { + account: { + findMany: vi.fn().mockResolvedValue([{ + id: 'account_1', + providerId: 'github', + providerType: 'github', + permissionSyncedAt: null, + permissionSyncIssue: null, + permissionSyncIssueAt: null, + latestPermissionSyncJobId: null, + }]), + }, + }, + }; + + await expect(getPermissionSyncStatus()).resolves.toMatchObject({ + hasPendingFirstSync: true, + }); + expect(mocks.getJobs).not.toHaveBeenCalled(); + }); }); diff --git a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts index 804016903..2c1e0708d 100644 --- a/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts +++ b/packages/web/src/app/api/(server)/ee/permissionSyncStatus/api.ts @@ -1,10 +1,17 @@ 'use server'; import { ServiceError } from "@/lib/serviceError"; +import { getBullMQClient } from "@/lib/bullmqClient"; import { withAuth } from "@/middleware/withAuth"; import { getEntitlements } from "@/lib/entitlements"; -import { env, PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS } from "@sourcebot/shared"; -import { AccountPermissionSyncJobStatus, type AccountPermissionSyncIssue } from "@sourcebot/db"; +import { + ACCOUNT_PERMISSION_SYNC_QUEUE, + env, + PERMISSION_SYNC_SUPPORTED_IDENTITY_PROVIDERS, + type WorkloadJob, + type WorkloadJobStatus, +} from "@sourcebot/shared"; +import type { AccountPermissionSyncIssue } from "@sourcebot/db"; import { StatusCodes } from "http-status-codes"; import { ErrorCode } from "@/lib/errorCodes"; import { sew } from "@/middleware/sew"; @@ -21,6 +28,9 @@ export interface PermissionSyncStatusResponse { }>; } +const isActiveStatus = (status: WorkloadJobStatus | undefined) => + status === "PENDING" || status === "IN_PROGRESS"; + /** * Returns initial-sync progress and action-required permission sync issues * for the authenticated user's linked accounts. @@ -49,35 +59,67 @@ export const getPermissionSyncStatus = async (): Promise + account.latestPermissionSyncJobId + ? [account.latestPermissionSyncJobId] + : [], + ); + const latestJobs = latestJobIds.length > 0 + ? await getBullMQClient().getJobs( + ACCOUNT_PERMISSION_SYNC_QUEUE, + latestJobIds, + ) + : new Map | null>(); + const latestJobsByAccountId = new Map< + string, + WorkloadJob<"account-permission-sync"> | null + >( + accounts.map((account): [ + string, + WorkloadJob<"account-permission-sync"> | null, + ] => { + const job = account.latestPermissionSyncJobId + ? latestJobs.get(account.latestPermissionSyncJobId) ?? null + : null; + return [ + account.id, + job?.data.accountId === account.id ? job : null, + ]; + }), + ); const hasPendingFirstSync = env.PERMISSION_SYNC_ENABLED === 'true' && - accounts.some(account => - account.permissionSyncedAt === null && - // @note: to handle the case where the permission sync job - // has not yet been scheduled for a new account, we consider - // accounts with no permission sync jobs as having a pending first sync. - (account.permissionSyncJobs.length === 0 || (account.permissionSyncJobs.length > 0 && activeStatuses.includes(account.permissionSyncJobs[0].status))) - ); + accounts.some((account) => { + const latestJob = latestJobsByAccountId.get(account.id); + return ( + account.permissionSyncedAt === null + // @note: to handle the case where the permission sync job + // has not yet been scheduled for a new account, we consider + // accounts with no available job as having a pending first sync. + && (!latestJob || isActiveStatus(latestJob.status)) + ); + }); + + const issues = accounts.flatMap((account) => { + if (account.permissionSyncIssue === null) { + return []; + } - const issues = accounts.flatMap(account => account.permissionSyncIssue === null ? [] : [{ - accountId: account.id, - providerId: account.providerId, - providerType: account.providerType, - reason: account.permissionSyncIssue, - occurredAt: account.permissionSyncIssueAt?.toISOString() ?? null, - isSyncing: account.permissionSyncJobs.some(job => activeStatuses.includes(job.status)), - }]); + return [{ + accountId: account.id, + providerId: account.providerId, + providerType: account.providerType, + reason: account.permissionSyncIssue, + occurredAt: account.permissionSyncIssueAt?.toISOString() ?? null, + isSyncing: isActiveStatus( + latestJobsByAccountId.get(account.id)?.status, + ), + }]; + }); return { hasPendingFirstSync, issues } satisfies PermissionSyncStatusResponse; }) From 82d440885d9767643152e131a8278839cd40fe74 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 14:33:01 -0700 Subject: [PATCH 11/34] rename connection workload --- packages/backend/src/configManager.ts | 2 +- ...ctionWorkload.test.ts => connectionSyncWorkload.test.ts} | 0 .../{connectionWorkload.ts => connectionSyncWorkload.ts} | 2 +- packages/backend/src/index.ts | 6 +++--- 4 files changed, 5 insertions(+), 5 deletions(-) rename packages/backend/src/{connectionWorkload.test.ts => connectionSyncWorkload.test.ts} (100%) rename packages/backend/src/{connectionWorkload.ts => connectionSyncWorkload.ts} (99%) diff --git a/packages/backend/src/configManager.ts b/packages/backend/src/configManager.ts index bf6d984d8..a3326ca8b 100644 --- a/packages/backend/src/configManager.ts +++ b/packages/backend/src/configManager.ts @@ -12,7 +12,7 @@ import { reconcileRepoIndexWork, reconcileRepoPermissionSyncWork, replaceConnectionRepositories, -} from "./connectionWorkload.js"; +} from "./connectionSyncWorkload.js"; import { SINGLE_TENANT_ORG_ID } from "./constants.js"; import { syncSearchContexts } from "./ee/syncSearchContexts.js"; import isEqual from 'fast-deep-equal'; diff --git a/packages/backend/src/connectionWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts similarity index 100% rename from packages/backend/src/connectionWorkload.test.ts rename to packages/backend/src/connectionSyncWorkload.test.ts diff --git a/packages/backend/src/connectionWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts similarity index 99% rename from packages/backend/src/connectionWorkload.ts rename to packages/backend/src/connectionSyncWorkload.ts index 2ed1fe944..5e257b7ce 100644 --- a/packages/backend/src/connectionWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -37,7 +37,7 @@ interface ConnectionSyncResult { reposToIndex: { id: number; name: string }[]; } -export const createConnectionWorkload = ({ +export const createConnectionSyncWorkload = ({ db, jobManager, settings, diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 1974421e6..efaa9447c 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -12,7 +12,7 @@ import { shutdownPosthog } from "./posthog.js"; import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { redis } from "./redis.js"; -import { createConnectionWorkload } from "./connectionWorkload.js"; +import { createConnectionSyncWorkload } from "./connectionSyncWorkload.js"; import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; @@ -50,7 +50,7 @@ logger.info('Worker started.'); const jobManager = new BullMQJobManager(redis); -const connectionWorkload = createConnectionWorkload({ +const connectionSyncWorkload = createConnectionSyncWorkload({ db: prisma, jobManager, settings, @@ -77,7 +77,7 @@ const auditLogPruneWorkload = createAuditLogPruneWorkload({ retentionDays: env.SOURCEBOT_EE_AUDIT_RETENTION_DAYS, }); -jobManager.register(connectionWorkload); +jobManager.register(connectionSyncWorkload); jobManager.register(repoIndexWorkload); jobManager.register(accountPermissionSyncWorkload); jobManager.register(repoPermissionSyncWorkload); From f3e63afaa120c3f568302e7a1baee280e26ad49c Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 14:33:39 -0700 Subject: [PATCH 12/34] remove connection sync notification dot --- packages/web/src/actions.ts | 38 +------------------ .../components/defaultSidebar/index.tsx | 7 +--- .../web/src/app/(app)/settings/layout.tsx | 6 --- 3 files changed, 3 insertions(+), 48 deletions(-) diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts index 2c9e2128f..0483e75f0 100644 --- a/packages/web/src/actions.ts +++ b/packages/web/src/actions.ts @@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit"; import { ErrorCode } from "@/lib/errorCodes"; import { notFound, ServiceError } from "@/lib/serviceError"; import { sew } from "@/middleware/sew"; -import { ConnectionSyncJobStatus, OrgRole, Prisma } from "@sourcebot/db"; +import { OrgRole, Prisma } from "@sourcebot/db"; import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type"; import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; @@ -220,42 +220,6 @@ export const getRepos = async ({ } satisfies RepositoryQuery)) })); -export const getConnectionStats = async () => sew(() => - withAuth(async ({ org, prisma }) => { - const [ - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - ] = await Promise.all([ - prisma.connection.count({ - where: { - orgId: org.id, - } - }), - prisma.connection.count({ - where: { - orgId: org.id, - syncedAt: null, - syncJobs: { - some: { - status: { - in: [ - ConnectionSyncJobStatus.PENDING, - ConnectionSyncJobStatus.IN_PROGRESS, - ] - } - } - } - } - }) - ]); - - return { - numberOfConnections, - numberOfConnectionsWithFirstTimeSyncJobsInProgress, - }; - }) -); - export const getRepoInfoByName = async (repoName: string) => sew(() => withOptionalAuth(async ({ org, prisma }) => { // @note: repo names are represented by their remote url diff --git a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx index a0ccd6269..85880f908 100644 --- a/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx +++ b/packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx @@ -2,7 +2,6 @@ import { cookies } from "next/headers"; import { auth } from "@/auth"; import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants"; import { HomeView } from "@/hooks/useHomeView"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { isServiceError } from "@/lib/utils"; import { ServiceErrorException } from "@/lib/serviceError"; @@ -46,11 +45,9 @@ export async function DefaultSidebar() { if (!isOwner) { return false; } - const connectionStats = await getConnectionStats(); const joinRequests = await getOrgAccountRequests(); - const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0; const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0; - return hasConnectionNotification || hasJoinRequestNotification; + return hasJoinRequestNotification; })(); return ( @@ -132,4 +129,4 @@ const getUserChatHistory = async () => sew(() => visibility: chat.visibility, })) }) -); \ No newline at end of file +); diff --git a/packages/web/src/app/(app)/settings/layout.tsx b/packages/web/src/app/(app)/settings/layout.tsx index 655ca28f5..40f53ef1b 100644 --- a/packages/web/src/app/(app)/settings/layout.tsx +++ b/packages/web/src/app/(app)/settings/layout.tsx @@ -3,7 +3,6 @@ import { Metadata } from "next" import { redirect } from "next/navigation"; import { auth } from "@/auth"; import { isServiceError } from "@/lib/utils"; -import { getConnectionStats } from "@/actions"; import { getOrgAccountRequests } from "@/features/membership/actions"; import { ServiceErrorException } from "@/lib/serviceError"; import { OrgRole } from "@prisma/client"; @@ -48,10 +47,6 @@ export const getSidebarNavGroups = async () => numJoinRequests = requests.length; } - const connectionStats = await getConnectionStats(); - if (isServiceError(connectionStats)) { - throw new ServiceErrorException(connectionStats); - } const hasAskEntitlement = await hasEntitlement("ask"); const groups: NavGroup[] = [ @@ -120,7 +115,6 @@ export const getSidebarNavGroups = async () => title: "Connections", href: `/settings/connections`, hrefRegex: `/settings/connections(/[^/]+)?$`, - isNotificationDotVisible: connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0, icon: "plug" as const, }, { From 61cfd7bdf2524e692d38494cca952654434f4460 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 15:48:23 -0700 Subject: [PATCH 13/34] workload job return type plumbing --- .../backend/src/attachmentPruneWorkload.ts | 8 +- .../src/connectionSyncWorkload.test.ts | 14 +--- .../backend/src/connectionSyncWorkload.ts | 12 +-- .../backend/src/ee/auditLogPruneWorkload.ts | 6 +- .../src/ee/repoPermissionSyncWorkload.ts | 9 +-- packages/backend/src/jobManager.test.ts | 19 +++-- packages/backend/src/jobManager.ts | 12 ++- packages/backend/src/types.ts | 7 +- packages/shared/src/bullmqClient.test.ts | 29 +++++++ packages/shared/src/bullmqClient.ts | 24 +++++- packages/shared/src/connectionSync.test.ts | 61 ++++++++++++++ packages/shared/src/connectionSync.ts | 68 ++++++++++++++++ packages/shared/src/index.server.ts | 15 ++++ packages/shared/src/queue.ts | 81 +++++++++++++++---- .../repos/components/reposTable.test.tsx | 9 +++ .../app/(app)/repos/components/reposTable.tsx | 1 + 16 files changed, 299 insertions(+), 76 deletions(-) create mode 100644 packages/shared/src/connectionSync.test.ts create mode 100644 packages/shared/src/connectionSync.ts diff --git a/packages/backend/src/attachmentPruneWorkload.ts b/packages/backend/src/attachmentPruneWorkload.ts index f9ddf862e..a21a60dd2 100644 --- a/packages/backend/src/attachmentPruneWorkload.ts +++ b/packages/backend/src/attachmentPruneWorkload.ts @@ -18,12 +18,6 @@ interface Props { storage?: StorageBackend; } -interface AttachmentPruneResult { - pendingClaimed: number; - committedClaimed: number; - reclaimed: number; -} - /** * Reclaims orphaned attachment blobs using the `DELETING` tombstone protocol: * an orphan is first atomically flipped to `DELETING`, then its bytes are @@ -47,7 +41,7 @@ export const createAttachmentPruneWorkload = ({ db, ttlHours, storage = getStorageBackend(), -}: Props): Workload<"attachment-prune", AttachmentPruneResult> => ({ +}: Props): Workload<"attachment-prune"> => ({ queueSpec: ATTACHMENT_PRUNE_QUEUE, concurrency: 1, ...(ttlHours > 0 diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index 3e5f8c6f6..b83896c3c 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -88,11 +88,11 @@ vi.mock("./ee/syncSearchContexts.js", () => ({ })); import { - createConnectionWorkload, + createConnectionSyncWorkload as createConnectionWorkload, replaceConnectionRepositories, reconcileRepoIndexWork, reconcileRepoPermissionSyncWork, -} from "./connectionWorkload.js"; +} from "./connectionSyncWorkload.js"; import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; const transactionClient = { @@ -233,8 +233,7 @@ describe("connectionWorkload", () => { test("marks the connection sync job as completed", async () => { await connectionWorkload.onCompleted?.(lifecycleContext, { - reposToCleanup: [], - reposToIndex: [], + outcome: "SUCCESS", }); expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ @@ -333,12 +332,7 @@ describe("connectionWorkload", () => { orgId: 7, contexts: undefined, }); - expect(result).toEqual({ - reposToCleanup: [], - reposToIndex: [ - { id: 4, name: "github.com/sourcebot/repo-4" }, - ], - }); + expect(result).toEqual({ outcome: "SUCCESS" }); expect(updateProgress).not.toHaveBeenCalled(); }); diff --git a/packages/backend/src/connectionSyncWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts index 5e257b7ce..78fe2a3a5 100644 --- a/packages/backend/src/connectionSyncWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -32,16 +32,11 @@ interface Props { settings: Settings; } -interface ConnectionSyncResult { - reposToCleanup: { id: number; name: string }[]; - reposToIndex: { id: number; name: string }[]; -} - export const createConnectionSyncWorkload = ({ db, jobManager, settings, -}: Props): Workload<"connection-sync", ConnectionSyncResult> => ({ +}: Props): Workload<"connection-sync"> => ({ queueSpec: CONNECTION_QUEUE, concurrency: settings.maxConnectionSyncJobConcurrency, executionLock: { @@ -159,10 +154,7 @@ export const createConnectionSyncWorkload = ({ connectionId, }); - return { - reposToCleanup: repoChanges.orphanedRepos, - reposToIndex: repoChanges.unindexedRepos, - }; + return { outcome: "SUCCESS" }; }, onStarted: async ({ data: { connectionId }, jobId }) => { await db.$transaction(async (tx) => { diff --git a/packages/backend/src/ee/auditLogPruneWorkload.ts b/packages/backend/src/ee/auditLogPruneWorkload.ts index bc8e0f18e..b70732ec5 100644 --- a/packages/backend/src/ee/auditLogPruneWorkload.ts +++ b/packages/backend/src/ee/auditLogPruneWorkload.ts @@ -16,15 +16,11 @@ interface Props { retentionDays: number; } -interface AuditLogPruneResult { - deleted: number; -} - export const createAuditLogPruneWorkload = ({ db, enabled, retentionDays, -}: Props): Workload<"audit-log-prune", AuditLogPruneResult> => ({ +}: Props): Workload<"audit-log-prune"> => ({ queueSpec: AUDIT_LOG_PRUNE_QUEUE, concurrency: 1, ...(enabled && retentionDays > 0 diff --git a/packages/backend/src/ee/repoPermissionSyncWorkload.ts b/packages/backend/src/ee/repoPermissionSyncWorkload.ts index 0e7d9dd4e..5e880433a 100644 --- a/packages/backend/src/ee/repoPermissionSyncWorkload.ts +++ b/packages/backend/src/ee/repoPermissionSyncWorkload.ts @@ -35,20 +35,13 @@ interface RepoPermissionSyncWorkloadDependencies { settings: Settings; } -interface RepoPermissionSyncResult { - repoName: string; -} - const REPO_PERMISSION_SYNC_LOCK_DURATION_MS = 60_000; const logger = createLogger("repo-permission-sync-workload"); export const createRepoPermissionSyncWorkload = ({ db, settings, -}: RepoPermissionSyncWorkloadDependencies): Workload< - "repo-permission-sync", - RepoPermissionSyncResult -> => ({ +}: RepoPermissionSyncWorkloadDependencies): Workload<"repo-permission-sync"> => ({ queueSpec: REPO_PERMISSION_SYNC_QUEUE, concurrency: settings.maxRepoPermissionSyncJobConcurrency, executionLock: { diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index ffa874c61..4c2c0cf8f 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -1,5 +1,7 @@ import { Redis } from "ioredis"; +import type { ConnectionSyncResult } from "@sourcebot/shared"; import { beforeEach, describe, expect, test, vi } from "vitest"; +import { z, type ZodType } from "zod"; import { ProcessContext, Workload } from "./types.js"; const mocks = vi.hoisted(() => { @@ -100,10 +102,11 @@ vi.mock("bullmq", () => ({ import { BullMQJobManager } from "./jobManager.js"; const createWorkload = ( - overrides: Partial> = {}, -): Workload<"connection-sync", { repoCount: number }> => ({ + overrides: Partial> = {}, +): Workload<"connection-sync"> => ({ queueSpec: { name: "connection-sync", + resultSchema: z.unknown() as ZodType, dedupKey: ({ connectionId }) => `connection:${connectionId}`, jobOptions: { attempts: 2, @@ -116,7 +119,7 @@ const createWorkload = ( }, }, concurrency: 2, - process: vi.fn(async () => ({ repoCount: 3 })), + process: vi.fn(async () => ({ outcome: "SUCCESS" as const })), ...overrides, }); @@ -256,7 +259,7 @@ describe("BullMQJobManager lifecycle", () => { }), process: vi.fn(async () => { calls.push("processed"); - return { repoCount: 3 }; + return { outcome: "SUCCESS" }; }), onCompleted: vi.fn(async () => { calls.push("completed"); @@ -282,7 +285,7 @@ describe("BullMQJobManager lifecycle", () => { jobId: "job-1", maxAttempts: 2, }), - { repoCount: 3 }, + { outcome: "SUCCESS" }, ); expect( vi.mocked(workload.onCompleted!).mock.calls[0][0], @@ -315,7 +318,7 @@ describe("BullMQJobManager lifecycle", () => { process: vi.fn(async ({ signal }) => { expect(signal).toBe(workloadSignal); calls.push("processed"); - return { repoCount: 3 }; + return { outcome: "SUCCESS" }; }), }); const manager = new BullMQJobManager({} as Redis); @@ -324,7 +327,7 @@ describe("BullMQJobManager lifecycle", () => { await expect( mocks.workers[0].processor({ ...job, attemptsMade: 0 }), - ).resolves.toEqual({ repoCount: 3 }); + ).resolves.toEqual({ outcome: "SUCCESS" }); expect(calls).toEqual([ "lock-acquired", @@ -358,7 +361,7 @@ describe("BullMQJobManager lifecycle", () => { const process = vi.fn( async (context: ProcessContext<"connection-sync">) => { expect(context).not.toHaveProperty("logger"); - return { repoCount: 3 }; + return { outcome: "SUCCESS" }; }, ); const manager = new BullMQJobManager({} as Redis); diff --git a/packages/backend/src/jobManager.ts b/packages/backend/src/jobManager.ts index 53beb1533..f2c33e542 100644 --- a/packages/backend/src/jobManager.ts +++ b/packages/backend/src/jobManager.ts @@ -6,6 +6,7 @@ import { DataOf, JobEnqueueOptions, QueueName, + ResultOf, Schedule, scheduleToMs, runWithJobLogContext, @@ -24,10 +25,7 @@ const STALLED_JOB_TERMINAL_ERROR = "job stalled more than allowable limit"; const logger = createLogger(LOG_TAG); export class BullMQJobManager implements JobManager { - private readonly workloads = new Map< - string, - Workload - >(); + private readonly workloads = new Map>(); private readonly workers = new Map(); private readonly bullmqClient: BullMQClient; private readonly abortController = new AbortController(); @@ -325,10 +323,10 @@ export class BullMQJobManager implements JobManager { ); } - private async onWorkloadJobCompleted( - workload: Workload, + private async onWorkloadJobCompleted( + workload: Workload, job: Job, - result: TResult, + result: ResultOf, ): Promise { const label = `${LOG_TAG}:${workload.queueSpec.name}:job:${job.id ?? "unknown"}`; const attempt = Math.max(job.attemptsMade, 1); diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index 52ba10096..895a858b3 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -11,6 +11,7 @@ import { JobEnqueueOptions, QueueName, QueueSpec, + ResultOf, Schedule, } from "@sourcebot/shared"; import type { Queue } from "bullmq"; @@ -69,7 +70,7 @@ export interface WorkloadExecutionLock { durationMs: number; } -export interface Workload { +export interface Workload { queueSpec: QueueSpec; concurrency: number; executionLock?: WorkloadExecutionLock; @@ -79,11 +80,11 @@ export interface Workload { options?: JobEnqueueOptions; }; rateLimit?: { max: number; per: string }; - process(ctx: ProcessContext): Promise; + process(ctx: ProcessContext): Promise>; onStarted?(ctx: JobLifecycleContext): Promise; onCompleted?( ctx: JobLifecycleContext, - result: TResult, + result: ResultOf, ): Promise; onTerminalFailure?( ctx: JobLifecycleContext, diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index 3f49990b6..d71b67bf1 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -51,6 +51,7 @@ describe("BullMQClient", () => { id: jobId, data: { connectionId: 1 }, failedReason: "", + returnvalue: null, getState: vi.fn(async () => "active"), }; } @@ -59,6 +60,7 @@ describe("BullMQClient", () => { id: jobId, data: { connectionId: 2 }, failedReason: "", + returnvalue: { outcome: "SUCCESS" }, getState: vi.fn(async () => "completed"), }; } @@ -74,6 +76,7 @@ describe("BullMQClient", () => { data: { connectionId: 1 }, status: "IN_PROGRESS", errorMessage: null, + result: null, }], ["missing", null], ["job-2", { @@ -81,15 +84,41 @@ describe("BullMQClient", () => { data: { connectionId: 2 }, status: "COMPLETED", errorMessage: null, + result: { outcome: "SUCCESS" }, }], ])); }); + test("returns null for an unrecognized legacy connection result", async () => { + mocks.getJob.mockResolvedValue({ + id: "job-1", + data: { connectionId: 1 }, + failedReason: "", + returnvalue: { + reposToCleanup: [], + reposToIndex: [], + }, + getState: vi.fn(async () => "completed"), + }); + const client = new BullMQClient({} as Redis); + + await expect( + client.getJob(CONNECTION_QUEUE, "job-1"), + ).resolves.toEqual({ + id: "job-1", + data: { connectionId: 1 }, + status: "COMPLETED", + errorMessage: null, + result: null, + }); + }); + test("deduplicates job ids when getting jobs", async () => { mocks.getJob.mockResolvedValue({ id: "job-1", data: { connectionId: 1 }, failedReason: "", + returnvalue: null, getState: vi.fn(async () => "waiting"), }); const client = new BullMQClient({} as Redis); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index 527e3c426..f3cfbcfe5 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -7,6 +7,7 @@ import type { JobEnqueueOptions, QueueName, QueueSpec, + ResultOf, } from "./queue.js"; import { scheduleToMs } from "./schedule.js"; import type { Schedule } from "./schedule.js"; @@ -24,14 +25,15 @@ export interface WorkloadJob { data: DataOf; status: WorkloadJobStatus; errorMessage: string | null; + result: ResultOf | null; } type WorkloadQueue = Queue< DataOf, - unknown, + ResultOf, string, DataOf, - unknown, + ResultOf, string >; @@ -85,11 +87,21 @@ export class BullMQClient { return null; } + const result = (() => { + if (status !== "COMPLETED" || !spec.resultSchema) { + return null; + } + + const parsed = spec.resultSchema.safeParse(job.returnvalue); + return parsed.success ? parsed.data : null; + })(); + return { id: job.id ?? jobId, data: job.data as DataOf, status, errorMessage: status === "FAILED" ? job.failedReason || null : null, + result, }; } @@ -133,7 +145,13 @@ export class BullMQClient { data: DataOf, options: JobEnqueueOptions = {}, ): Promise { - const dedupKey = spec.dedupKey?.(data); + // QueueSpec is distributive so queue names, data, and result schemas stay + // correlated when TName is a union. Re-establish the shared generic here + // before invoking the optional method. + const { dedupKey: getDedupKey }: { + dedupKey?(data: DataOf): string; + } = spec; + const dedupKey = getDedupKey?.(data); const queue = this.getQueue(spec); const requestedJobId = randomUUID(); diff --git a/packages/shared/src/connectionSync.test.ts b/packages/shared/src/connectionSync.test.ts new file mode 100644 index 000000000..42b9ffbf0 --- /dev/null +++ b/packages/shared/src/connectionSync.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "vitest"; +import { + connectionSyncResultSchema, + connectionSyncPartialSuccessReasonSchema, +} from "./connectionSync.js"; + +describe("connectionSyncResultSchema", () => { + test("accepts a successful result", () => { + expect( + connectionSyncResultSchema.parse({ outcome: "SUCCESS" }), + ).toEqual({ outcome: "SUCCESS" }); + }); + + test("accepts a partial success with structured reasons", () => { + const reason = { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "sourcebot-dev/missing-repo", + }, + message: + "Repository sourcebot-dev/missing-repo was not found or is inaccessible.", + }; + + expect( + connectionSyncResultSchema.parse({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }), + ).toEqual({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }); + }); + + test("requires at least one reason for a partial success", () => { + expect( + connectionSyncResultSchema.safeParse({ + outcome: "PARTIAL_SUCCESS", + reasons: [], + }).success, + ).toBe(false); + }); +}); + +describe("connectionSyncPartialSuccessReasonSchema", () => { + test("allows a reason without a subject", () => { + expect( + connectionSyncPartialSuccessReasonSchema.parse({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "The provider returned an invalid repository.", + }), + ).toEqual({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "The provider returned an invalid repository.", + }); + }); +}); diff --git a/packages/shared/src/connectionSync.ts b/packages/shared/src/connectionSync.ts new file mode 100644 index 000000000..7a64b768a --- /dev/null +++ b/packages/shared/src/connectionSync.ts @@ -0,0 +1,68 @@ +import { z } from "zod"; + +export const connectionSyncPartialSuccessReasonCodeSchema = z.enum([ + "NOT_FOUND_OR_INACCESSIBLE", + "INVALID_TARGET", + "UNSUPPORTED_CONFIGURATION", + "INVALID_REPOSITORY_SOURCE", + "ENUMERATION_FAILED", + "INVALID_PROVIDER_RESPONSE", +]); + +export type ConnectionSyncPartialSuccessReasonCode = z.infer< + typeof connectionSyncPartialSuccessReasonCodeSchema +>; + +export const connectionSyncPartialSuccessEffectSchema = z.enum([ + "TARGET_SKIPPED", + "CONFIGURATION_IGNORED", + "DISCOVERY_INCOMPLETE", +]); + +export type ConnectionSyncPartialSuccessEffect = z.infer< + typeof connectionSyncPartialSuccessEffectSchema +>; + +export const connectionSyncPartialSuccessSubjectSchema = z.object({ + kind: z.enum([ + "organization", + "group", + "user", + "workspace", + "project", + "repository", + "path", + "url", + "configuration", + ]), + value: z.string().min(1), +}); + +export type ConnectionSyncPartialSuccessSubject = z.infer< + typeof connectionSyncPartialSuccessSubjectSchema +>; + +export const connectionSyncPartialSuccessReasonSchema = z.object({ + code: connectionSyncPartialSuccessReasonCodeSchema, + effect: connectionSyncPartialSuccessEffectSchema, + subject: connectionSyncPartialSuccessSubjectSchema.optional(), + message: z.string().min(1), +}); + +export type ConnectionSyncPartialSuccessReason = z.infer< + typeof connectionSyncPartialSuccessReasonSchema +>; + +export const connectionSyncResultSchema = z.discriminatedUnion("outcome", [ + z.object({ + outcome: z.literal("SUCCESS"), + }), + z.object({ + outcome: z.literal("PARTIAL_SUCCESS"), + reasons: z.array(connectionSyncPartialSuccessReasonSchema).min(1), + }), +]); + +export type ConnectionSyncResult = z.infer< + typeof connectionSyncResultSchema +>; diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index ff92ac127..486f346ed 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -94,9 +94,24 @@ export { compareVersions, } from "./versionUtils.js"; export type { Version } from "./versionUtils.js"; +export { + connectionSyncResultSchema, + connectionSyncPartialSuccessEffectSchema, + connectionSyncPartialSuccessReasonCodeSchema, + connectionSyncPartialSuccessReasonSchema, + connectionSyncPartialSuccessSubjectSchema, +} from "./connectionSync.js"; +export type { + ConnectionSyncResult, + ConnectionSyncPartialSuccessEffect, + ConnectionSyncPartialSuccessReason, + ConnectionSyncPartialSuccessReasonCode, + ConnectionSyncPartialSuccessSubject, +} from "./connectionSync.js"; export type { QueueName, DataOf, + ResultOf, JobEnqueueOptions, QueueSpec, JobOptions, diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 993fc9b6c..6c63db6ca 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,12 +1,27 @@ import type { KeepJobs } from "bullmq"; +import { z, type ZodType } from "zod"; +import { + connectionSyncResultSchema, + type ConnectionSyncResult, +} from "./connectionSync.js"; import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; -export interface QueueSpec { +interface BaseQueueSpec { name: TName; dedupKey?(data: DataOf): string; jobOptions: JobOptions; } +type ResultSchemaSpec = TName extends QueueName + ? [ResultOf] extends [void] + ? { resultSchema?: never } + : { resultSchema: ZodType> } + : never; + +export type QueueSpec = TName extends QueueName + ? BaseQueueSpec & ResultSchemaSpec + : never; + export type JobOptions = { attempts: number; backoff: { @@ -48,40 +63,76 @@ export const DEFAULT_JOB_OPTIONS: JobOptions = { }; export type QueueName = keyof QueueRegistry; -export type DataOf = QueueRegistry[TName]; +export type DataOf = QueueRegistry[TName]["data"]; +export type ResultOf = QueueRegistry[TName]["result"]; + +const attachmentPruneResultSchema = z.object({ + pendingClaimed: z.number().int().nonnegative(), + committedClaimed: z.number().int().nonnegative(), + reclaimed: z.number().int().nonnegative(), +}); + +const auditLogPruneResultSchema = z.object({ + deleted: z.number().int().nonnegative(), +}); + +const repoPermissionSyncResultSchema = z.object({ + repoName: z.string().min(1), +}); interface QueueRegistry { - "attachment-prune": Record; - "audit-log-prune": Record; + "attachment-prune": { + data: Record; + result: z.infer; + }; + "audit-log-prune": { + data: Record; + result: z.infer; + }; "connection-sync": { - connectionId: number; + data: { + connectionId: number; + }; + result: ConnectionSyncResult; }; "repo-index": { - repoId: number; - type: "INDEX" | "CLEANUP"; + data: { + repoId: number; + type: "INDEX" | "CLEANUP"; + }; + result: void; }; "account-permission-sync": { - accountId: string; + data: { + accountId: string; + }; + result: void; }; "repo-permission-sync": { - repoId: number; + data: { + repoId: number; + }; + result: z.infer; }; } export const ATTACHMENT_PRUNE_QUEUE: QueueSpec<"attachment-prune"> = { name: "attachment-prune", + resultSchema: attachmentPruneResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: () => "global", }; export const AUDIT_LOG_PRUNE_QUEUE: QueueSpec<"audit-log-prune"> = { name: "audit-log-prune", + resultSchema: auditLogPruneResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: () => "global", }; export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { name: "connection-sync", + resultSchema: connectionSyncResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `connection:${data.connectionId}`, }; @@ -91,15 +142,15 @@ export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { jobOptions: DEFAULT_JOB_OPTIONS, }; -export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = - { - name: "account-permission-sync", - jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `account:${data.accountId}`, - }; +export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = { + name: "account-permission-sync", + jobOptions: DEFAULT_JOB_OPTIONS, + dedupKey: (data) => `account:${data.accountId}`, +}; export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { name: "repo-permission-sync", + resultSchema: repoPermissionSyncResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `repo:${data.repoId}`, }; diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index a2dd801af..22b5a8d40 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -40,6 +40,7 @@ const repos: Repo[] = [ data: { repoId: 1, type: "INDEX" }, status: "IN_PROGRESS", errorMessage: null, + result: null, }, imageUrl: null, webUrl: "https://github.com/acme/first", @@ -175,6 +176,7 @@ describe("ReposTable", () => { data: { repoId: 2, type: "INDEX" }, status: "COMPLETED", errorMessage: null, + result: null, }, }]); @@ -274,6 +276,7 @@ describe("ReposTable", () => { data: { repoId: 1, type: "INDEX" }, status: "COMPLETED", errorMessage: null, + result: null, }, }], }; @@ -317,6 +320,7 @@ describe("ReposTable", () => { data: { repoId: 2, type: "INDEX" }, status: "FAILED", errorMessage: "The remote repository could not be reached", + result: null, }, }]); @@ -339,6 +343,7 @@ describe("ReposTable", () => { ...repos[0].latestJob!, status: "FAILED", errorMessage: "Authentication failed while cloning", + result: null, }, }]); @@ -366,6 +371,7 @@ describe("ReposTable", () => { data: { repoId: 1, type: "INDEX" }, status: "FAILED", errorMessage: "Authentication failed while cloning", + result: null, }, }], } satisfies RepoIndexingStatusesResponse))); @@ -375,6 +381,7 @@ describe("ReposTable", () => { ...repos[0].latestJob!, status: "FAILED", errorMessage: "Authentication failed while cloning", + result: null, }, }]); @@ -416,6 +423,7 @@ describe("ReposTable", () => { data: { repoId: 1, type: "INDEX" }, status: "COMPLETED", errorMessage: null, + result: null, }, }], }; @@ -455,6 +463,7 @@ describe("ReposTable", () => { data: { repoId: 1, type: "INDEX" }, status: "FAILED", errorMessage: "Indexing failed", + result: null, }, }], }; diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index a61d9a357..af7ffcd0c 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -539,6 +539,7 @@ export const ReposTable = ({ data: { repoId, type: "INDEX" }, status: "PENDING", errorMessage: null, + result: null, }); return nextJobs; }); From 066690c9f8fa35a332700844a59a803b898d8799 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 16:24:32 -0700 Subject: [PATCH 14/34] add concept of repositoryDiscoveryIssueContext --- .../src/connectionSyncWorkload.test.ts | 62 +++++++--- .../backend/src/connectionSyncWorkload.ts | 29 ++--- packages/backend/src/github.test.ts | 113 ++++++++++++++++- packages/backend/src/github.ts | 114 +++++++++--------- packages/backend/src/githubAppAuth.test.ts | 35 ++++++ packages/backend/src/repoCompileUtils.test.ts | 76 +++++------- packages/backend/src/repoCompileUtils.ts | 91 +++----------- .../repositoryDiscoveryIssueContext.test.ts | 101 ++++++++++++++++ .../src/repositoryDiscoveryIssueContext.ts | 53 ++++++++ packages/shared/src/connectionSync.test.ts | 21 +--- packages/shared/src/connectionSync.ts | 56 +-------- packages/shared/src/index.server.ts | 22 ++-- .../shared/src/repositoryDiscovery.test.ts | 42 +++++++ packages/shared/src/repositoryDiscovery.ts | 55 +++++++++ 14 files changed, 577 insertions(+), 293 deletions(-) create mode 100644 packages/backend/src/repositoryDiscoveryIssueContext.test.ts create mode 100644 packages/backend/src/repositoryDiscoveryIssueContext.ts create mode 100644 packages/shared/src/repositoryDiscovery.test.ts create mode 100644 packages/shared/src/repositoryDiscovery.ts diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index b83896c3c..46793d0cc 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -71,6 +71,9 @@ vi.mock("@sourcebot/shared", () => ({ ], createLogger: vi.fn(() => mocks.logger), loadConfig: mocks.loadConfig, + repositoryDiscoveryIssueSchema: { + parse: (issue: unknown) => issue, + }, })); vi.mock("./repoCompileUtils.js", () => ({ @@ -93,6 +96,7 @@ import { reconcileRepoIndexWork, reconcileRepoPermissionSyncWork, } from "./connectionSyncWorkload.js"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; const transactionClient = { @@ -280,10 +284,7 @@ describe("connectionWorkload", () => { orgId: 7, config, }); - mocks.compileGithubConfig.mockResolvedValue({ - repoData: [discoveredRepo], - warnings: ["Repository was archived"], - }); + mocks.compileGithubConfig.mockResolvedValue([discoveredRepo]); mocks.repoUpsert.mockResolvedValue({ id: 4, name: "github.com/sourcebot/repo-4", @@ -304,10 +305,6 @@ describe("connectionWorkload", () => { 42, expect.any(AbortSignal), ); - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { id: "job-1" }, - data: { warningMessages: ["Repository was archived"] }, - }); expect(mocks.repoUpsert).toHaveBeenCalledOnce(); expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( "repo-index", @@ -336,22 +333,53 @@ describe("connectionWorkload", () => { expect(updateProgress).not.toHaveBeenCalled(); }); - test("does not mark the connection synced when repo work reconciliation fails", async () => { + test("returns partial success reasons reported during discovery", async () => { + const reason = { + code: "NOT_FOUND_OR_INACCESSIBLE" as const, + effect: "TARGET_SKIPPED" as const, + subject: { + kind: "repository" as const, + value: "sourcebot-dev/legacy", + }, + message: "Repository was not found or is inaccessible.", + }; mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, name: "github", orgId: 7, config: { type: "github" }, }); - mocks.compileGithubConfig.mockResolvedValue({ - repoData: [ - { - external_id: "repo-4", - external_codeHostUrl: "https://github.com", - }, - ], - warnings: [], + mocks.compileGithubConfig.mockImplementation(async () => { + reportRepositoryDiscoveryIssue(reason); + return []; + }); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), + }), + ).resolves.toEqual({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }); + }); + + test("does not mark the connection synced when repo work reconciliation fails", async () => { + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config: { type: "github" }, }); + mocks.compileGithubConfig.mockResolvedValue([ + { + external_id: "repo-4", + external_codeHostUrl: "https://github.com", + }, + ]); mocks.repoUpsert.mockResolvedValue({ id: 4, name: "github.com/sourcebot/repo-4", diff --git a/packages/backend/src/connectionSyncWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts index 78fe2a3a5..f4aaf6b45 100644 --- a/packages/backend/src/connectionSyncWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -21,6 +21,7 @@ import { compileGitlabConfig, } from "./repoCompileUtils.js"; import type { RepoData } from "./repoCompileUtils.js"; +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; import { JobManager, ProcessContext, Settings, Workload } from "./types.js"; const CONNECTION_SYNC_LOCK_DURATION_MS = 60_000; @@ -47,7 +48,6 @@ export const createConnectionSyncWorkload = ({ process: async ({ data: { connectionId }, signal, - jobId, trigger, }) => { signal.throwIfAborted(); @@ -64,21 +64,14 @@ export const createConnectionSyncWorkload = ({ orgId, }); - const { repoData, warnings } = await discoverConnectionRepositories({ - config: connection.config as unknown as ConnectionConfig, - connectionId, - signal, - }); - - signal.throwIfAborted(); - await db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - warningMessages: warnings, - }, - }); + const { value: repoData, issues } = + await collectRepositoryDiscoveryIssues(() => + discoverConnectionRepositories({ + config: connection.config as unknown as ConnectionConfig, + connectionId, + signal, + }) + ); logger.debug(`Discovered ${repoData.length} repositories`, { connectionId, @@ -154,7 +147,9 @@ export const createConnectionSyncWorkload = ({ connectionId, }); - return { outcome: "SUCCESS" }; + return issues.length === 0 + ? { outcome: "SUCCESS" } + : { outcome: "PARTIAL_SUCCESS", reasons: issues }; }, onStarted: async ({ data: { connectionId }, jobId }) => { await db.$transaction(async (tx) => { diff --git a/packages/backend/src/github.test.ts b/packages/backend/src/github.test.ts index 7c9082db7..ce6c64077 100644 --- a/packages/backend/src/github.test.ts +++ b/packages/backend/src/github.test.ts @@ -1,11 +1,122 @@ -import { expect, test, describe } from 'vitest'; +import type { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type"; +import { expect, test, describe, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { status: 404 }); + + return { + reposGet: vi.fn(async () => { + throw notFound; + }), + paginateIterator: vi.fn(() => ({ + async *[Symbol.asyncIterator]() { + throw notFound; + }, + })), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@octokit/rest", () => ({ + Octokit: class { + repos = { + get: mocks.reposGet, + listForOrg: vi.fn(), + }; + rest = { + search: { + repos: vi.fn(), + }, + users: { + getAuthenticated: vi.fn(), + }, + }; + paginate = { + iterator: mocks.paginateIterator, + }; + }, +})); + +vi.mock("./ee/githubAppManager.js", () => ({ + GithubAppManager: { + getInstance: () => ({ + ensureInitialized: vi.fn(), + appsConfigured: () => false, + }), + }, +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; import { OctokitRepository, shouldExcludeRepo, detectGitHubTokenType, supportsOAuthScopeIntrospection, + getGitHubReposFromConfig, } from './github'; +describe("GitHub repository discovery", () => { + test("reports inaccessible configured targets as partial successes", async () => { + const config = { + type: "github", + url: "https://github.example.com", + orgs: ["missing-org"], + repos: ["missing-owner/missing-repo"], + users: ["missing-user"], + } satisfies GithubConnectionConfig; + const result = await collectRepositoryDiscoveryIssues(() => + getGitHubReposFromConfig( + config, + new AbortController().signal, + ) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: "missing-org", + }, + message: "GitHub organization was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "missing-owner/missing-repo", + }, + message: "GitHub repository was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: "missing-user", + }, + message: "GitHub user was not found or is inaccessible.", + }, + ], + }); + }); +}); + describe('detectGitHubTokenType', () => { test('detects classic PAT (ghp_)', () => { expect(detectGitHubTokenType('ghp_abc123def456')).toBe('classic_pat'); diff --git a/packages/backend/src/github.ts b/packages/backend/src/github.ts index 3b5101b7b..998d6a6e1 100644 --- a/packages/backend/src/github.ts +++ b/packages/backend/src/github.ts @@ -8,7 +8,8 @@ import { env } from "@sourcebot/shared"; import { hasEntitlement } from "./entitlements.js"; import micromatch from "micromatch"; import pLimit from "p-limit"; -import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; +import { throwIfAnyFailed } from "./connectionUtils.js"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { GithubAppManager } from "./ee/githubAppManager.js"; import { fetchWithRetry, measure } from "./utils.js"; @@ -142,6 +143,16 @@ export const getOctokitWithGithubApp = async ( logger.warn( `No matching GitHub App installation found for ${context} on ${hostname}; falling back to legacy GitHub authentication.` ); + reportRepositoryDiscoveryIssue({ + code: "AUTHENTICATION_FALLBACK", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "configuration", + value: `GitHub App installation for ${owner} on ${hostname}`, + }, + message: + "No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.", + }); return octokit; } @@ -152,7 +163,7 @@ export const getOctokitWithGithubApp = async ( return octokitFromToken; } -export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, signal: AbortSignal): Promise<{ repos: OctokitRepository[], warnings: string[] }> => { +export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, signal: AbortSignal): Promise => { const hostname = config.url ? new URL(config.url).hostname : GITHUB_CLOUD_HOSTNAME; @@ -180,24 +191,20 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, s } let allRepos: OctokitRepository[] = []; - let allWarnings: string[] = []; if (config.orgs) { - const { repos, warnings } = await getReposForOrgs(config.orgs, octokit, signal, config.url); + const repos = await getReposForOrgs(config.orgs, octokit, signal, config.url); allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); } if (config.repos) { - const { repos, warnings } = await getRepos(config.repos, octokit, signal, config.url); + const repos = await getRepos(config.repos, octokit, signal, config.url); allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); } if (config.users) { - const { repos, warnings } = await getReposOwnedByUsers(config.users, octokit, signal, config.url); + const repos = await getReposOwnedByUsers(config.users, octokit, signal, config.url); allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); } let repos = allRepos @@ -215,10 +222,7 @@ export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, s logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; } export const getRepoCollaborators = async (owner: string, repo: string, octokit: Octokit) => { @@ -337,10 +341,7 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A }); logger.debug(`Found ${data.length} owned by user ${user} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for user ${user}.`, error); @@ -348,22 +349,25 @@ const getReposOwnedByUsers = async (users: string[], octokit: Octokit, signal: A if (isHttpError(error, 404)) { const warning = `User ${user} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: user, + }, + message: "GitHub user was not found or is inaccessible.", + }); + return []; } throw error; } }))); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return results.flatMap((result) => + result.status === "fulfilled" ? result.value : [] + ); } const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { @@ -400,10 +404,7 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi }); logger.debug(`Found ${data.length} in org ${org} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for org ${org}.`, error); @@ -411,22 +412,25 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi if (isHttpError(error, 404)) { const warning = `Organization ${org} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: org, + }, + message: "GitHub organization was not found or is inaccessible.", + }); + return []; } throw error; } }))); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return results.flatMap((result) => + result.status === "fulfilled" ? result.value : [] + ); } const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { @@ -449,10 +453,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna }); logger.debug(`Found info for repository ${repo} in ${durationMs}ms`); - return { - type: 'valid' as const, - data: [result.data] - }; + return [result.data]; } catch (error) { Sentry.captureException(error); @@ -461,22 +462,25 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna if (isHttpError(error, 404)) { const warning = `Repository ${repo} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "GitHub repository was not found or is inaccessible.", + }); + return []; } throw error; } }))); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return results.flatMap((result) => + result.status === "fulfilled" ? result.value : [] + ); } export const shouldExcludeRepo = ({ diff --git a/packages/backend/src/githubAppAuth.test.ts b/packages/backend/src/githubAppAuth.test.ts index b9e56072f..8a28aa4f5 100644 --- a/packages/backend/src/githubAppAuth.test.ts +++ b/packages/backend/src/githubAppAuth.test.ts @@ -23,6 +23,9 @@ vi.mock('@sourcebot/shared', () => ({ FALLBACK_GITHUB_CLOUD_TOKEN: undefined, }, getTokenFromConfig: vi.fn(), + repositoryDiscoveryIssueSchema: { + parse: (issue: unknown) => issue, + }, })); vi.mock('./entitlements.js', () => ({ @@ -40,6 +43,7 @@ vi.mock('./ee/githubAppManager.js', () => ({ })); import { getOctokitWithGithubApp } from './github.js'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; describe('getOctokitWithGithubApp', () => { beforeEach(() => { @@ -89,6 +93,37 @@ describe('getOctokitWithGithubApp', () => { expect(mocks.hasEntitlement).not.toHaveBeenCalled(); }); + test('reports incomplete discovery when no matching installation exists', async () => { + const fallbackOctokit = {} as Octokit; + mocks.hasEntitlement.mockResolvedValue(true); + mocks.getInstallationToken.mockResolvedValue(null); + + await expect( + collectRepositoryDiscoveryIssues(() => + getOctokitWithGithubApp( + fallbackOctokit, + 'example', + undefined, + 'org example', + ) + ), + ).resolves.toEqual({ + value: fallbackOctokit, + issues: [ + { + code: 'AUTHENTICATION_FALLBACK', + effect: 'DISCOVERY_INCOMPLETE', + subject: { + kind: 'configuration', + value: 'GitHub App installation for example on github.com', + }, + message: + 'No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.', + }, + ], + }); + }); + test('does not fall back when GitHub App token resolution fails', async () => { const error = new Error('rate limited'); mocks.hasEntitlement.mockResolvedValue(true); diff --git a/packages/backend/src/repoCompileUtils.test.ts b/packages/backend/src/repoCompileUtils.test.ts index 344079f87..ee60d580c 100644 --- a/packages/backend/src/repoCompileUtils.test.ts +++ b/packages/backend/src/repoCompileUtils.test.ts @@ -43,7 +43,7 @@ describe('compileGenericGitHostConfig_file', () => { vi.resetAllMocks(); }); - test('should return warning when glob pattern matches no paths', async () => { + test('should return no repositories when glob pattern matches no paths', async () => { mockedGlob.mockResolvedValue([]); const config = { @@ -53,13 +53,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings).toHaveLength(1); - expect(result.warnings[0]).toContain('No paths matched the pattern'); - expect(result.warnings[0]).toContain('/path/to/nonexistent/repo'); + expect(result).toHaveLength(0); }); - test('should return warning when path is a file, not a directory', async () => { + test('should return no repositories when path is a file, not a directory', async () => { mockedGlob.mockResolvedValue(['/path/to/a-file.txt']); mockedFsStat.mockResolvedValue({ isDirectory: () => false } as any); @@ -70,12 +67,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings.length).toBeGreaterThanOrEqual(1); - expect(result.warnings.some(w => w.includes('not a directory'))).toBe(true); + expect(result).toHaveLength(0); }); - test('should return warning when path is not a valid git repo', async () => { + test('should return no repositories when path is not a valid git repo', async () => { mockedGlob.mockResolvedValue(['/path/to/not-a-repo']); mockedIsPathAValidGitRepoRoot.mockResolvedValue(false); @@ -86,13 +81,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings.length).toBeGreaterThanOrEqual(1); - expect(result.warnings.some(w => w.includes('not a git repository'))).toBe(true); - expect(result.warnings.some(w => w.includes('No valid git repositories found'))).toBe(true); + expect(result).toHaveLength(0); }); - test('should return warning when git repo has no origin url', async () => { + test('should return no repositories when git repo has no origin url', async () => { mockedGlob.mockResolvedValue(['/path/to/repo']); mockedIsPathAValidGitRepoRoot.mockResolvedValue(true); mockedGetOriginUrl.mockResolvedValue(null); @@ -104,10 +96,7 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings.length).toBeGreaterThanOrEqual(1); - expect(result.warnings.some(w => w.includes('remote.origin.url not found'))).toBe(true); - expect(result.warnings.some(w => w.includes('No valid git repositories found'))).toBe(true); + expect(result).toHaveLength(0); }); test('should successfully compile when valid git repo is found', async () => { @@ -122,10 +111,9 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); - expect(result.repoData[0].cloneUrl).toBe('file:///path/to/valid/repo'); - expect(result.repoData[0].name).toBe('github.com/test/repo'); + expect(result).toHaveLength(1); + expect(result[0].cloneUrl).toBe('file:///path/to/valid/repo'); + expect(result[0].name).toBe('github.com/test/repo'); }); test('should include port in repo name when origin url has a port', async () => { @@ -140,14 +128,13 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); - expect(result.repoData[0].cloneUrl).toBe('file:///path/to/valid/repo'); + expect(result).toHaveLength(1); + expect(result[0].cloneUrl).toBe('file:///path/to/valid/repo'); // The name should include the port to match what zoekt derives from the origin URL - expect(result.repoData[0].name).toBe('git.kernel.org:443/pub/scm/bluetooth/bluez'); + expect(result[0].name).toBe('git.kernel.org:443/pub/scm/bluetooth/bluez'); }); - test('should return warnings for invalid repos and success for valid ones', async () => { + test('should return valid repositories and omit invalid ones', async () => { mockedGlob.mockResolvedValue(['/path/to/valid/repo', '/path/to/invalid/repo']); mockedIsPathAValidGitRepoRoot.mockImplementation(async ({ path }) => { return path === '/path/to/valid/repo'; @@ -166,10 +153,7 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(1); - expect(result.warnings[0]).toContain('/path/to/invalid/repo'); - expect(result.warnings[0]).toContain('not a git repository'); + expect(result).toHaveLength(1); }); test('should decode URL-encoded characters in origin url pathname', async () => { @@ -185,11 +169,10 @@ describe('compileGenericGitHostConfig_file', () => { const result = await compileGenericGitHostConfig_file(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); + expect(result).toHaveLength(1); // The repo name should have decoded spaces, not %20 - expect(result.repoData[0].name).toBe('github.com/test/Project Name With Spaces'); - expect(result.repoData[0].displayName).toBe('github.com/test/Project Name With Spaces'); + expect(result[0].name).toBe('github.com/test/Project Name With Spaces'); + expect(result[0].displayName).toBe('github.com/test/Project Name With Spaces'); }); }); @@ -202,7 +185,7 @@ describe('compileGenericGitHostConfig_url', () => { vi.resetAllMocks(); }); - test('should return warning when url is not a valid git repo', async () => { + test('should return no repositories when url is not a valid git repo', async () => { mockedIsUrlAValidGitRepo.mockResolvedValue(false); const config = { @@ -212,9 +195,7 @@ describe('compileGenericGitHostConfig_url', () => { const result = await compileGenericGitHostConfig_url(config, 1); - expect(result.repoData).toHaveLength(0); - expect(result.warnings).toHaveLength(1); - expect(result.warnings[0]).toContain('not a git repository'); + expect(result).toHaveLength(0); }); test('should successfully compile with gitConfig when valid git repo url is found', async () => { @@ -227,13 +208,12 @@ describe('compileGenericGitHostConfig_url', () => { const result = await compileGenericGitHostConfig_url(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.warnings).toHaveLength(0); - expect(result.repoData[0].cloneUrl).toBe('https://git.kernel.org/pub/scm/bluetooth/bluez.git'); - expect(result.repoData[0].name).toBe('git.kernel.org/pub/scm/bluetooth/bluez'); + expect(result).toHaveLength(1); + expect(result[0].cloneUrl).toBe('https://git.kernel.org/pub/scm/bluetooth/bluez.git'); + expect(result[0].name).toBe('git.kernel.org/pub/scm/bluetooth/bluez'); // Verify gitConfig is set properly (this is the key fix for SOU-218) - const metadata = result.repoData[0].metadata as { gitConfig?: Record }; + const metadata = result[0].metadata as { gitConfig?: Record }; expect(metadata.gitConfig).toBeDefined(); expect(metadata.gitConfig!['zoekt.name']).toBe('git.kernel.org/pub/scm/bluetooth/bluez'); expect(metadata.gitConfig!['zoekt.web-url']).toBe('https://git.kernel.org/pub/scm/bluetooth/bluez.git'); @@ -253,10 +233,10 @@ describe('compileGenericGitHostConfig_url', () => { const result = await compileGenericGitHostConfig_url(config, 1); - expect(result.repoData).toHaveLength(1); - expect(result.repoData[0].name).toBe('github.com/test/repo'); + expect(result).toHaveLength(1); + expect(result[0].name).toBe('github.com/test/repo'); - const metadata = result.repoData[0].metadata as { gitConfig?: Record }; + const metadata = result[0].metadata as { gitConfig?: Record }; expect(metadata.gitConfig!['zoekt.name']).toBe('github.com/test/repo'); }); }); diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index 1e65565f1..ad8c728d2 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -49,18 +49,11 @@ const extractHostWithPort = (url: string): string | null => { return match ? match[1] : null; }; -type CompileResult = { - repoData: RepoData[], - warnings: string[], -} - export const compileGithubConfig = async ( config: GithubConnectionConfig, connectionId: number, - signal: AbortSignal): Promise => { - const gitHubReposResult = await getGitHubReposFromConfig(config, signal); - const gitHubRepos = gitHubReposResult.repos; - const warnings = gitHubReposResult.warnings; + signal: AbortSignal): Promise => { + const gitHubRepos = await getGitHubReposFromConfig(config, signal); const hostUrl = (config.url ?? 'https://github.com').replace(/\/+$/, ''); @@ -82,10 +75,7 @@ export const compileGithubConfig = async ( }; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const createGitHubRepoRecord = ({ @@ -160,11 +150,10 @@ export const createGitHubRepoRecord = ({ export const compileGitlabConfig = async ( config: GitlabConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { const gitlabReposResult = await getGitLabReposFromConfig(config); const gitlabRepos = gitlabReposResult.repos; - const warnings = gitlabReposResult.warnings; const hostUrl = (config.url ?? 'https://gitlab.com').replace(/\/+$/, ''); const webUrl = (config.webUrl ?? hostUrl).replace(/\/+$/, ''); @@ -240,19 +229,15 @@ export const compileGitlabConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const compileGiteaConfig = async ( config: GiteaConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { const giteaReposResult = await getGiteaReposFromConfig(config); const giteaRepos = giteaReposResult.repos; - const warnings = giteaReposResult.warnings; const hostUrl = (config.url ?? 'https://gitea.com').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -310,15 +295,12 @@ export const compileGiteaConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const compileGerritConfig = async ( config: GerritConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { const gerritRepos = await getGerritReposFromConfig(config); const hostUrl = config.url.replace(/\/+$/, ''); @@ -394,19 +376,15 @@ export const compileGerritConfig = async ( return record; }) - return { - repoData: repos, - warnings: [], - }; + return repos; } export const compileBitbucketConfig = async ( config: BitbucketConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { const bitbucketReposResult = await getBitbucketReposFromConfig(config); const bitbucketRepos = bitbucketReposResult.repos; - const warnings = bitbucketReposResult.warnings; const hostUrl = (config.url ?? 'https://bitbucket.org').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -561,16 +539,13 @@ export const compileBitbucketConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } export const compileGenericGitHostConfig = async ( config: GenericGitHostConnectionConfig, connectionId: number -): Promise => { +): Promise => { const configUrl = new URL(config.url); if (configUrl.protocol === 'file:') { return compileGenericGitHostConfig_file(config, connectionId); @@ -587,7 +562,7 @@ export const compileGenericGitHostConfig = async ( export const compileGenericGitHostConfig_file = async ( config: GenericGitHostConnectionConfig, connectionId: number, -): Promise => { +): Promise => { const configUrl = new URL(config.url); assert(configUrl.protocol === 'file:', 'config.url must be a file:// URL'); @@ -597,17 +572,11 @@ export const compileGenericGitHostConfig_file = async ( }); const repos: RepoData[] = []; - const warnings: string[] = []; - // Warn if the glob pattern matched no paths at all if (repoPaths.length === 0) { const warning = `No paths matched the pattern '${configUrl.pathname}'. Please verify the path exists and is accessible.`; logger.warn(warning); - warnings.push(warning); - return { - repoData: repos, - warnings, - }; + return repos; } logger.debug(`Found ${repoPaths.length} path(s) matching pattern '${configUrl.pathname}'`); @@ -617,7 +586,6 @@ export const compileGenericGitHostConfig_file = async ( if (!stat || !stat.isDirectory()) { const warning = `Skipping ${repoPath} - path is not a directory.`; logger.warn(warning); - warnings.push(warning); return; } @@ -627,7 +595,6 @@ export const compileGenericGitHostConfig_file = async ( if (!isGitRepo) { const warning = `Skipping ${repoPath} - not a git repository.`; logger.warn(warning); - warnings.push(warning); return; } @@ -635,7 +602,6 @@ export const compileGenericGitHostConfig_file = async ( if (!origin) { const warning = `Skipping ${repoPath} - remote.origin.url not found in git config.`; logger.warn(warning); - warnings.push(warning); return; } @@ -690,26 +656,20 @@ export const compileGenericGitHostConfig_file = async ( if (repos.length === 0) { const warning = `No valid git repositories found from ${repoPaths.length} matched path(s). Check the warnings for details on individual paths.`; logger.warn(warning); - warnings.push(warning); } else { logger.debug(`Successfully found ${repos.length} valid git repository(s) from ${repoPaths.length} matched path(s)`); } - return { - repoData: repos, - warnings, - } + return repos; } export const compileGenericGitHostConfig_url = async ( config: GenericGitHostConnectionConfig, connectionId: number, -): Promise => { +): Promise => { const remoteUrl = new URL(config.url); assert(remoteUrl.protocol === 'http:' || remoteUrl.protocol === 'https:', 'config.url must be a http:// or https:// URL'); - const warnings: string[] = []; - // Validate that we are dealing with a valid git repo. const isGitRepo = await isUrlAValidGitRepo({ cloneUrl: remoteUrl.toString(), @@ -717,11 +677,7 @@ export const compileGenericGitHostConfig_url = async ( if (!isGitRepo) { const warning = `Skipping ${remoteUrl.toString()} - not a git repository.`; logger.warn(warning); - warnings.push(warning); - return { - repoData: [], - warnings, - } + return []; } // @note: matches the naming here: @@ -765,19 +721,15 @@ export const compileGenericGitHostConfig_url = async ( } satisfies RepoMetadata, }; - return { - repoData: [repo], - warnings, - } + return [repo]; } export const compileAzureDevOpsConfig = async ( config: AzureDevOpsConnectionConfig, - connectionId: number): Promise => { + connectionId: number): Promise => { const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config); const azureDevOpsRepos = azureDevOpsReposResult.repos; - const warnings = azureDevOpsReposResult.warnings; const hostUrl = (config.url ?? 'https://dev.azure.com').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -846,8 +798,5 @@ export const compileAzureDevOpsConfig = async ( return record; }) - return { - repoData: repos, - warnings, - }; + return repos; } diff --git a/packages/backend/src/repositoryDiscoveryIssueContext.test.ts b/packages/backend/src/repositoryDiscoveryIssueContext.test.ts new file mode 100644 index 000000000..6db8ff351 --- /dev/null +++ b/packages/backend/src/repositoryDiscoveryIssueContext.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from "vitest"; +import { + collectRepositoryDiscoveryIssues, + reportRepositoryDiscoveryIssue, +} from "./repositoryDiscoveryIssueContext.js"; + +const repositoryIssue = (repository: string) => ({ + code: "NOT_FOUND_OR_INACCESSIBLE" as const, + effect: "TARGET_SKIPPED" as const, + subject: { + kind: "repository" as const, + value: repository, + }, + message: `Repository ${repository} was not found or is inaccessible.`, +}); + +describe("repository discovery issue context", () => { + test("returns the routine value and collected issues", async () => { + const issue = repositoryIssue("sourcebot-dev/legacy"); + + await expect( + collectRepositoryDiscoveryIssues(async () => { + reportRepositoryDiscoveryIssue(issue); + return ["repository"]; + }), + ).resolves.toEqual({ + value: ["repository"], + issues: [issue], + }); + }); + + test("does nothing when reporting outside a collection context", () => { + expect(() => + reportRepositoryDiscoveryIssue( + repositoryIssue("sourcebot-dev/legacy"), + ) + ).not.toThrow(); + }); + + test("deduplicates identical issues", async () => { + const issue = repositoryIssue("sourcebot-dev/legacy"); + + await expect( + collectRepositoryDiscoveryIssues(() => { + reportRepositoryDiscoveryIssue(issue); + reportRepositoryDiscoveryIssue(issue); + }), + ).resolves.toEqual({ + value: undefined, + issues: [issue], + }); + }); + + test("keeps concurrent collection contexts isolated", async () => { + let releaseFirst: () => void = () => {}; + const firstCanFinish = new Promise((resolve) => { + releaseFirst = resolve; + }); + + const first = collectRepositoryDiscoveryIssues(async () => { + reportRepositoryDiscoveryIssue(repositoryIssue("org/first")); + await firstCanFinish; + reportRepositoryDiscoveryIssue( + repositoryIssue("org/first-after-await"), + ); + }); + const second = collectRepositoryDiscoveryIssues(async () => { + reportRepositoryDiscoveryIssue(repositoryIssue("org/second")); + releaseFirst(); + }); + + await expect(Promise.all([first, second])).resolves.toEqual([ + { + value: undefined, + issues: [ + repositoryIssue("org/first"), + repositoryIssue("org/first-after-await"), + ], + }, + { + value: undefined, + issues: [repositoryIssue("org/second")], + }, + ]); + }); + + test("does not retain a context after the routine rejects", async () => { + await expect( + collectRepositoryDiscoveryIssues(async () => { + throw new Error("Discovery failed"); + }), + ).rejects.toThrow("Discovery failed"); + + reportRepositoryDiscoveryIssue( + repositoryIssue("sourcebot-dev/legacy"), + ); + await expect( + collectRepositoryDiscoveryIssues(async () => {}), + ).resolves.toEqual({ value: undefined, issues: [] }); + }); +}); diff --git a/packages/backend/src/repositoryDiscoveryIssueContext.ts b/packages/backend/src/repositoryDiscoveryIssueContext.ts new file mode 100644 index 000000000..983cd4082 --- /dev/null +++ b/packages/backend/src/repositoryDiscoveryIssueContext.ts @@ -0,0 +1,53 @@ +import { + repositoryDiscoveryIssueSchema, + type RepositoryDiscoveryIssue, +} from "@sourcebot/shared"; +import { AsyncLocalStorage } from "node:async_hooks"; + +interface RepositoryDiscoveryIssueContext { + issuesByKey: Map; +} + +export interface RepositoryDiscoveryIssueCollection { + value: T; + issues: RepositoryDiscoveryIssue[]; +} + +const repositoryDiscoveryIssueStorage = + new AsyncLocalStorage(); + +const getIssueKey = (issue: RepositoryDiscoveryIssue): string => + JSON.stringify([ + issue.code, + issue.effect, + issue.subject?.kind ?? null, + issue.subject?.value ?? null, + issue.message, + ]); + +export const collectRepositoryDiscoveryIssues = async ( + routine: () => T | Promise, +): Promise> => { + const context: RepositoryDiscoveryIssueContext = { + issuesByKey: new Map(), + }; + + const value = await repositoryDiscoveryIssueStorage.run(context, routine); + + return { + value, + issues: [...context.issuesByKey.values()], + }; +}; + +export const reportRepositoryDiscoveryIssue = ( + issue: RepositoryDiscoveryIssue, +): void => { + const context = repositoryDiscoveryIssueStorage.getStore(); + if (!context) { + return; + } + + const parsedIssue = repositoryDiscoveryIssueSchema.parse(issue); + context.issuesByKey.set(getIssueKey(parsedIssue), parsedIssue); +}; diff --git a/packages/shared/src/connectionSync.test.ts b/packages/shared/src/connectionSync.test.ts index 42b9ffbf0..9076d3be8 100644 --- a/packages/shared/src/connectionSync.test.ts +++ b/packages/shared/src/connectionSync.test.ts @@ -1,8 +1,5 @@ import { describe, expect, test } from "vitest"; -import { - connectionSyncResultSchema, - connectionSyncPartialSuccessReasonSchema, -} from "./connectionSync.js"; +import { connectionSyncResultSchema } from "./connectionSync.js"; describe("connectionSyncResultSchema", () => { test("accepts a successful result", () => { @@ -43,19 +40,3 @@ describe("connectionSyncResultSchema", () => { ).toBe(false); }); }); - -describe("connectionSyncPartialSuccessReasonSchema", () => { - test("allows a reason without a subject", () => { - expect( - connectionSyncPartialSuccessReasonSchema.parse({ - code: "INVALID_PROVIDER_RESPONSE", - effect: "DISCOVERY_INCOMPLETE", - message: "The provider returned an invalid repository.", - }), - ).toEqual({ - code: "INVALID_PROVIDER_RESPONSE", - effect: "DISCOVERY_INCOMPLETE", - message: "The provider returned an invalid repository.", - }); - }); -}); diff --git a/packages/shared/src/connectionSync.ts b/packages/shared/src/connectionSync.ts index 7a64b768a..3df66e5bb 100644 --- a/packages/shared/src/connectionSync.ts +++ b/packages/shared/src/connectionSync.ts @@ -1,57 +1,5 @@ import { z } from "zod"; - -export const connectionSyncPartialSuccessReasonCodeSchema = z.enum([ - "NOT_FOUND_OR_INACCESSIBLE", - "INVALID_TARGET", - "UNSUPPORTED_CONFIGURATION", - "INVALID_REPOSITORY_SOURCE", - "ENUMERATION_FAILED", - "INVALID_PROVIDER_RESPONSE", -]); - -export type ConnectionSyncPartialSuccessReasonCode = z.infer< - typeof connectionSyncPartialSuccessReasonCodeSchema ->; - -export const connectionSyncPartialSuccessEffectSchema = z.enum([ - "TARGET_SKIPPED", - "CONFIGURATION_IGNORED", - "DISCOVERY_INCOMPLETE", -]); - -export type ConnectionSyncPartialSuccessEffect = z.infer< - typeof connectionSyncPartialSuccessEffectSchema ->; - -export const connectionSyncPartialSuccessSubjectSchema = z.object({ - kind: z.enum([ - "organization", - "group", - "user", - "workspace", - "project", - "repository", - "path", - "url", - "configuration", - ]), - value: z.string().min(1), -}); - -export type ConnectionSyncPartialSuccessSubject = z.infer< - typeof connectionSyncPartialSuccessSubjectSchema ->; - -export const connectionSyncPartialSuccessReasonSchema = z.object({ - code: connectionSyncPartialSuccessReasonCodeSchema, - effect: connectionSyncPartialSuccessEffectSchema, - subject: connectionSyncPartialSuccessSubjectSchema.optional(), - message: z.string().min(1), -}); - -export type ConnectionSyncPartialSuccessReason = z.infer< - typeof connectionSyncPartialSuccessReasonSchema ->; +import { repositoryDiscoveryIssueSchema } from "./repositoryDiscovery.js"; export const connectionSyncResultSchema = z.discriminatedUnion("outcome", [ z.object({ @@ -59,7 +7,7 @@ export const connectionSyncResultSchema = z.discriminatedUnion("outcome", [ }), z.object({ outcome: z.literal("PARTIAL_SUCCESS"), - reasons: z.array(connectionSyncPartialSuccessReasonSchema).min(1), + reasons: z.array(repositoryDiscoveryIssueSchema).min(1), }), ]); diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 486f346ed..2f38be5b3 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -96,18 +96,20 @@ export { export type { Version } from "./versionUtils.js"; export { connectionSyncResultSchema, - connectionSyncPartialSuccessEffectSchema, - connectionSyncPartialSuccessReasonCodeSchema, - connectionSyncPartialSuccessReasonSchema, - connectionSyncPartialSuccessSubjectSchema, } from "./connectionSync.js"; +export type { ConnectionSyncResult } from "./connectionSync.js"; +export { + repositoryDiscoveryIssueCodeSchema, + repositoryDiscoveryIssueEffectSchema, + repositoryDiscoveryIssueSchema, + repositoryDiscoveryIssueSubjectSchema, +} from "./repositoryDiscovery.js"; export type { - ConnectionSyncResult, - ConnectionSyncPartialSuccessEffect, - ConnectionSyncPartialSuccessReason, - ConnectionSyncPartialSuccessReasonCode, - ConnectionSyncPartialSuccessSubject, -} from "./connectionSync.js"; + RepositoryDiscoveryIssue, + RepositoryDiscoveryIssueCode, + RepositoryDiscoveryIssueEffect, + RepositoryDiscoveryIssueSubject, +} from "./repositoryDiscovery.js"; export type { QueueName, DataOf, diff --git a/packages/shared/src/repositoryDiscovery.test.ts b/packages/shared/src/repositoryDiscovery.test.ts new file mode 100644 index 000000000..b0364516d --- /dev/null +++ b/packages/shared/src/repositoryDiscovery.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { repositoryDiscoveryIssueSchema } from "./repositoryDiscovery.js"; + +describe("repositoryDiscoveryIssueSchema", () => { + test("allows an issue without a subject", () => { + expect( + repositoryDiscoveryIssueSchema.parse({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "The provider returned an invalid repository.", + }), + ).toEqual({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "The provider returned an invalid repository.", + }); + }); + + test("accepts an authentication fallback issue", () => { + expect( + repositoryDiscoveryIssueSchema.parse({ + code: "AUTHENTICATION_FALLBACK", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "configuration", + value: "GitHub App installation for sourcebot on github.com", + }, + message: + "No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.", + }), + ).toEqual({ + code: "AUTHENTICATION_FALLBACK", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "configuration", + value: "GitHub App installation for sourcebot on github.com", + }, + message: + "No matching GitHub App installation was found. Discovery used legacy credentials and may be incomplete.", + }); + }); +}); diff --git a/packages/shared/src/repositoryDiscovery.ts b/packages/shared/src/repositoryDiscovery.ts new file mode 100644 index 000000000..8cded6ddc --- /dev/null +++ b/packages/shared/src/repositoryDiscovery.ts @@ -0,0 +1,55 @@ +import { z } from "zod"; + +export const repositoryDiscoveryIssueCodeSchema = z.enum([ + "NOT_FOUND_OR_INACCESSIBLE", + "INVALID_TARGET", + "UNSUPPORTED_CONFIGURATION", + "INVALID_REPOSITORY_SOURCE", + "ENUMERATION_FAILED", + "INVALID_PROVIDER_RESPONSE", + "AUTHENTICATION_FALLBACK", +]); + +export type RepositoryDiscoveryIssueCode = z.infer< + typeof repositoryDiscoveryIssueCodeSchema +>; + +export const repositoryDiscoveryIssueEffectSchema = z.enum([ + "TARGET_SKIPPED", + "CONFIGURATION_IGNORED", + "DISCOVERY_INCOMPLETE", +]); + +export type RepositoryDiscoveryIssueEffect = z.infer< + typeof repositoryDiscoveryIssueEffectSchema +>; + +export const repositoryDiscoveryIssueSubjectSchema = z.object({ + kind: z.enum([ + "organization", + "group", + "user", + "workspace", + "project", + "repository", + "path", + "url", + "configuration", + ]), + value: z.string().min(1), +}); + +export type RepositoryDiscoveryIssueSubject = z.infer< + typeof repositoryDiscoveryIssueSubjectSchema +>; + +export const repositoryDiscoveryIssueSchema = z.object({ + code: repositoryDiscoveryIssueCodeSchema, + effect: repositoryDiscoveryIssueEffectSchema, + subject: repositoryDiscoveryIssueSubjectSchema.optional(), + message: z.string().min(1), +}); + +export type RepositoryDiscoveryIssue = z.infer< + typeof repositoryDiscoveryIssueSchema +>; From e3242ee9c3f479d0445db06ccdef9e6539aa4c9c Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 18:01:03 -0700 Subject: [PATCH 15/34] connections table --- packages/shared/src/index.server.ts | 1 + packages/shared/src/queue.ts | 11 +- .../app/(app)/components/jobLogsDialog.tsx | 280 ++++++++ .../repos/components/reposTable.test.tsx | 76 +++ .../repos/components/syncIssuePopover.tsx | 37 +- .../components/connectionActionsMenu.tsx | 91 +++ .../components/connectionsTable.test.tsx | 459 +++++++++++++ .../components/connectionsTable.tsx | 643 ++++++++++++++++++ .../components/syncAnnotation.tsx | 169 +++++ .../components/syncIssuePopover.tsx | 272 ++++++++ .../(app)/settings/connectionsv2/layout.tsx | 10 + .../app/(app)/settings/connectionsv2/page.tsx | 121 ++++ .../app/(app)/settings/connectionsv2/types.ts | 11 + packages/web/src/app/api/(client)/client.ts | 23 + .../(server)/connection-sync-status/route.ts | 63 ++ .../src/app/api/(server)/job-logs/route.ts | 70 ++ 16 files changed, 2332 insertions(+), 5 deletions(-) create mode 100644 packages/web/src/app/(app)/components/jobLogsDialog.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/components/connectionActionsMenu.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/layout.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/page.tsx create mode 100644 packages/web/src/app/(app)/settings/connectionsv2/types.ts create mode 100644 packages/web/src/app/api/(server)/connection-sync-status/route.ts create mode 100644 packages/web/src/app/api/(server)/job-logs/route.ts diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 2f38be5b3..0c16dd456 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -125,6 +125,7 @@ export { CONNECTION_QUEUE, DEFAULT_JOB_OPTIONS, JOB_PRIORITIES, + QUEUE_SPECS, REPO_INDEX_QUEUE, REPO_PERMISSION_SYNC_QUEUE, } from "./queue.js"; diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 6c63db6ca..7036928f0 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -49,7 +49,7 @@ export const JOB_PRIORITIES = { const TWO_WEEKS_IN_SECONDS = 14 * 24 * 60 * 60; export const DEFAULT_JOB_OPTIONS: JobOptions = { - attempts: 4, + attempts: 2, backoff: { type: "exponential", delayMs: 30_000, @@ -154,3 +154,12 @@ export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { jobOptions: DEFAULT_JOB_OPTIONS, dedupKey: (data) => `repo:${data.repoId}`, }; + +export const QUEUE_SPECS = { + [ATTACHMENT_PRUNE_QUEUE.name]: ATTACHMENT_PRUNE_QUEUE, + [AUDIT_LOG_PRUNE_QUEUE.name]: AUDIT_LOG_PRUNE_QUEUE, + [CONNECTION_QUEUE.name]: CONNECTION_QUEUE, + [REPO_INDEX_QUEUE.name]: REPO_INDEX_QUEUE, + [ACCOUNT_PERMISSION_SYNC_QUEUE.name]: ACCOUNT_PERMISSION_SYNC_QUEUE, + [REPO_PERMISSION_SYNC_QUEUE.name]: REPO_PERMISSION_SYNC_QUEUE, +} as const satisfies { [TName in QueueName]: QueueSpec }; diff --git a/packages/web/src/app/(app)/components/jobLogsDialog.tsx b/packages/web/src/app/(app)/components/jobLogsDialog.tsx new file mode 100644 index 000000000..421ae7150 --- /dev/null +++ b/packages/web/src/app/(app)/components/jobLogsDialog.tsx @@ -0,0 +1,280 @@ +"use client"; + +import { CopyIconButton } from "@/app/(app)/components/copyIconButton"; +import { + getJobLogs, +} from "@/app/api/(client)/client"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import type { JobLogEntry, JobLogLevel, QueueName } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { Check, Copy, Loader2 } from "lucide-react"; +import { useRef, useState } from "react"; + +type JobLogsDialogProps = { + queue: QueueName; + subject: string; + jobId: string; + open: boolean; + onOpenChange: (open: boolean) => void; +}; + +const timestampFormatter = new Intl.DateTimeFormat("en-CA", { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + fractionalSecondDigits: 3, + timeZoneName: "short", + hourCycle: "h23", +}); + +const formatTimestamp = (timestamp: string | null) => { + if (!timestamp) { + return "Unknown time"; + } + + const date = new Date(timestamp); + return Number.isNaN(date.getTime()) + ? timestamp + : timestampFormatter.format(date).replace(",", ""); +}; + +const formatLogLine = (entry: JobLogEntry) => { + const fields = entry.fields + ? ` ${JSON.stringify(entry.fields)}` + : ""; + return `${formatTimestamp(entry.timestamp)} ${entry.level.toUpperCase()} ${entry.message}${fields}`; +}; + +const levelStyles: Record< + JobLogLevel, + { dot: string; text: string } +> = { + debug: { + dot: "bg-muted-foreground", + text: "text-muted-foreground", + }, + info: { + dot: "bg-blue-500", + text: "text-blue-600 dark:text-blue-400", + }, + warn: { + dot: "bg-amber-500", + text: "text-amber-700 dark:text-amber-400", + }, + error: { + dot: "bg-destructive", + text: "text-destructive", + }, +}; + +const LogLevel = ({ level }: { level: JobLogLevel }) => ( + + + {level} + +); + +export const JobLogsDialog = ({ + queue, + subject, + jobId, + open, + onOpenChange, +}: JobLogsDialogProps) => { + const titleRef = useRef(null); + const [copiedAll, setCopiedAll] = useState(false); + const { data, isPending, isError } = useQuery({ + queryKey: ["job-logs", queue, jobId], + queryFn: ({ signal }) => + getJobLogs(queue, jobId, signal), + enabled: open, + }); + const latestAttempt = data?.logs.reduce( + (latest, entry) => entry.attempt === null + ? latest + : Math.max(latest ?? entry.attempt, entry.attempt), + null, + ) ?? null; + const displayedLogs = latestAttempt === null + ? data?.logs ?? [] + : data?.logs.filter((entry) => entry.attempt === latestAttempt) ?? []; + + const copyAllLogs = async () => { + try { + await navigator.clipboard.writeText( + displayedLogs.map(formatLogLine).join("\n"), + ); + setCopiedAll(true); + setTimeout(() => setCopiedAll(false), 2_000); + } catch { + setCopiedAll(false); + } + }; + + return ( + + { + event.preventDefault(); + titleRef.current?.focus(); + }} + > + +
+
+ + Job logs + + + {subject} · {jobId} + +
+ +
+ +
+
+ + + + + + + + + {[ + "Time", + "Level", + "Message", + ].map((heading) => ( + + ))} + + + + {isPending + ? ( + + + + ) + : isError + ? ( + + + + ) + : displayedLogs.length > 0 + ? displayedLogs.map((entry, index) => ( + + + + + + )) + : ( + + + + )} + +
+ {heading} +
+
+ + + Loading job logs + +
+
+ Failed to load job logs. +
+ + + + +
+ {entry.message} + {entry.fields && ( + + {` ${JSON.stringify(entry.fields)}`} + + )} +
+ { + try { + void navigator.clipboard.writeText( + formatLogLine( + entry, + ), + ); + return true; + } catch { + return false; + } + }} + /> +
+ No logs were recorded for this job. +
+
+
+
+ +
+ ); +}; diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index 22b5a8d40..ed5a43277 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -1,5 +1,6 @@ import { TooltipProvider } from "@/components/ui/tooltip"; import type { CodeHostType } from "@sourcebot/db"; +import type { JobLogs } from "@sourcebot/shared"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; import { afterEach, describe, expect, test, vi } from "vitest"; @@ -359,6 +360,81 @@ describe("ReposTable", () => { expect(screen.getByText("job-1")).toBeTruthy(); }); + test("opens retained repository indexing logs", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal("navigator", { clipboard: { writeText } }); + const jobLogs: JobLogs = { + count: 1, + logs: [{ + version: 1, + timestamp: "2026-08-18T23:00:00.000Z", + level: "error", + message: "Repository indexing failed", + attempt: 2, + }], + }; + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValueOnce(Response.json(jobLogs)), + ); + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }]); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + fireEvent.click(screen.getByRole("button", { name: "View logs" })); + + const dialog = await screen.findByRole("dialog"); + await waitFor(() => { + expect(dialog.textContent).toContain("Repository indexing failed"); + }); + fireEvent.click(within(dialog).getByRole("button", { + name: "Copy all", + })); + await waitFor(() => expect(writeText).toHaveBeenCalledOnce()); + expect(writeText.mock.calls[0]?.[0]).toContain( + "Repository indexing failed", + ); + expect(fetch).toHaveBeenCalledWith( + "/api/job-logs", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + queue: "repo-index", + jobId: "job-1", + }), + }), + ); + }); + + test("hides job log access from non-owners", () => { + renderTable([{ + ...repos[0], + latestJob: { + ...repos[0].latestJob!, + status: "FAILED", + errorMessage: "Authentication failed while cloning", + result: null, + }, + }], false); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for acme/first", + })); + + expect( + screen.queryByRole("button", { name: "View logs" }), + ).toBeNull(); + }); + test("schedules a retry and transitions an unindexed repository to syncing", async () => { reposActions.indexRepo.mockResolvedValue({ jobId: "retry-job" }); vi.stubGlobal("fetch", vi.fn().mockResolvedValue(Response.json({ diff --git a/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx index 39e5ecec3..dbe846b10 100644 --- a/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx +++ b/packages/web/src/app/(app)/repos/components/syncIssuePopover.tsx @@ -1,5 +1,6 @@ "use client"; +import { JobLogsDialog } from "@/app/(app)/components/jobLogsDialog"; import { badgeVariants } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { useToast } from "@/components/hooks/use-toast"; @@ -7,7 +8,14 @@ import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { indexRepo } from "@/features/repos/actions"; import { cn, isServiceError } from "@/lib/utils"; -import { ChevronDown, CircleX, Loader2, RotateCw, TriangleAlert } from "lucide-react"; +import { + ChevronDown, + CircleX, + Loader2, + RotateCw, + ScrollText, + TriangleAlert, +} from "lucide-react"; import { useState } from "react"; import { DisplayDate } from "../../components/DisplayDate"; import { LightweightCodeHighlighter } from "../../components/lightweightCodeHighlighter"; @@ -27,6 +35,7 @@ export const SyncIssuePopover = ({ onRetryScheduled, }: SyncIssuePopoverProps) => { const [isOpen, setIsOpen] = useState(false); + const [isLogsOpen, setIsLogsOpen] = useState(false); const [isRetrying, setIsRetrying] = useState(false); const { toast } = useToast(); const latestJob = repo.latestJob; @@ -71,7 +80,8 @@ export const SyncIssuePopover = ({ }; return ( - + <> + @@ -161,7 +171,18 @@ export const SyncIssuePopover = ({ {canRetry && ( -
+
+
- + + + ); }; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionActionsMenu.tsx b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionActionsMenu.tsx new file mode 100644 index 000000000..c61b959bd --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionActionsMenu.tsx @@ -0,0 +1,91 @@ +"use client"; + +import { useToast } from "@/components/hooks/use-toast"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { syncConnection } from "@/features/connections/actions"; +import { isServiceError } from "@/lib/utils"; +import { EllipsisVertical, Loader2, RefreshCw } from "lucide-react"; +import { useState } from "react"; + +type ConnectionActionsMenuProps = { + connection: { + id: number; + name: string; + }; + isSyncing: boolean; + onSyncScheduled: (connectionId: number, jobId: string) => void; +}; + +export const ConnectionActionsMenu = ({ + connection, + isSyncing, + onSyncScheduled, +}: ConnectionActionsMenuProps) => { + const [isScheduling, setIsScheduling] = useState(false); + const { toast } = useToast(); + + const scheduleSync = async () => { + setIsScheduling(true); + + try { + const response = await syncConnection(connection.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to sync connection", + description: response.message, + }); + return; + } + + onSyncScheduled(connection.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${connection.name} was queued for syncing.`, + }); + } catch { + toast({ + variant: "destructive", + title: "Failed to sync connection", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsScheduling(false); + } + }; + + return ( + + + + + + void scheduleSync()} + > + {isScheduling ? ( + + ) : ( + + )} + Sync + + + + ); +}; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx new file mode 100644 index 000000000..217ccbb6a --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx @@ -0,0 +1,459 @@ +import type { ConnectionType } from "@sourcebot/db"; +import type { JobLogs } from "@sourcebot/shared"; +import { TooltipProvider } from "@/components/ui/tooltip"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { + act, + cleanup, + fireEvent, + render, + screen, + waitFor, + within, +} from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import type { ConnectionSyncStatusesResponse } from "../types"; +import type { Connection } from "./connectionsTable"; + +const navigation = vi.hoisted(() => ({ + push: vi.fn(), + replace: vi.fn(), + refresh: vi.fn(), + searchParams: "", +})); +const connectionActions = vi.hoisted(() => ({ + syncConnection: vi.fn(), +})); +const toast = vi.hoisted(() => vi.fn()); + +vi.mock("next/navigation", () => ({ + usePathname: () => "/settings/connectionsv2", + useRouter: () => navigation, + useSearchParams: () => new URLSearchParams(navigation.searchParams), +})); + +vi.mock("@/features/connections/actions", () => ({ + syncConnection: connectionActions.syncConnection, +})); + +vi.mock("@/components/hooks/use-toast", () => ({ + useToast: () => ({ toast }), +})); + +const { ConnectionsTable } = await import("./connectionsTable"); + +const connections: Connection[] = [ + { + id: 1, + name: "Primary GitHub", + connectionType: "github" as ConnectionType, + syncedAt: new Date("2026-08-18T12:00:00.000Z"), + latestJob: null, + }, + { + id: 2, + name: "Internal GitLab", + connectionType: "gitlab" as ConnectionType, + syncedAt: null, + latestJob: null, + }, +]; + +type RenderTableOptions = { + data?: Connection[]; + currentPage?: number; + totalCount?: number; + sortBy?: "name" | "syncedAt"; + sortOrder?: "asc" | "desc"; +}; + +const renderTable = (options: RenderTableOptions = {}) => { + const data = options.data ?? connections; + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + + return render( + + + + + , + ); +}; + +beforeEach(() => { + vi.stubGlobal("fetch", vi.fn(() => new Promise(() => {}))); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); + vi.unstubAllGlobals(); + navigation.searchParams = ""; +}); + +describe("ConnectionsTable", () => { + test("renders connection names without detail-page links", () => { + renderTable(); + + expect(screen.getByText("Primary GitHub")).toBeTruthy(); + expect( + screen.queryByRole("link", { name: "Primary GitHub" }), + ).toBeNull(); + }); + + test("uses URL-driven server pagination", () => { + renderTable({ totalCount: 29 }); + + expect(screen.getByText("Showing 1-20 of 29")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: "Next" })); + + expect(navigation.push).toHaveBeenCalledWith( + "/settings/connectionsv2?page=2", + ); + }); + + test("toggles server-side name sorting", () => { + renderTable(); + + fireEvent.click(screen.getByRole("button", { name: "Sort by Name" })); + + expect(navigation.push).toHaveBeenCalledWith( + "/settings/connectionsv2?sortOrder=desc", + ); + }); + + test("focuses connection search when slash is pressed", () => { + renderTable(); + + const searchInput = screen.getByPlaceholderText("Search connections..."); + fireEvent.keyDown(document, { key: "/" }); + + expect(document.activeElement).toBe(searchInput); + }); + + test("shows and polls an explicitly scheduled sync", async () => { + connectionActions.syncConnection.mockResolvedValue({ jobId: "job-1" }); + renderTable(); + + fireEvent.keyDown( + screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), + { key: "Enter" }, + ); + + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + await waitFor(() => { + expect(connectionActions.syncConnection).toHaveBeenCalledWith(1); + expect(screen.getByText("Syncing")).toBeTruthy(); + expect(fetch).toHaveBeenCalledOnce(); + }); + + fireEvent.keyDown( + screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), + { key: "Enter" }, + ); + expect( + screen + .getByRole("menuitem", { name: "Sync" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("updates an actively syncing connection in place", async () => { + let resolveRequest: ((response: Response) => void) | undefined; + vi.mocked(fetch).mockImplementation(() => + new Promise((resolve) => { + resolveRequest = resolve; + }) + ); + const activeConnection: Connection = { + ...connections[0], + latestJob: { + id: "active-job", + data: { connectionId: connections[0].id }, + status: "IN_PROGRESS", + errorMessage: null, + result: null, + }, + }; + renderTable({ data: [activeConnection, connections[1]] }); + + expect(screen.getByText("Syncing")).toBeTruthy(); + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + + const response: ConnectionSyncStatusesResponse = { + connections: [{ + connectionId: activeConnection.id, + syncedAt: new Date().toISOString(), + latestJob: { + ...activeConnection.latestJob!, + status: "COMPLETED", + result: { outcome: "SUCCESS" }, + }, + }], + }; + await act(async () => { + resolveRequest?.(Response.json(response)); + }); + + await waitFor(() => expect(screen.getByText("Completed")).toBeTruthy()); + expect(screen.queryByText("Syncing")).toBeNull(); + const rows = within(screen.getByRole("table")).getAllByRole("row"); + expect(within(rows[1]).getByText("Primary GitHub")).toBeTruthy(); + expect(within(rows[1]).getByText("just now")).toBeTruthy(); + expect(within(rows[2]).getByText("Internal GitLab")).toBeTruthy(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("preserves the latest synced timestamp when scheduling another sync", async () => { + const previousSyncedAt = new Date( + Date.now() - 2 * 24 * 60 * 60 * 1_000, + ); + const completedAt = new Date(); + const connection = { + ...connections[0], + syncedAt: previousSyncedAt, + }; + connectionActions.syncConnection + .mockResolvedValueOnce({ jobId: "first-job" }) + .mockResolvedValueOnce({ jobId: "second-job" }); + vi.mocked(fetch) + .mockResolvedValueOnce(Response.json({ + connections: [{ + connectionId: connection.id, + syncedAt: completedAt.toISOString(), + latestJob: { + id: "first-job", + data: { connectionId: connection.id }, + status: "COMPLETED", + errorMessage: null, + result: { outcome: "SUCCESS" }, + }, + }], + } satisfies ConnectionSyncStatusesResponse)) + .mockImplementation(() => new Promise(() => {})); + renderTable({ data: [connection] }); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => expect(screen.getByText("just now")).toBeTruthy()); + + fireEvent.keyDown(screen.getByRole("button", { + name: "Open actions for Primary GitHub", + }), { key: "Enter" }); + fireEvent.click(screen.getByRole("menuitem", { name: "Sync" })); + + await waitFor(() => { + expect(fetch).toHaveBeenCalledTimes(2); + expect(screen.getByText("Syncing")).toBeTruthy(); + }); + expect(screen.getByText("just now")).toBeTruthy(); + expect(screen.queryByText("2 days ago")).toBeNull(); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test.each([ + ["PENDING", "Syncing"], + ["IN_PROGRESS", "Syncing"], + ["FAILED", "Failed"], + ] as const)("renders a %s sync annotation", (status, label) => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "job-1", + data: { connectionId: connections[0].id }, + status, + errorMessage: status === "FAILED" ? "Sync failed" : null, + result: null, + }, + }], + }); + + expect(screen.getByText(label)).toBeTruthy(); + }); + + test("shows structured discovery issues for a partial success", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "job-1", + data: { connectionId: connections[0].id }, + status: "COMPLETED", + errorMessage: null, + result: { + outcome: "PARTIAL_SUCCESS", + reasons: [{ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "acme/private", + }, + message: "Not found or inaccessible", + }], + }, + }, + }], + }); + + const warning = screen.getByText("Warning"); + expect(warning).toBeTruthy(); + expect(warning.closest("td")).toBe( + screen.getByText("Primary GitHub").closest("td"), + ); + fireEvent.click(screen.getByRole("button", { + name: "View warning details for Primary GitHub", + })); + + expect( + screen.getByText("Connection sync completed with warnings"), + ).toBeTruthy(); + expect(screen.getByText("repository")).toBeTruthy(); + expect(screen.getByText("acme/private")).toBeTruthy(); + expect(screen.getByText("Not found or inaccessible")).toBeTruthy(); + expect(screen.getByText("job-1")).toBeTruthy(); + }); + + test("shows the worker error for a failed sync and supports retry", async () => { + connectionActions.syncConnection.mockResolvedValue({ + jobId: "retry-job", + }); + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "failed-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "Authentication failed while discovering repositories", + result: null, + }, + }], + }); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for Primary GitHub", + })); + + expect(screen.getByText("Connection sync failed")).toBeTruthy(); + expect( + screen.getByText( + "Authentication failed while discovering repositories", + ), + ).toBeTruthy(); + expect(screen.getByText("failed-job")).toBeTruthy(); + + fireEvent.click(screen.getByRole("button", { name: "Retry sync" })); + + await waitFor(() => { + expect(connectionActions.syncConnection).toHaveBeenCalledWith(1); + expect(screen.getByText("Syncing")).toBeTruthy(); + }); + expect(screen.queryByText("Connection sync failed")).toBeNull(); + }); + + test("opens retained job logs in a large dialog", async () => { + const jobLogs: JobLogs = { + count: 3, + logs: [ + { + version: 1, + timestamp: "2026-08-18T23:00:00.000Z", + level: "info", + message: "Earlier attempt failed", + attempt: 1, + }, + { + version: 1, + timestamp: "2026-08-18T23:00:01.000Z", + level: "info", + message: "Starting repository discovery", + attempt: 2, + }, + { + version: 1, + timestamp: "2026-08-18T23:00:02.000Z", + level: "error", + message: "Repository discovery failed", + attempt: 2, + fields: { provider: "gitlab" }, + }, + ], + }; + vi.mocked(fetch).mockResolvedValueOnce(Response.json(jobLogs)); + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "failed-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "fetch failed", + result: null, + }, + }], + }); + + fireEvent.click(screen.getByRole("button", { + name: "View failed details for Primary GitHub", + })); + fireEvent.click(screen.getByRole("button", { name: "View logs" })); + + const dialog = await screen.findByRole("dialog"); + expect(within(dialog).getByText("Job logs")).toBeTruthy(); + await waitFor(() => { + expect(dialog.textContent).toContain("Starting repository discovery"); + expect(dialog.textContent).toContain("Repository discovery failed"); + expect(dialog.textContent).toContain("gitlab"); + }); + expect(dialog.textContent).not.toContain("Earlier attempt failed"); + expect(fetch).toHaveBeenCalledWith( + "/api/job-logs", + expect.objectContaining({ + method: "POST", + body: JSON.stringify({ + queue: "connection-sync", + jobId: "failed-job", + }), + }), + ); + }); + + test("does not annotate a successful sync", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "job-1", + data: { connectionId: connections[0].id }, + status: "COMPLETED", + errorMessage: null, + result: { outcome: "SUCCESS" }, + }, + }], + }); + + expect(screen.queryByText("Syncing")).toBeNull(); + expect(screen.queryByText("Failed")).toBeNull(); + expect(screen.queryByText("Warning")).toBeNull(); + expect(screen.queryByText("Completed")).toBeNull(); + }); +}); diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx new file mode 100644 index 000000000..4536b88a4 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx @@ -0,0 +1,643 @@ +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { Button } from "@/components/ui/button"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, +} from "@/components/ui/input-group"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { cn, getCodeHostIcon } from "@/lib/utils"; +import type { ConnectionType } from "@sourcebot/db"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { useDebounce } from "@uidotdev/usehooks"; +import { + type ColumnDef, + flexRender, + getCoreRowModel, + useReactTable, +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, Loader2, Search } from "lucide-react"; +import Image from "next/image"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import type { ConnectionSyncStatusesResponse } from "../types"; +import { ConnectionActionsMenu } from "./connectionActionsMenu"; +import { + getConnectionSyncAnnotation, + SyncAnnotation, +} from "./syncAnnotation"; + +const POLL_INTERVAL_MS = 5_000; + +export type Connection = { + id: number; + name: string; + connectionType: ConnectionType; + syncedAt: Date | null; + latestJob: WorkloadJob<"connection-sync"> | null; +}; + +type DisplayedConnection = Connection & { + showCompleted: boolean; +}; + +type SortBy = "name" | "syncedAt"; +type SortOrder = "asc" | "desc"; + +const getConnectionSyncStatuses = async ( + connectionIds: number[], + signal: AbortSignal, +): Promise => { + const response = await fetch("/api/connection-sync-status", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ connectionIds }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load connection sync statuses"); + } + + return response.json() as Promise; +}; + +const SortableHeader = ({ + label, + column, + sortBy, + sortOrder, + onSortChange, +}: { + label: string; + column: SortBy; + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; +}) => { + const isActive = sortBy === column; + const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; + + return ( + + ); +}; + +const getColumns = ({ + sortBy, + sortOrder, + onSortChange, + onSyncScheduled, +}: { + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + onSyncScheduled: (connectionId: number, jobId: string) => void; +}): ColumnDef[] => [ + { + accessorKey: "name", + header: () => ( + + ), + cell: ({ row }) => { + const connection = row.original; + const codeHostIcon = getCodeHostIcon(connection.connectionType); + + return ( +
+ {`${connection.connectionType} + + {connection.name} + + +
+ ); + }, + }, + { + accessorKey: "syncedAt", + header: () => ( + + ), + cell: ({ row }) => row.original.syncedAt + ? + : "-", + }, + { + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ +
+ ), + }, +]; + +type ConnectionsTableProps = { + data: Connection[]; + currentPage: number; + pageSize: number; + totalCount: number; + sortBy: SortBy; + sortOrder: SortOrder; +}; + +export const ConnectionsTable = ({ + data, + currentPage, + pageSize, + totalCount, + sortBy, + sortOrder, +}: ConnectionsTableProps) => { + const pathname = usePathname(); + const router = useRouter(); + const searchParams = useSearchParams(); + const searchParamsString = searchParams.toString(); + const urlSearchValue = searchParams.get("search") ?? ""; + const [searchValue, setSearchValue] = useState(urlSearchValue); + const [scheduledSyncJobs, setScheduledSyncJobs] = useState< + Map> + >(() => new Map()); + const debouncedSearchValue = useDebounce(searchValue, 300); + const [isSearchNavigationPending, startSearchTransition] = useTransition(); + const pendingSearchValuesRef = useRef>(new Set()); + const searchInputRef = useRef(null); + const isSearchPending = searchValue !== debouncedSearchValue + || isSearchNavigationPending; + + useHotkeys("/", (event) => { + event.preventDefault(); + searchInputRef.current?.focus(); + }); + + useEffect(() => { + if (pendingSearchValuesRef.current.delete(urlSearchValue)) { + return; + } + + setSearchValue(urlSearchValue); + }, [urlSearchValue]); + + useEffect(() => { + if (debouncedSearchValue !== searchValue) { + return; + } + + const nextSearchValue = debouncedSearchValue.trim(); + if (nextSearchValue === urlSearchValue) { + return; + } + + const params = new URLSearchParams(searchParamsString); + if (nextSearchValue) { + params.set("search", nextSearchValue); + } else { + params.delete("search"); + } + params.delete("page"); + + const nextSearchParamsString = params.toString(); + if (nextSearchParamsString === searchParamsString) { + return; + } + + pendingSearchValuesRef.current.add(nextSearchValue); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }, [ + debouncedSearchValue, + pathname, + router, + searchParamsString, + searchValue, + urlSearchValue, + ]); + + const onSyncScheduled = useCallback(( + connectionId: number, + jobId: string, + ) => { + setScheduledSyncJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + nextJobs.set(connectionId, { + id: jobId, + data: { connectionId }, + status: "PENDING", + errorMessage: null, + result: null, + }); + return nextJobs; + }); + }, []); + const syncAwareData = useMemo( + () => data.map((connection) => { + const scheduledJob = scheduledSyncJobs.get(connection.id); + return scheduledJob + ? { ...connection, latestJob: scheduledJob } + : connection; + }), + [data, scheduledSyncJobs], + ); + const pollingTargets = useMemo( + () => syncAwareData.flatMap((connection) => { + const latestJob = connection.latestJob; + if ( + !latestJob + || getConnectionSyncAnnotation( + connection.id, + latestJob, + ) !== "SYNCING" + ) { + return []; + } + + return [{ connectionId: connection.id, jobId: latestJob.id }]; + }), + [syncAwareData], + ); + const pollingConnectionIds = useMemo( + () => pollingTargets.map(({ connectionId }) => connectionId), + [pollingTargets], + ); + const pollingKey = useMemo( + () => pollingTargets.map(({ connectionId, jobId }) => + `${connectionId}:${jobId}` + ), + [pollingTargets], + ); + const { data: polledStatuses } = useQuery({ + queryKey: [ + "connectionsv2-sync-status", + pollingConnectionIds, + pollingKey, + ], + queryFn: ({ signal }) => + getConnectionSyncStatuses(pollingConnectionIds, signal), + enabled: pollingConnectionIds.length > 0, + placeholderData: (previousData) => previousData, + refetchInterval: (query) => { + const statuses = query.state.data?.connections; + if (!statuses) { + return POLL_INTERVAL_MS; + } + + return pollingTargets.some((target) => { + const status = statuses.find( + ({ connectionId }) => + connectionId === target.connectionId, + ); + if (!status || status.latestJob?.id !== target.jobId) { + return true; + } + + return status.latestJob.status === "PENDING" + || status.latestJob.status === "IN_PROGRESS"; + }) + ? POLL_INTERVAL_MS + : false; + }, + }); + const completedDuringPollingConnectionIds = useMemo(() => { + const pollingTargetsByConnectionId = new Map( + pollingTargets.map((target) => [target.connectionId, target]), + ); + return new Set( + polledStatuses?.connections.flatMap((status) => { + const target = pollingTargetsByConnectionId.get( + status.connectionId, + ); + return target + && status.latestJob?.id === target.jobId + && status.latestJob.status === "COMPLETED" + && status.syncedAt + ? [status.connectionId] + : []; + }) ?? [], + ); + }, [polledStatuses, pollingTargets]); + const displayedData = useMemo(() => { + const statusesByConnectionId = new Map( + polledStatuses?.connections.map((status) => [ + status.connectionId, + status, + ]) ?? [], + ); + + return syncAwareData.map((connection): DisplayedConnection => { + const showCompleted = completedDuringPollingConnectionIds.has( + connection.id, + ); + const status = statusesByConnectionId.get(connection.id); + const scheduledJob = scheduledSyncJobs.get(connection.id); + if (!status) { + return { ...connection, showCompleted }; + } + if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { + return { + ...connection, + syncedAt: status.syncedAt + ? new Date(status.syncedAt) + : connection.syncedAt, + showCompleted, + }; + } + + return { + ...connection, + syncedAt: status.syncedAt + ? new Date(status.syncedAt) + : null, + latestJob: status.latestJob, + showCompleted, + }; + }); + }, [ + completedDuringPollingConnectionIds, + polledStatuses, + scheduledSyncJobs, + syncAwareData, + ]); + + const onSortChange = useCallback((column: SortBy) => { + const params = new URLSearchParams(searchParams.toString()); + const nextSortOrder = sortBy === column && sortOrder === "asc" + ? "desc" + : "asc"; + params.delete("page"); + if (column === "name") { + params.delete("sortBy"); + } else { + params.set("sortBy", column); + } + if (nextSortOrder === "asc") { + params.delete("sortOrder"); + } else { + params.set("sortOrder", nextSortOrder); + } + + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }, [pathname, router, searchParams, sortBy, sortOrder]); + const columns = useMemo( + () => getColumns({ + sortBy, + sortOrder, + onSortChange, + onSyncScheduled, + }), + [onSortChange, onSyncScheduled, sortBy, sortOrder], + ); + const table = useReactTable({ + data: displayedData, + columns, + getCoreRowModel: getCoreRowModel(), + manualPagination: true, + manualSorting: true, + rowCount: totalCount, + state: { + pagination: { + pageIndex: currentPage - 1, + pageSize, + }, + sorting: [{ id: sortBy, desc: sortOrder === "desc" }], + }, + }); + const totalPages = Math.max(1, table.getPageCount()); + const firstVisibleConnection = totalCount === 0 + ? 0 + : (currentPage - 1) * pageSize + 1; + const lastVisibleConnection = Math.min(currentPage * pageSize, totalCount); + const hasActiveSearch = urlSearchValue.trim().length > 0; + + const goToPage = (page: number) => { + const params = new URLSearchParams(searchParams.toString()); + if (page === 1) { + params.delete("page"); + } else { + params.set("page", page.toString()); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }; + + const clearSearch = () => { + const params = new URLSearchParams(searchParamsString); + params.delete("search"); + params.delete("page"); + + const nextSearchParamsString = params.toString(); + setSearchValue(""); + pendingSearchValuesRef.current.add(""); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }; + + return ( +
+
+ + + + + setSearchValue(event.target.value)} + placeholder="Search connections..." + /> + {isSearchPending && ( + + + + )} + +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} + + ))} + + + {table.getRowModel().rows.length > 0 ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + ))} + + )) + ) : ( + + +
+

No connections found.

+ {hasActiveSearch && ( + + )} +
+ + + )} + +
+
+ {totalCount > 0 && ( +
+

+ Showing {firstVisibleConnection}-{lastVisibleConnection} of {totalCount} +

+
+

+ Page {currentPage} of {totalPages} +

+
+ + +
+
+
+ )} +
+ ); +}; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx b/packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx new file mode 100644 index 000000000..9dfbb1a11 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx @@ -0,0 +1,169 @@ +import { Badge } from "@/components/ui/badge"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { Check, Loader2 } from "lucide-react"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; +import { useEffect, useState } from "react"; +import { SyncIssuePopover } from "./syncIssuePopover"; + +const COMPLETED_BADGE_VISIBLE_MS = 5_000; + +type SyncAnnotationProps = { + connectionId: number; + latestJob: WorkloadJob<"connection-sync"> | null; + showCompleted: boolean; + connectionName: string; + syncedAt: Date | null; + onRetryScheduled: (connectionId: number, jobId: string) => void; +}; + +export type ConnectionSyncAnnotation = + | "SYNCING" + | "WARNING" + | "FAILED" + | null; + +export const getConnectionSyncAnnotation = ( + connectionId: number, + latestJob: WorkloadJob<"connection-sync"> | null, +): ConnectionSyncAnnotation => { + if (!latestJob || latestJob.data.connectionId !== connectionId) { + return null; + } + + if ( + latestJob.status === "PENDING" + || latestJob.status === "IN_PROGRESS" + ) { + return "SYNCING"; + } + + if (latestJob.status === "FAILED") { + return "FAILED"; + } + + if ( + latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ) { + return "WARNING"; + } + + return null; +}; + +export const SyncAnnotation = ({ + connectionId, + latestJob, + showCompleted, + connectionName, + syncedAt, + onRetryScheduled, +}: SyncAnnotationProps) => { + const prefersReducedMotion = useReducedMotion(); + const completionKey = showCompleted + ? latestJob?.id ?? `connection:${connectionId}` + : null; + const [expiredCompletionKey, setExpiredCompletionKey] = useState< + string | null + >(null); + useEffect(() => { + if (!completionKey || expiredCompletionKey === completionKey) { + return; + } + + const timeout = window.setTimeout(() => { + setExpiredCompletionKey(completionKey); + }, COMPLETED_BADGE_VISIBLE_MS); + return () => window.clearTimeout(timeout); + }, [completionKey, expiredCompletionKey]); + const annotation = completionKey !== null + && expiredCompletionKey !== completionKey + ? "COMPLETED" + : getConnectionSyncAnnotation(connectionId, latestJob); + const badge = (() => { + switch (annotation) { + case "COMPLETED": + return ( + + + Completed + + ); + case "SYNCING": + return ( + + + Syncing + + ); + case "FAILED": + return latestJob + ? ( + + ) + : null; + case "WARNING": + return latestJob + ? ( + + ) + : null; + default: + return null; + } + })(); + const isCompleted = annotation === "COMPLETED"; + + return ( + + {annotation && badge && ( + + {badge} + + )} + + ); +}; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx new file mode 100644 index 000000000..a4bc6638f --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx @@ -0,0 +1,272 @@ +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { JobLogsDialog } from "@/app/(app)/components/jobLogsDialog"; +import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter"; +import { useToast } from "@/components/hooks/use-toast"; +import { badgeVariants } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { syncConnection } from "@/features/connections/actions"; +import { cn, isServiceError } from "@/lib/utils"; +import type { + RepositoryDiscoveryIssue, + WorkloadJob, +} from "@sourcebot/shared"; +import { + ChevronDown, + CircleX, + Loader2, + RotateCw, + ScrollText, + TriangleAlert, +} from "lucide-react"; +import { useState } from "react"; + +type SyncIssuePopoverProps = { + connection: { + id: number; + name: string; + syncedAt: Date | null; + }; + latestJob: WorkloadJob<"connection-sync">; + annotation: "WARNING" | "FAILED"; + onRetryScheduled: (connectionId: number, jobId: string) => void; +}; + +const DiscoveryIssues = ({ + issues, +}: { + issues: RepositoryDiscoveryIssue[]; +}) => ( +
+
+ {issues.map((issue, index) => ( +
+ {issue.subject && ( +
+ + {issue.subject.kind} + + + {issue.subject.value} + +
+ )} +

+ {issue.message} +

+
+ ))} +
+
+); + +export const SyncIssuePopover = ({ + connection, + latestJob, + annotation, + onRetryScheduled, +}: SyncIssuePopoverProps) => { + const [isOpen, setIsOpen] = useState(false); + const [isLogsOpen, setIsLogsOpen] = useState(false); + const [isRetrying, setIsRetrying] = useState(false); + const { toast } = useToast(); + const isWarning = annotation === "WARNING"; + const reasons = latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ? latestJob.result.reasons + : null; + if ( + (isWarning && !reasons) + || (!isWarning && latestJob.status !== "FAILED") + ) { + return null; + } + + const label = isWarning ? "Warning" : "Failed"; + const Icon = isWarning ? TriangleAlert : CircleX; + + const retrySync = async () => { + setIsRetrying(true); + + try { + const response = await syncConnection(connection.id); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: response.message, + }); + return; + } + + onRetryScheduled(connection.id, response.jobId); + toast({ + title: "Sync scheduled", + description: `${connection.name} was queued for syncing.`, + }); + setIsOpen(false); + } catch { + toast({ + variant: "destructive", + title: "Failed to retry sync", + description: "An unexpected error occurred while scheduling the sync.", + }); + } finally { + setIsRetrying(false); + } + }; + + return ( + <> + + + + + + + + +

+ {isWarning + ? "View warning details" + : "View failure details"} +

+ + + +
+ +
+
+

+ {isWarning + ? "Connection sync completed with warnings" + : "Connection sync failed"} +

+

+ {isWarning + ? "Sourcebot could not honor the full configured discovery scope. Some repositories may be missing." + : "Sourcebot could not complete the latest sync for this connection."} +

+
+ {isWarning && reasons + ? + : ( +
+

+ Error +

+
+ + {latestJob.errorMessage + ?? "No error details were reported."} + +
+
+ )} +
+ {connection.syncedAt && ( + <> +
+ Last synced +
+
+ +
+ + )} +
Job ID
+
+ {latestJob.id} +
+
+
+ + +
+
+
+ + + + + ); +}; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/layout.tsx b/packages/web/src/app/(app)/settings/connectionsv2/layout.tsx new file mode 100644 index 000000000..feb4a09a8 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/layout.tsx @@ -0,0 +1,10 @@ +import { authenticatedPage } from "@/middleware/authenticatedPage"; +import { OrgRole } from "@sourcebot/db"; +import { SettingsContainer } from "../components/settingsContainer"; + +export default authenticatedPage<{ children: React.ReactNode }>( + async (_auth, { children }) => ( + {children} + ), + { minRole: OrgRole.OWNER, redirectTo: "/settings" }, +); diff --git a/packages/web/src/app/(app)/settings/connectionsv2/page.tsx b/packages/web/src/app/(app)/settings/connectionsv2/page.tsx new file mode 100644 index 000000000..4a918adc8 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/page.tsx @@ -0,0 +1,121 @@ +import { getBullMQClient } from "@/lib/bullmqClient"; +import { authenticatedPage } from "@/middleware/authenticatedPage"; +import { OrgRole, type Prisma } from "@sourcebot/db"; +import { + CONNECTION_QUEUE, + type WorkloadJob, +} from "@sourcebot/shared"; +import Link from "next/link"; +import { z } from "zod"; +import { ConnectionsTable } from "./components/connectionsTable"; + +const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; +const DEFAULT_PAGE_SIZE = 20; +const pageSchema = z.coerce.number().int().positive(); +const sortBySchema = z.enum(["name", "syncedAt"]); +const sortOrderSchema = z.enum(["asc", "desc"]); + +type ConnectionsPageProps = { + searchParams: Promise<{ + page?: string; + search?: string; + sortBy?: string; + sortOrder?: string; + }>; +}; + +export default authenticatedPage(async ( + { org, prisma }, + { searchParams }, +) => { + const params = await searchParams; + const page = pageSchema.safeParse(params.page).data ?? 1; + const search = z.string().optional().safeParse(params.search).data?.trim() + ?? ""; + const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; + const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const skip = (page - 1) * DEFAULT_PAGE_SIZE; + const where: Prisma.ConnectionWhereInput = { + orgId: org.id, + ...(search + ? { + name: { + contains: search, + mode: "insensitive" as const, + }, + } + : {}), + }; + const orderBy: Prisma.ConnectionOrderByWithRelationInput[] = sortBy === "syncedAt" + ? [{ syncedAt: sortOrder }, { id: "asc" }] + : [{ name: sortOrder }, { id: "asc" }]; + + const [connections, totalCount] = await Promise.all([ + prisma.connection.findMany({ + where, + orderBy, + skip, + take: DEFAULT_PAGE_SIZE, + select: { + id: true, + name: true, + connectionType: true, + syncedAt: true, + latestSyncJobId: true, + }, + }), + prisma.connection.count({ where }), + ]); + const latestJobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + let latestJobs = new Map< + string, + WorkloadJob<"connection-sync"> | null + >(); + try { + latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); + } catch (error) { + console.error("Failed to load latest connection sync jobs", error); + } + + return ( +
+
+

Code Host Connections

+

+ Manage connections to external code hosts.{" "} + + Learn more + +

+
+ ({ + id: connection.id, + name: connection.name, + connectionType: connection.connectionType, + syncedAt: connection.syncedAt, + latestJob: connection.latestSyncJobId + ? latestJobs.get(connection.latestSyncJobId) ?? null + : null, + }))} + currentPage={page} + pageSize={DEFAULT_PAGE_SIZE} + totalCount={totalCount} + sortBy={sortBy} + sortOrder={sortOrder} + /> +
+ ); +}, { + minRole: OrgRole.OWNER, + redirectTo: "/settings", +}); diff --git a/packages/web/src/app/(app)/settings/connectionsv2/types.ts b/packages/web/src/app/(app)/settings/connectionsv2/types.ts new file mode 100644 index 000000000..faf6ce997 --- /dev/null +++ b/packages/web/src/app/(app)/settings/connectionsv2/types.ts @@ -0,0 +1,11 @@ +import type { WorkloadJob } from "@sourcebot/shared"; + +export type ConnectionSyncStatus = { + connectionId: number; + syncedAt: string | null; + latestJob: WorkloadJob<"connection-sync"> | null; +}; + +export type ConnectionSyncStatusesResponse = { + connections: ConnectionSyncStatus[]; +}; diff --git a/packages/web/src/app/api/(client)/client.ts b/packages/web/src/app/api/(client)/client.ts index 17326787e..af05cce3f 100644 --- a/packages/web/src/app/api/(client)/client.ts +++ b/packages/web/src/app/api/(client)/client.ts @@ -36,6 +36,7 @@ import type { SearchChatShareableMembersQueryParams, SearchChatShareableMembersResponse, } from "../(server)/ee/chat/[chatId]/searchMembers/route"; +import type { JobLogs, QueueName } from "@sourcebot/shared"; import type { OffersResponse } from "@sourcebot/shared/client"; import { ConnectMcpResponse } from "../(server)/ee/askmcp/connect/types"; import type { GetMcpServersResponse } from "../(server)/ee/askmcp/servers/route"; @@ -395,3 +396,25 @@ export const getMcpServerToolPermissions = async (serverId: string): Promise => { + const response = await fetch("/api/job-logs", { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Sourcebot-Client-Source": "sourcebot-web-client", + }, + body: JSON.stringify({ queue, jobId }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load job logs"); + } + + return response.json() as Promise; +}; diff --git a/packages/web/src/app/api/(server)/connection-sync-status/route.ts b/packages/web/src/app/api/(server)/connection-sync-status/route.ts new file mode 100644 index 000000000..cb77ef583 --- /dev/null +++ b/packages/web/src/app/api/(server)/connection-sync-status/route.ts @@ -0,0 +1,63 @@ +import type { ConnectionSyncStatusesResponse } from "@/app/(app)/settings/connectionsv2/types"; +import { apiHandler } from "@/lib/apiHandler"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + requestBodySchemaValidationError, + serviceErrorResponse, +} from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withOptionalAuth } from "@/middleware/withAuth"; +import { CONNECTION_QUEUE } from "@sourcebot/shared"; +import { z } from "zod"; + +const requestSchema = z.object({ + connectionIds: z.array(z.number().int().positive()).min(1).max(100), +}); + +export const POST = apiHandler(async (request) => { + const parsed = requestSchema.safeParse( + await request.json().catch(() => null), + ); + if (!parsed.success) { + return serviceErrorResponse( + requestBodySchemaValidationError(parsed.error), + ); + } + + const result = await withOptionalAuth(async ({ org, prisma }) => { + const connections = await prisma.connection.findMany({ + where: { + orgId: org.id, + id: { in: parsed.data.connectionIds }, + }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + }, + }); + const jobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + const jobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + jobIds, + ); + + return { + connections: connections.map((connection) => ({ + connectionId: connection.id, + syncedAt: connection.syncedAt?.toISOString() ?? null, + latestJob: connection.latestSyncJobId + ? jobs.get(connection.latestSyncJobId) ?? null + : null, + })), + } satisfies ConnectionSyncStatusesResponse; + }); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result); +}); diff --git a/packages/web/src/app/api/(server)/job-logs/route.ts b/packages/web/src/app/api/(server)/job-logs/route.ts new file mode 100644 index 000000000..38e9ace3e --- /dev/null +++ b/packages/web/src/app/api/(server)/job-logs/route.ts @@ -0,0 +1,70 @@ +import { apiHandler } from "@/lib/apiHandler"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { + notFound, + requestBodySchemaValidationError, + serviceErrorResponse, +} from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { OrgRole } from "@sourcebot/db"; +import { + QUEUE_SPECS, + type QueueName, + type QueueSpec, +} from "@sourcebot/shared"; +import { z } from "zod"; + +const queueSchema = z.custom( + (value) => + typeof value === "string" && Object.hasOwn(QUEUE_SPECS, value), + "Unsupported queue", +); + +const requestSchema = z.object({ + queue: queueSchema, + jobId: z.string().min(1).max(200), +}); + +const getJobLogs = async ( + spec: QueueSpec, + jobId: string, +) => { + const client = getBullMQClient(); + const job = await client.getJob(spec, jobId); + if (!job) { + return null; + } + + return client.getJobLogs(spec, jobId, { ascending: true }); +}; + +export const POST = apiHandler(async (request) => { + const parsed = requestSchema.safeParse( + await request.json().catch(() => null), + ); + if (!parsed.success) { + return serviceErrorResponse( + requestBodySchemaValidationError(parsed.error), + ); + } + + const result = await withAuth(({ role }) => + withMinimumOrgRole(role, OrgRole.OWNER, () => + getJobLogs( + QUEUE_SPECS[parsed.data.queue], + parsed.data.jobId, + ) + ) + ); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + if (!result) { + return serviceErrorResponse(notFound("Job not found")); + } + + return Response.json(result); +}); From 9435e16d128775bf065d7e13b6a33499b30b414d Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 18:30:31 -0700 Subject: [PATCH 16/34] replace existing connections table & rework what 'warning' means --- .../(app)/settings/connections/[id]/page.tsx | 211 ----- .../components/connectionActionsMenu.tsx | 0 .../components/connectionJobsTable.tsx | 344 ------- .../components/connectionsTable.test.tsx | 109 ++- .../components/connectionsTable.tsx | 836 +++++++++++++----- .../components/syncAnnotation.tsx | 5 +- .../components/syncIssuePopover.tsx | 16 +- .../app/(app)/settings/connections/layout.tsx | 9 +- .../app/(app)/settings/connections/page.tsx | 208 +++-- .../{connectionsv2 => connections}/types.ts | 0 .../components/connectionsTable.tsx | 643 -------------- .../(app)/settings/connectionsv2/layout.tsx | 10 - .../app/(app)/settings/connectionsv2/page.tsx | 121 --- .../(server)/connection-sync-status/route.ts | 2 +- 14 files changed, 884 insertions(+), 1630 deletions(-) delete mode 100644 packages/web/src/app/(app)/settings/connections/[id]/page.tsx rename packages/web/src/app/(app)/settings/{connectionsv2 => connections}/components/connectionActionsMenu.tsx (100%) delete mode 100644 packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx rename packages/web/src/app/(app)/settings/{connectionsv2 => connections}/components/connectionsTable.test.tsx (82%) rename packages/web/src/app/(app)/settings/{connectionsv2 => connections}/components/syncAnnotation.tsx (97%) rename packages/web/src/app/(app)/settings/{connectionsv2 => connections}/components/syncIssuePopover.tsx (95%) rename packages/web/src/app/(app)/settings/{connectionsv2 => connections}/types.ts (100%) delete mode 100644 packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx delete mode 100644 packages/web/src/app/(app)/settings/connectionsv2/layout.tsx delete mode 100644 packages/web/src/app/(app)/settings/connectionsv2/page.tsx diff --git a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx b/packages/web/src/app/(app)/settings/connections/[id]/page.tsx deleted file mode 100644 index edcc61069..000000000 --- a/packages/web/src/app/(app)/settings/connections/[id]/page.tsx +++ /dev/null @@ -1,211 +0,0 @@ -import { sew } from "@/middleware/sew"; -import { BackButton } from "@/app/(app)/components/backButton"; -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; -import { Skeleton } from "@/components/ui/skeleton"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { notFound as notFoundServiceError, ServiceErrorException } from "@/lib/serviceError"; -import { notFound } from "next/navigation"; -import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; -import { AzureDevOpsConnectionConfig, BitbucketConnectionConfig, GenericGitHostConnectionConfig, GerritConnectionConfig, GiteaConnectionConfig, GithubConnectionConfig, GitlabConnectionConfig } from "@sourcebot/schemas/v3/index.type"; -import { env, getConfigSettings } from "@sourcebot/shared"; -import { Info } from "lucide-react"; -import Link from "next/link"; -import { Suspense } from "react"; -import { ConnectionJobsTable } from "../components/connectionJobsTable"; - -interface ConnectionDetailPageProps { - params: Promise<{ - id: string - }> -} - -export default async function ConnectionDetailPage(props: ConnectionDetailPageProps) { - const params = await props.params; - const { id } = params; - - const connectionId = Number.parseInt(id); - if (isNaN(connectionId)) { - return notFound(); - } - - const connection = await getConnectionWithJobs(connectionId); - if (isServiceError(connection)) { - throw new ServiceErrorException(connection); - } - - const configSettings = await getConfigSettings(env.CONFIG_PATH); - - const nextSyncAttempt = (() => { - const latestJob = connection.syncJobs.length > 0 ? connection.syncJobs[0] : null; - if (!latestJob) { - return undefined; - } - - if (latestJob.completedAt) { - return new Date(latestJob.completedAt.getTime() + configSettings.resyncConnectionIntervalMs); - } - - return undefined; - })(); - - // Extracts the code host URL from the connection config. - const codeHostUrl: string = (() => { - const connectionType = connection.connectionType; - switch (connectionType) { - case 'github': { - const config = connection.config as unknown as GithubConnectionConfig; - return config.url ?? 'https://github.com'; - } - case 'gitlab': { - const config = connection.config as unknown as GitlabConnectionConfig; - return config.url ?? 'https://gitlab.com'; - } - case 'gitea': { - const config = connection.config as unknown as GiteaConnectionConfig; - return config.url ?? 'https://gitea.com'; - } - case 'gerrit': { - const config = connection.config as unknown as GerritConnectionConfig; - return config.url; - } - case 'bitbucket': { - const config = connection.config as unknown as BitbucketConnectionConfig; - if (config.deploymentType === 'cloud') { - return config.url ?? 'https://bitbucket.org'; - } else { - return config.url!; - } - } - case 'azuredevops': { - const config = connection.config as unknown as AzureDevOpsConnectionConfig; - return config.url ?? 'https://dev.azure.com'; - } - case 'git': { - const config = connection.config as unknown as GenericGitHostConnectionConfig; - return config.url; - } - } - })(); - - return ( -
- -
-

{connection.name}

- - - {codeHostUrl} - -
- -
- - - - Created - - - - - -

When this connection was first added to Sourcebot

- - - - - - - - - - - - - Last synced - - - - - -

The last time this connection was successfully synced

- - - - - - {connection.syncedAt ? : "Never"} - - - - - - - Scheduled - - - - - -

When the connection will be resynced next. Modifying the config will also trigger a resync.

- - - - - - {nextSyncAttempt ? : "-"} - - -
- - - - Sync History - History of all sync jobs for this connection. - - - }> - - - - -
- ) -} - -const getConnectionWithJobs = async (id: number) => sew(() => - withAuth(async ({ prisma, org }) => { - const connection = await prisma.connection.findUnique({ - where: { - id, - orgId: org.id, - }, - include: { - syncJobs: { - orderBy: { - createdAt: 'desc', - }, - }, - }, - }); - - if (!connection) { - return notFoundServiceError(); - } - - return connection; - }) -) \ No newline at end of file diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionActionsMenu.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionActionsMenu.tsx similarity index 100% rename from packages/web/src/app/(app)/settings/connectionsv2/components/connectionActionsMenu.tsx rename to packages/web/src/app/(app)/settings/connections/components/connectionActionsMenu.tsx diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx deleted file mode 100644 index 0600a5c37..000000000 --- a/packages/web/src/app/(app)/settings/connections/components/connectionJobsTable.tsx +++ /dev/null @@ -1,344 +0,0 @@ -"use client" - -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" -import { - type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, - flexRender, - getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { AlertCircle, AlertTriangle, ArrowUpDown, PlusCircleIcon, RefreshCwIcon } from "lucide-react" -import * as React from "react" -import { CopyIconButton } from "@/app/(app)/components/copyIconButton" -import { useMemo } from "react" -import { LightweightCodeHighlighter } from "@/app/(app)/components/lightweightCodeHighlighter" -import { useRouter } from "next/navigation" -import { useToast } from "@/components/hooks/use-toast" -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { LoadingButton } from "@/components/ui/loading-button" -import { syncConnection } from "@/features/connections/actions" -import { isServiceError } from "@/lib/utils" - - -export type ConnectionSyncJob = { - id: string - status: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" - createdAt: Date - updatedAt: Date - completedAt: Date | null - errorMessage: string | null - warningMessages: string[] -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) - -const getStatusBadge = (status: ConnectionSyncJob["status"]) => { - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", - } - - return {labels[status]} -} - -const getDuration = (start: Date, end: Date | null) => { - if (!end) return "-" - const diff = end.getTime() - start.getTime() - const minutes = Math.floor(diff / 60000) - const seconds = Math.floor((diff % 60000) / 1000) - return `${minutes}m ${seconds}s` -} - -export const columns: ColumnDef[] = [ - { - accessorKey: "status", - header: "Status", - cell: ({ row }) => { - const job = row.original - return ( -
- {getStatusBadge(row.getValue("status"))} - {job.errorMessage ? ( - - - - - - - - {job.errorMessage} - - - - - ) : job.warningMessages.length > 0 ? ( - - - - - - -

{job.warningMessages.length} warning(s) while syncing:

-
- {job.warningMessages.map((warning, index) => ( -
- {index + 1}. - {warning} -
- ))} -
- - - - ) : null} -
- ) - }, - filterFn: (row, id, value) => { - return value.includes(row.getValue(id)) - }, - }, - { - accessorKey: "createdAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => , - }, - { - accessorKey: "completedAt", - header: ({ column }) => { - return ( - - ) - }, - cell: ({ row }) => { - const completedAt = row.getValue("completedAt") as Date | null; - if (!completedAt) { - return "-"; - } - - return - }, - }, - { - id: "duration", - header: "Duration", - cell: ({ row }) => { - const job = row.original - return getDuration(job.createdAt, job.completedAt) - }, - }, - { - accessorKey: "id", - header: "Job ID", - cell: ({ row }) => { - const id = row.getValue("id") as string - return ( -
- {id} - { - navigator.clipboard.writeText(id); - return true; - }} /> -
- ) - }, - }, -] - -export const ConnectionJobsTable = ({ data, connectionId }: { data: ConnectionSyncJob[], connectionId: number }) => { - const [sorting, setSorting] = React.useState([{ id: "createdAt", desc: true }]) - const [columnFilters, setColumnFilters] = React.useState([]) - const [columnVisibility, setColumnVisibility] = React.useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const [isSyncSubmitting, setIsSyncSubmitting] = React.useState(false); - const onSyncButtonClick = React.useCallback(async () => { - setIsSyncSubmitting(true); - const response = await syncConnection(connectionId); - - if (!isServiceError(response)) { - const { jobId } = response; - toast({ - description: `✅ Connection sync scheduled. Job ID: ${jobId}`, - }) - router.refresh(); - } else { - toast({ - description: `❌ Failed to sync connection. ${response.message}`, - }); - } - - setIsSyncSubmitting(false); - }, [connectionId, router, toast]); - - const table = useReactTable({ - data, - columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, - getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - state: { - sorting, - columnFilters, - columnVisibility, - }, - }) - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - } = useMemo(() => { - return { - numCompleted: data.filter((job) => job.status === "COMPLETED").length, - numInProgress: data.filter((job) => job.status === "IN_PROGRESS").length, - numPending: data.filter((job) => job.status === "PENDING").length, - numFailed: data.filter((job) => job.status === "FAILED").length, - }; - }, [data]); - - return ( -
-
- - -
- - - - - Trigger sync - -
-
- -
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} - - ))} - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} - ))} - - )) - ) : ( - - - No sync jobs found. - - - )} - -
-
- -
-
- {table.getFilteredRowModel().rows.length} job(s) total -
-
- - -
-
-
- ) -} diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx similarity index 82% rename from packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx rename to packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx index 217ccbb6a..698a6301e 100644 --- a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.test.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx @@ -27,7 +27,7 @@ const connectionActions = vi.hoisted(() => ({ const toast = vi.hoisted(() => vi.fn()); vi.mock("next/navigation", () => ({ - usePathname: () => "/settings/connectionsv2", + usePathname: () => "/settings/connections", useRouter: () => navigation, useSearchParams: () => new URLSearchParams(navigation.searchParams), })); @@ -110,6 +110,47 @@ describe("ConnectionsTable", () => { ).toBeNull(); }); + test("reflects the status filter from the URL", () => { + navigation.searchParams = "status=failed"; + + renderTable(); + + expect( + screen + .getByRole("combobox", { + name: "Filter connections by status", + }) + .textContent, + ).toContain("Failed"); + }); + + test("renders a status-specific empty state", () => { + navigation.searchParams = "status=warning"; + + renderTable({ data: [] }); + + expect(screen.getByText("No connections with warnings.")).toBeTruthy(); + expect(screen.queryByText("Page 1 of 1")).toBeNull(); + }); + + test("clears search and status filters from the empty state", () => { + navigation.searchParams = + "search=missing&status=failed&page=2&sortBy=syncedAt"; + + renderTable({ data: [] }); + + fireEvent.click(screen.getByRole("button", { name: "Clear filters" })); + + expect( + (screen.getByPlaceholderText("Search connections...") as HTMLInputElement) + .value, + ).toBe(""); + expect(navigation.replace).toHaveBeenCalledWith( + "/settings/connections?sortBy=syncedAt", + { scroll: false }, + ); + }); + test("uses URL-driven server pagination", () => { renderTable({ totalCount: 29 }); @@ -117,7 +158,7 @@ describe("ConnectionsTable", () => { fireEvent.click(screen.getByRole("button", { name: "Next" })); expect(navigation.push).toHaveBeenCalledWith( - "/settings/connectionsv2?page=2", + "/settings/connections?page=2", ); }); @@ -127,7 +168,7 @@ describe("ConnectionsTable", () => { fireEvent.click(screen.getByRole("button", { name: "Sort by Name" })); expect(navigation.push).toHaveBeenCalledWith( - "/settings/connectionsv2?sortOrder=desc", + "/settings/connections?sortOrder=desc", ); }); @@ -271,7 +312,6 @@ describe("ConnectionsTable", () => { test.each([ ["PENDING", "Syncing"], ["IN_PROGRESS", "Syncing"], - ["FAILED", "Failed"], ] as const)("renders a %s sync annotation", (status, label) => { renderTable({ data: [{ @@ -280,7 +320,7 @@ describe("ConnectionsTable", () => { id: "job-1", data: { connectionId: connections[0].id }, status, - errorMessage: status === "FAILED" ? "Sync failed" : null, + errorMessage: null, result: null, }, }], @@ -289,12 +329,58 @@ describe("ConnectionsTable", () => { expect(screen.getByText(label)).toBeTruthy(); }); - test("shows structured discovery issues for a partial success", () => { + test("renders a failed annotation when the connection has never synced", () => { renderTable({ data: [{ ...connections[0], + syncedAt: null, latestJob: { - id: "job-1", + id: "failed-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "Authentication failed", + result: null, + }, + }], + }); + + const failed = screen.getByText("Failed"); + expect(failed.closest("td")).toBe( + screen.getByText("Primary GitHub").closest("td"), + ); + }); + + test("renders a warning when a previously synced connection fails", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "warning-job", + data: { connectionId: connections[0].id }, + status: "FAILED", + errorMessage: "Authentication failed", + result: null, + }, + }], + }); + + expect(screen.getByText("Warning")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { + name: "View warning details for Primary GitHub", + })); + + expect(screen.getByText("Latest connection sync failed")).toBeTruthy(); + expect(screen.getByText(/may be stale/)).toBeTruthy(); + expect(screen.getByText("Authentication failed")).toBeTruthy(); + expect(screen.getByText("warning-job")).toBeTruthy(); + }); + + test("renders structured warnings for a partial success", () => { + renderTable({ + data: [{ + ...connections[0], + latestJob: { + id: "partial-job", data: { connectionId: connections[0].id }, status: "COMPLETED", errorMessage: null, @@ -314,11 +400,6 @@ describe("ConnectionsTable", () => { }], }); - const warning = screen.getByText("Warning"); - expect(warning).toBeTruthy(); - expect(warning.closest("td")).toBe( - screen.getByText("Primary GitHub").closest("td"), - ); fireEvent.click(screen.getByRole("button", { name: "View warning details for Primary GitHub", })); @@ -329,7 +410,7 @@ describe("ConnectionsTable", () => { expect(screen.getByText("repository")).toBeTruthy(); expect(screen.getByText("acme/private")).toBeTruthy(); expect(screen.getByText("Not found or inaccessible")).toBeTruthy(); - expect(screen.getByText("job-1")).toBeTruthy(); + expect(screen.getByText("partial-job")).toBeTruthy(); }); test("shows the worker error for a failed sync and supports retry", async () => { @@ -339,6 +420,7 @@ describe("ConnectionsTable", () => { renderTable({ data: [{ ...connections[0], + syncedAt: null, latestJob: { id: "failed-job", data: { connectionId: connections[0].id }, @@ -402,6 +484,7 @@ describe("ConnectionsTable", () => { renderTable({ data: [{ ...connections[0], + syncedAt: null, latestJob: { id: "failed-job", data: { connectionId: connections[0].id }, diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx index e52e7ef1e..6880a5fc2 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx @@ -1,294 +1,702 @@ -"use client" - -import { DisplayDate } from "@/app/(app)/components/DisplayDate" -import { NotificationDot } from "@/app/(app)/components/notificationDot" -import { useToast } from "@/components/hooks/use-toast" -import { Badge } from "@/components/ui/badge" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table" -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip" -import { getCodeHostIcon } from "@/lib/utils" -import { ConnectionType } from "@sourcebot/db" +"use client"; + +import { DisplayDate } from "@/app/(app)/components/DisplayDate"; +import { Button } from "@/components/ui/button"; +import { + InputGroup, + InputGroupAddon, + InputGroupInput, +} from "@/components/ui/input-group"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { cn, getCodeHostIcon } from "@/lib/utils"; +import type { ConnectionType } from "@sourcebot/db"; +import type { WorkloadJob } from "@sourcebot/shared"; +import { useQuery } from "@tanstack/react-query"; +import { useDebounce } from "@uidotdev/usehooks"; import { type ColumnDef, - type ColumnFiltersState, - type SortingState, - type VisibilityState, flexRender, getCoreRowModel, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, useReactTable, -} from "@tanstack/react-table" -import { cva } from "class-variance-authority" -import { ArrowUpDown, RefreshCwIcon } from "lucide-react" -import Image from "next/image" -import Link from "next/link" -import { useRouter } from "next/navigation" -import { useMemo, useState } from "react" +} from "@tanstack/react-table"; +import { ArrowDown, ArrowUp, Loader2, Search } from "lucide-react"; +import Image from "next/image"; +import { usePathname, useRouter, useSearchParams } from "next/navigation"; +import { + useCallback, + useEffect, + useMemo, + useRef, + useState, + useTransition, +} from "react"; +import { useHotkeys } from "react-hotkeys-hook"; +import type { ConnectionSyncStatusesResponse } from "../types"; +import { ConnectionActionsMenu } from "./connectionActionsMenu"; +import { + getConnectionSyncAnnotation, + SyncAnnotation, +} from "./syncAnnotation"; +const POLL_INTERVAL_MS = 5_000; export type Connection = { - id: number - name: string - syncedAt: Date | null - connectionType: ConnectionType - latestJobStatus: "PENDING" | "IN_PROGRESS" | "COMPLETED" | "FAILED" | null - isFirstTimeSync: boolean -} - -const statusBadgeVariants = cva("", { - variants: { - status: { - PENDING: "bg-secondary text-secondary-foreground hover:bg-secondary/80", - IN_PROGRESS: "bg-primary text-primary-foreground hover:bg-primary/90", - COMPLETED: "bg-green-600 text-white hover:bg-green-700", - FAILED: "bg-destructive text-destructive-foreground hover:bg-destructive/90", - }, - }, -}) + id: number; + name: string; + connectionType: ConnectionType; + syncedAt: Date | null; + latestJob: WorkloadJob<"connection-sync"> | null; +}; + +type DisplayedConnection = Connection & { + showCompleted: boolean; +}; + +type SortBy = "name" | "syncedAt"; +type SortOrder = "asc" | "desc"; +type StatusFilter = "all" | "failed" | "warning"; -const getStatusBadge = (status: Connection["latestJobStatus"]) => { - if (!status) { - return "-"; +const getStatusFilter = (value: string | null): StatusFilter => { + if (value === "failed" || value === "warning") { + return value; } - const labels = { - PENDING: "Pending", - IN_PROGRESS: "In Progress", - COMPLETED: "Completed", - FAILED: "Failed", + return "all"; +}; + +const getConnectionSyncStatuses = async ( + connectionIds: number[], + signal: AbortSignal, +): Promise => { + const response = await fetch("/api/connection-sync-status", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ connectionIds }), + signal, + }); + + if (!response.ok) { + throw new Error("Failed to load connection sync statuses"); } - return {labels[status]} -} + return response.json() as Promise; +}; -export const columns: ColumnDef[] = [ +const SortableHeader = ({ + label, + column, + sortBy, + sortOrder, + onSortChange, +}: { + label: string; + column: SortBy; + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; +}) => { + const isActive = sortBy === column; + const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; + + return ( + + ); +}; + +const getColumns = ({ + sortBy, + sortOrder, + onSortChange, + onSyncScheduled, +}: { + sortBy: SortBy; + sortOrder: SortOrder; + onSortChange: (column: SortBy) => void; + onSyncScheduled: (connectionId: number, jobId: string) => void; +}): ColumnDef[] => [ { accessorKey: "name", - size: 400, - header: ({ column }) => { - return ( - - ) - }, + header: () => ( + + ), cell: ({ row }) => { const connection = row.original; const codeHostIcon = getCodeHostIcon(connection.connectionType); return ( -
+
{`${connection.connectionType} - + {connection.name} - - {connection.isFirstTimeSync && ( - - - - - - - - This is the first time Sourcebot is syncing this connection. It may take a few minutes to complete. - - - )} + +
- ) + ); }, }, { - accessorKey: "latestJobStatus", - size: 150, - header: "Lastest status", - cell: ({ row }) => getStatusBadge(row.getValue("latestJobStatus")), + accessorKey: "syncedAt", + header: () => ( + + ), + cell: ({ row }) => row.original.syncedAt + ? + : "-", }, { - accessorKey: "syncedAt", - size: 200, - header: ({ column }) => { - return ( - - ) + id: "actions", + header: () => Actions, + cell: ({ row }) => ( +
+ +
+ ), + }, +]; + +type ConnectionsTableProps = { + data: Connection[]; + currentPage: number; + pageSize: number; + totalCount: number; + sortBy: SortBy; + sortOrder: SortOrder; +}; + +export const ConnectionsTable = ({ + data, + currentPage, + pageSize, + totalCount, + sortBy, + sortOrder, +}: ConnectionsTableProps) => { + const pathname = usePathname(); + const router = useRouter(); + const searchParams = useSearchParams(); + const searchParamsString = searchParams.toString(); + const urlSearchValue = searchParams.get("search") ?? ""; + const statusFilter = getStatusFilter(searchParams.get("status")); + const [searchValue, setSearchValue] = useState(urlSearchValue); + const [scheduledSyncJobs, setScheduledSyncJobs] = useState< + Map> + >(() => new Map()); + const debouncedSearchValue = useDebounce(searchValue, 300); + const [isSearchNavigationPending, startSearchTransition] = useTransition(); + const pendingSearchValuesRef = useRef>(new Set()); + const searchInputRef = useRef(null); + const isSearchPending = searchValue !== debouncedSearchValue + || isSearchNavigationPending; + + useHotkeys("/", (event) => { + event.preventDefault(); + searchInputRef.current?.focus(); + }); + + useEffect(() => { + if (pendingSearchValuesRef.current.delete(urlSearchValue)) { + return; + } + + setSearchValue(urlSearchValue); + }, [urlSearchValue]); + + useEffect(() => { + if (debouncedSearchValue !== searchValue) { + return; + } + + const nextSearchValue = debouncedSearchValue.trim(); + if (nextSearchValue === urlSearchValue) { + return; + } + + const params = new URLSearchParams(searchParamsString); + if (nextSearchValue) { + params.set("search", nextSearchValue); + } else { + params.delete("search"); + } + params.delete("page"); + + const nextSearchParamsString = params.toString(); + if (nextSearchParamsString === searchParamsString) { + return; + } + + pendingSearchValuesRef.current.add(nextSearchValue); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }, [ + debouncedSearchValue, + pathname, + router, + searchParamsString, + searchValue, + urlSearchValue, + ]); + + const onSyncScheduled = useCallback(( + connectionId: number, + jobId: string, + ) => { + setScheduledSyncJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + nextJobs.set(connectionId, { + id: jobId, + data: { connectionId }, + status: "PENDING", + errorMessage: null, + result: null, + }); + return nextJobs; + }); + }, []); + const syncAwareData = useMemo( + () => data.map((connection) => { + const scheduledJob = scheduledSyncJobs.get(connection.id); + return scheduledJob + ? { ...connection, latestJob: scheduledJob } + : connection; + }), + [data, scheduledSyncJobs], + ); + const pollingTargets = useMemo( + () => syncAwareData.flatMap((connection) => { + const latestJob = connection.latestJob; + if ( + !latestJob + || getConnectionSyncAnnotation( + connection.id, + latestJob, + connection.syncedAt, + ) !== "SYNCING" + ) { + return []; + } + + return [{ connectionId: connection.id, jobId: latestJob.id }]; + }), + [syncAwareData], + ); + const pollingConnectionIds = useMemo( + () => pollingTargets.map(({ connectionId }) => connectionId), + [pollingTargets], + ); + const pollingKey = useMemo( + () => pollingTargets.map(({ connectionId, jobId }) => + `${connectionId}:${jobId}` + ), + [pollingTargets], + ); + const { data: polledStatuses } = useQuery({ + queryKey: [ + "connections-sync-status", + pollingConnectionIds, + pollingKey, + ], + queryFn: ({ signal }) => + getConnectionSyncStatuses(pollingConnectionIds, signal), + enabled: pollingConnectionIds.length > 0, + placeholderData: (previousData) => previousData, + refetchInterval: (query) => { + const statuses = query.state.data?.connections; + if (!statuses) { + return POLL_INTERVAL_MS; + } + + return pollingTargets.some((target) => { + const status = statuses.find( + ({ connectionId }) => + connectionId === target.connectionId, + ); + if (!status || status.latestJob?.id !== target.jobId) { + return true; + } + + return status.latestJob.status === "PENDING" + || status.latestJob.status === "IN_PROGRESS"; + }) + ? POLL_INTERVAL_MS + : false; }, - cell: ({ row }) => { - const syncedAt = row.getValue("syncedAt") as Date | null; - if (!syncedAt) { - return "-"; + }); + const completedDuringPollingConnectionIds = useMemo(() => { + const pollingTargetsByConnectionId = new Map( + pollingTargets.map((target) => [target.connectionId, target]), + ); + return new Set( + polledStatuses?.connections.flatMap((status) => { + const target = pollingTargetsByConnectionId.get( + status.connectionId, + ); + return target + && status.latestJob?.id === target.jobId + && status.latestJob.status === "COMPLETED" + && status.syncedAt + ? [status.connectionId] + : []; + }) ?? [], + ); + }, [polledStatuses, pollingTargets]); + const displayedData = useMemo(() => { + const statusesByConnectionId = new Map( + polledStatuses?.connections.map((status) => [ + status.connectionId, + status, + ]) ?? [], + ); + + return syncAwareData.map((connection): DisplayedConnection => { + const showCompleted = completedDuringPollingConnectionIds.has( + connection.id, + ); + const status = statusesByConnectionId.get(connection.id); + const scheduledJob = scheduledSyncJobs.get(connection.id); + if (!status) { + return { ...connection, showCompleted }; + } + if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { + return { + ...connection, + syncedAt: status.syncedAt + ? new Date(status.syncedAt) + : connection.syncedAt, + showCompleted, + }; } - return ( - - ) - } - }, -] + return { + ...connection, + syncedAt: status.syncedAt + ? new Date(status.syncedAt) + : null, + latestJob: status.latestJob, + showCompleted, + }; + }); + }, [ + completedDuringPollingConnectionIds, + polledStatuses, + scheduledSyncJobs, + syncAwareData, + ]); -export const ConnectionsTable = ({ data }: { data: Connection[] }) => { - const [sorting, setSorting] = useState([]) - const [columnFilters, setColumnFilters] = useState([]) - const [columnVisibility, setColumnVisibility] = useState({}) - const [rowSelection, setRowSelection] = useState({}) - const router = useRouter(); - const { toast } = useToast(); - - const { - numCompleted, - numInProgress, - numPending, - numFailed, - numNoJobs, - } = useMemo(() => { - return { - numCompleted: data.filter((connection) => connection.latestJobStatus === "COMPLETED").length, - numInProgress: data.filter((connection) => connection.latestJobStatus === "IN_PROGRESS").length, - numPending: data.filter((connection) => connection.latestJobStatus === "PENDING").length, - numFailed: data.filter((connection) => connection.latestJobStatus === "FAILED").length, - numNoJobs: data.filter((connection) => connection.latestJobStatus === null).length, + const onSortChange = useCallback((column: SortBy) => { + const params = new URLSearchParams(searchParams.toString()); + const nextSortOrder = sortBy === column && sortOrder === "asc" + ? "desc" + : "asc"; + params.delete("page"); + if (column === "name") { + params.delete("sortBy"); + } else { + params.set("sortBy", column); + } + if (nextSortOrder === "asc") { + params.delete("sortOrder"); + } else { + params.set("sortOrder", nextSortOrder); } - }, [data]); + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }, [pathname, router, searchParams, sortBy, sortOrder]); + const columns = useMemo( + () => getColumns({ + sortBy, + sortOrder, + onSortChange, + onSyncScheduled, + }), + [onSortChange, onSyncScheduled, sortBy, sortOrder], + ); const table = useReactTable({ - data, + data: displayedData, columns, - onSortingChange: setSorting, - onColumnFiltersChange: setColumnFilters, getCoreRowModel: getCoreRowModel(), - getPaginationRowModel: getPaginationRowModel(), - getSortedRowModel: getSortedRowModel(), - getFilteredRowModel: getFilteredRowModel(), - onColumnVisibilityChange: setColumnVisibility, - onRowSelectionChange: setRowSelection, - columnResizeMode: 'onChange', - enableColumnResizing: false, + manualPagination: true, + manualSorting: true, + rowCount: totalCount, state: { - sorting, - columnFilters, - columnVisibility, - rowSelection, + pagination: { + pageIndex: currentPage - 1, + pageSize, + }, + sorting: [{ id: sortBy, desc: sortOrder === "desc" }], }, - }) + }); + const totalPages = Math.max(1, table.getPageCount()); + const firstVisibleConnection = totalCount === 0 + ? 0 + : (currentPage - 1) * pageSize + 1; + const lastVisibleConnection = Math.min(currentPage * pageSize, totalCount); + const hasActiveFilters = statusFilter !== "all" + || urlSearchValue.trim().length > 0; + + const goToPage = (page: number) => { + const params = new URLSearchParams(searchParams.toString()); + if (page === 1) { + params.delete("page"); + } else { + params.set("page", page.toString()); + } + const query = params.toString(); + router.push(query ? `${pathname}?${query}` : pathname); + }; + + const onStatusFilterChange = (status: StatusFilter) => { + const params = new URLSearchParams(searchParams.toString()); + if (status === "all") { + params.delete("status"); + } else { + params.set("status", status); + } + params.delete("page"); + + const query = params.toString(); + router.replace(query ? `${pathname}?${query}` : pathname, { + scroll: false, + }); + }; + + const clearFilters = () => { + const params = new URLSearchParams(searchParamsString); + params.delete("search"); + params.delete("status"); + params.delete("page"); + + const nextSearchParamsString = params.toString(); + setSearchValue(""); + pendingSearchValuesRef.current.add(""); + startSearchTransition(() => { + router.replace( + `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, + { scroll: false }, + ); + }); + }; + + const emptyMessage = statusFilter === "failed" + ? "No failed connections." + : statusFilter === "warning" + ? "No connections with warnings." + : "No connections found."; return ( -
-
- table.getColumn("name")?.setFilterValue(event.target.value)} - className="max-w-sm" - /> +
+
+ + + + + setSearchValue(event.target.value)} + placeholder="Search connections..." + /> + {isSearchPending && ( + + + + )} + -
- +
{table.getHeaderGroups().map((headerGroup) => ( - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - - ) - })} + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} ))} - {table.getRowModel().rows?.length ? ( + {table.getRowModel().rows.length > 0 ? ( table.getRowModel().rows.map((row) => ( - + {row.getVisibleCells().map((cell) => ( - {flexRender(cell.column.columnDef.cell, cell.getContext())} + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} ))} )) ) : ( - - No results. + +
+

{emptyMessage}

+ {hasActiveFilters && ( + + )} +
)}
-
-
- {table.getFilteredRowModel().rows.length} {data.length > 1 ? 'connections' : 'connection'} total -
-
- - + {totalCount > 0 && ( +
+

+ Showing {firstVisibleConnection}-{lastVisibleConnection} of {totalCount} +

+
+

+ Page {currentPage} of {totalPages} +

+
+ + +
+
-
+ )}
- ) -} + ); +}; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx b/packages/web/src/app/(app)/settings/connections/components/syncAnnotation.tsx similarity index 97% rename from packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx rename to packages/web/src/app/(app)/settings/connections/components/syncAnnotation.tsx index 9dfbb1a11..6a7fca507 100644 --- a/packages/web/src/app/(app)/settings/connectionsv2/components/syncAnnotation.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/syncAnnotation.tsx @@ -25,6 +25,7 @@ export type ConnectionSyncAnnotation = export const getConnectionSyncAnnotation = ( connectionId: number, latestJob: WorkloadJob<"connection-sync"> | null, + syncedAt: Date | null, ): ConnectionSyncAnnotation => { if (!latestJob || latestJob.data.connectionId !== connectionId) { return null; @@ -38,7 +39,7 @@ export const getConnectionSyncAnnotation = ( } if (latestJob.status === "FAILED") { - return "FAILED"; + return syncedAt ? "WARNING" : "FAILED"; } if ( @@ -79,7 +80,7 @@ export const SyncAnnotation = ({ const annotation = completionKey !== null && expiredCompletionKey !== completionKey ? "COMPLETED" - : getConnectionSyncAnnotation(connectionId, latestJob); + : getConnectionSyncAnnotation(connectionId, latestJob, syncedAt); const badge = (() => { switch (annotation) { case "COMPLETED": diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx similarity index 95% rename from packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx rename to packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx index a4bc6638f..be9cae5be 100644 --- a/packages/web/src/app/(app)/settings/connectionsv2/components/syncIssuePopover.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx @@ -90,8 +90,8 @@ export const SyncIssuePopover = ({ ? latestJob.result.reasons : null; if ( - (isWarning && !reasons) - || (!isWarning && latestJob.status !== "FAILED") + latestJob.status !== "FAILED" + && (!isWarning || !reasons) ) { return null; } @@ -185,17 +185,21 @@ export const SyncIssuePopover = ({

- {isWarning + {reasons ? "Connection sync completed with warnings" - : "Connection sync failed"} + : isWarning + ? "Latest connection sync failed" + : "Connection sync failed"}

- {isWarning + {reasons ? "Sourcebot could not honor the full configured discovery scope. Some repositories may be missing." + : isWarning + ? "Previously discovered repositories remain available, but they may be stale." : "Sourcebot could not complete the latest sync for this connection."}

- {isWarning && reasons + {reasons ? : (
diff --git a/packages/web/src/app/(app)/settings/connections/layout.tsx b/packages/web/src/app/(app)/settings/connections/layout.tsx index c89c498e6..feb4a09a8 100644 --- a/packages/web/src/app/(app)/settings/connections/layout.tsx +++ b/packages/web/src/app/(app)/settings/connections/layout.tsx @@ -2,6 +2,9 @@ import { authenticatedPage } from "@/middleware/authenticatedPage"; import { OrgRole } from "@sourcebot/db"; import { SettingsContainer } from "../components/settingsContainer"; -export default authenticatedPage<{ children: React.ReactNode }>(async (_auth, { children }) => { - return {children}; -}, { minRole: OrgRole.OWNER, redirectTo: '/settings' }); +export default authenticatedPage<{ children: React.ReactNode }>( + async (_auth, { children }) => ( + {children} + ), + { minRole: OrgRole.OWNER, redirectTo: "/settings" }, +); diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index fdce7b08f..a03faee0a 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -1,77 +1,161 @@ -import { sew } from "@/middleware/sew"; -import { ServiceErrorException } from "@/lib/serviceError"; -import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; +import { getBullMQClient } from "@/lib/bullmqClient"; +import { authenticatedPage } from "@/middleware/authenticatedPage"; +import { OrgRole, type Prisma } from "@sourcebot/db"; +import { + CONNECTION_QUEUE, + type WorkloadJob, +} from "@sourcebot/shared"; import Link from "next/link"; +import { z } from "zod"; import { ConnectionsTable } from "./components/connectionsTable"; -import { ConnectionSyncJobStatus } from "@prisma/client"; const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; +const DEFAULT_PAGE_SIZE = 20; +const pageSchema = z.coerce.number().int().positive(); +const sortBySchema = z.enum(["name", "syncedAt"]); +const sortOrderSchema = z.enum(["asc", "desc"]); +const statusSchema = z.enum(["failed", "warning"]); -export default async function ConnectionsPage() { - const _connections = await getConnectionsWithLatestJob(); - if (isServiceError(_connections)) { - throw new ServiceErrorException(_connections); - } +type ConnectionsPageProps = { + searchParams: Promise<{ + page?: string; + search?: string; + status?: string; + sortBy?: string; + sortOrder?: string; + }>; +}; - // Sort connections so that first time syncs are at the top. - const connections = _connections - .map((connection) => ({ - ...connection, - isFirstTimeSync: connection.syncedAt === null && connection.syncJobs.filter((job) => job.status === ConnectionSyncJobStatus.PENDING || job.status === ConnectionSyncJobStatus.IN_PROGRESS).length > 0, - latestJobStatus: connection.syncJobs.length > 0 ? connection.syncJobs[0].status : null, - })) - .sort((a, b) => { - if (a.isFirstTimeSync && !b.isFirstTimeSync) { - return -1; - } - if (!a.isFirstTimeSync && b.isFirstTimeSync) { - return 1; +export default authenticatedPage(async ( + { org, prisma }, + { searchParams }, +) => { + const params = await searchParams; + const page = pageSchema.safeParse(params.page).data ?? 1; + const search = z.string().optional().safeParse(params.search).data?.trim() + ?? ""; + const status = statusSchema.safeParse(params.status).data ?? "all"; + const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; + const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const skip = (page - 1) * DEFAULT_PAGE_SIZE; + const baseWhere: Prisma.ConnectionWhereInput = { + orgId: org.id, + ...(search + ? { + name: { + contains: search, + mode: "insensitive" as const, + }, + } + : {}), + }; + let where = baseWhere; + let latestJobs = new Map< + string, + WorkloadJob<"connection-sync"> | null + >(); + if (status !== "all") { + const candidates = await prisma.connection.findMany({ + where: baseWhere, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + }, + }); + const candidateJobIds = candidates.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + candidateJobIds, + ); + const matchingConnectionIds = candidates.flatMap((connection) => { + if (!connection.latestSyncJobId) { + return []; } - return a.name.localeCompare(b.name); + + const job = latestJobs.get(connection.latestSyncJobId); + const matches = status === "failed" + ? job?.status === "FAILED" && connection.syncedAt === null + : (job?.status === "FAILED" && connection.syncedAt !== null) + || (job?.status === "COMPLETED" + && job.result?.outcome === "PARTIAL_SUCCESS"); + return matches ? [connection.id] : []; }); + where = { + ...baseWhere, + id: { in: matchingConnectionIds }, + }; + } + const orderBy: Prisma.ConnectionOrderByWithRelationInput[] = sortBy === "syncedAt" + ? [{ syncedAt: sortOrder }, { id: "asc" }] + : [{ name: sortOrder }, { id: "asc" }]; + + const [connections, totalCount] = await Promise.all([ + prisma.connection.findMany({ + where, + orderBy, + skip, + take: DEFAULT_PAGE_SIZE, + select: { + id: true, + name: true, + connectionType: true, + syncedAt: true, + latestSyncJobId: true, + }, + }), + prisma.connection.count({ where }), + ]); + const latestJobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + if (status === "all") { + try { + latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); + } catch (error) { + console.error("Failed to load latest connection sync jobs", error); + } + } return (

Code Host Connections

-

Manage your connections to external code hosts. Learn more

+

+ Manage connections to external code hosts.{" "} + + Learn more + +

- ({ - id: connection.id, - name: connection.name, - connectionType: connection.connectionType, - syncedAt: connection.syncedAt, - latestJobStatus: connection.latestJobStatus, - isFirstTimeSync: connection.isFirstTimeSync, - }))} /> + ({ + id: connection.id, + name: connection.name, + connectionType: connection.connectionType, + syncedAt: connection.syncedAt, + latestJob: connection.latestSyncJobId + ? latestJobs.get(connection.latestSyncJobId) ?? null + : null, + }))} + currentPage={page} + pageSize={DEFAULT_PAGE_SIZE} + totalCount={totalCount} + sortBy={sortBy} + sortOrder={sortOrder} + />
- ) -} - -const getConnectionsWithLatestJob = async () => sew(() => - withAuth(async ({ prisma, org }) => { - const connections = await prisma.connection.findMany({ - where: { - orgId: org.id, - }, - include: { - _count: { - select: { - syncJobs: true, - } - }, - syncJobs: { - orderBy: { - createdAt: 'desc' - }, - take: 1 - }, - }, - orderBy: { - name: 'asc' - }, - }); - - return connections; - })); \ No newline at end of file + ); +}, { + minRole: OrgRole.OWNER, + redirectTo: "/settings", +}); diff --git a/packages/web/src/app/(app)/settings/connectionsv2/types.ts b/packages/web/src/app/(app)/settings/connections/types.ts similarity index 100% rename from packages/web/src/app/(app)/settings/connectionsv2/types.ts rename to packages/web/src/app/(app)/settings/connections/types.ts diff --git a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx deleted file mode 100644 index 4536b88a4..000000000 --- a/packages/web/src/app/(app)/settings/connectionsv2/components/connectionsTable.tsx +++ /dev/null @@ -1,643 +0,0 @@ -"use client"; - -import { DisplayDate } from "@/app/(app)/components/DisplayDate"; -import { Button } from "@/components/ui/button"; -import { - InputGroup, - InputGroupAddon, - InputGroupInput, -} from "@/components/ui/input-group"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table"; -import { cn, getCodeHostIcon } from "@/lib/utils"; -import type { ConnectionType } from "@sourcebot/db"; -import type { WorkloadJob } from "@sourcebot/shared"; -import { useQuery } from "@tanstack/react-query"; -import { useDebounce } from "@uidotdev/usehooks"; -import { - type ColumnDef, - flexRender, - getCoreRowModel, - useReactTable, -} from "@tanstack/react-table"; -import { ArrowDown, ArrowUp, Loader2, Search } from "lucide-react"; -import Image from "next/image"; -import { usePathname, useRouter, useSearchParams } from "next/navigation"; -import { - useCallback, - useEffect, - useMemo, - useRef, - useState, - useTransition, -} from "react"; -import { useHotkeys } from "react-hotkeys-hook"; -import type { ConnectionSyncStatusesResponse } from "../types"; -import { ConnectionActionsMenu } from "./connectionActionsMenu"; -import { - getConnectionSyncAnnotation, - SyncAnnotation, -} from "./syncAnnotation"; - -const POLL_INTERVAL_MS = 5_000; - -export type Connection = { - id: number; - name: string; - connectionType: ConnectionType; - syncedAt: Date | null; - latestJob: WorkloadJob<"connection-sync"> | null; -}; - -type DisplayedConnection = Connection & { - showCompleted: boolean; -}; - -type SortBy = "name" | "syncedAt"; -type SortOrder = "asc" | "desc"; - -const getConnectionSyncStatuses = async ( - connectionIds: number[], - signal: AbortSignal, -): Promise => { - const response = await fetch("/api/connection-sync-status", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ connectionIds }), - signal, - }); - - if (!response.ok) { - throw new Error("Failed to load connection sync statuses"); - } - - return response.json() as Promise; -}; - -const SortableHeader = ({ - label, - column, - sortBy, - sortOrder, - onSortChange, -}: { - label: string; - column: SortBy; - sortBy: SortBy; - sortOrder: SortOrder; - onSortChange: (column: SortBy) => void; -}) => { - const isActive = sortBy === column; - const SortIcon = isActive && sortOrder === "desc" ? ArrowUp : ArrowDown; - - return ( - - ); -}; - -const getColumns = ({ - sortBy, - sortOrder, - onSortChange, - onSyncScheduled, -}: { - sortBy: SortBy; - sortOrder: SortOrder; - onSortChange: (column: SortBy) => void; - onSyncScheduled: (connectionId: number, jobId: string) => void; -}): ColumnDef[] => [ - { - accessorKey: "name", - header: () => ( - - ), - cell: ({ row }) => { - const connection = row.original; - const codeHostIcon = getCodeHostIcon(connection.connectionType); - - return ( -
- {`${connection.connectionType} - - {connection.name} - - -
- ); - }, - }, - { - accessorKey: "syncedAt", - header: () => ( - - ), - cell: ({ row }) => row.original.syncedAt - ? - : "-", - }, - { - id: "actions", - header: () => Actions, - cell: ({ row }) => ( -
- -
- ), - }, -]; - -type ConnectionsTableProps = { - data: Connection[]; - currentPage: number; - pageSize: number; - totalCount: number; - sortBy: SortBy; - sortOrder: SortOrder; -}; - -export const ConnectionsTable = ({ - data, - currentPage, - pageSize, - totalCount, - sortBy, - sortOrder, -}: ConnectionsTableProps) => { - const pathname = usePathname(); - const router = useRouter(); - const searchParams = useSearchParams(); - const searchParamsString = searchParams.toString(); - const urlSearchValue = searchParams.get("search") ?? ""; - const [searchValue, setSearchValue] = useState(urlSearchValue); - const [scheduledSyncJobs, setScheduledSyncJobs] = useState< - Map> - >(() => new Map()); - const debouncedSearchValue = useDebounce(searchValue, 300); - const [isSearchNavigationPending, startSearchTransition] = useTransition(); - const pendingSearchValuesRef = useRef>(new Set()); - const searchInputRef = useRef(null); - const isSearchPending = searchValue !== debouncedSearchValue - || isSearchNavigationPending; - - useHotkeys("/", (event) => { - event.preventDefault(); - searchInputRef.current?.focus(); - }); - - useEffect(() => { - if (pendingSearchValuesRef.current.delete(urlSearchValue)) { - return; - } - - setSearchValue(urlSearchValue); - }, [urlSearchValue]); - - useEffect(() => { - if (debouncedSearchValue !== searchValue) { - return; - } - - const nextSearchValue = debouncedSearchValue.trim(); - if (nextSearchValue === urlSearchValue) { - return; - } - - const params = new URLSearchParams(searchParamsString); - if (nextSearchValue) { - params.set("search", nextSearchValue); - } else { - params.delete("search"); - } - params.delete("page"); - - const nextSearchParamsString = params.toString(); - if (nextSearchParamsString === searchParamsString) { - return; - } - - pendingSearchValuesRef.current.add(nextSearchValue); - startSearchTransition(() => { - router.replace( - `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, - { scroll: false }, - ); - }); - }, [ - debouncedSearchValue, - pathname, - router, - searchParamsString, - searchValue, - urlSearchValue, - ]); - - const onSyncScheduled = useCallback(( - connectionId: number, - jobId: string, - ) => { - setScheduledSyncJobs((currentJobs) => { - const nextJobs = new Map(currentJobs); - nextJobs.set(connectionId, { - id: jobId, - data: { connectionId }, - status: "PENDING", - errorMessage: null, - result: null, - }); - return nextJobs; - }); - }, []); - const syncAwareData = useMemo( - () => data.map((connection) => { - const scheduledJob = scheduledSyncJobs.get(connection.id); - return scheduledJob - ? { ...connection, latestJob: scheduledJob } - : connection; - }), - [data, scheduledSyncJobs], - ); - const pollingTargets = useMemo( - () => syncAwareData.flatMap((connection) => { - const latestJob = connection.latestJob; - if ( - !latestJob - || getConnectionSyncAnnotation( - connection.id, - latestJob, - ) !== "SYNCING" - ) { - return []; - } - - return [{ connectionId: connection.id, jobId: latestJob.id }]; - }), - [syncAwareData], - ); - const pollingConnectionIds = useMemo( - () => pollingTargets.map(({ connectionId }) => connectionId), - [pollingTargets], - ); - const pollingKey = useMemo( - () => pollingTargets.map(({ connectionId, jobId }) => - `${connectionId}:${jobId}` - ), - [pollingTargets], - ); - const { data: polledStatuses } = useQuery({ - queryKey: [ - "connectionsv2-sync-status", - pollingConnectionIds, - pollingKey, - ], - queryFn: ({ signal }) => - getConnectionSyncStatuses(pollingConnectionIds, signal), - enabled: pollingConnectionIds.length > 0, - placeholderData: (previousData) => previousData, - refetchInterval: (query) => { - const statuses = query.state.data?.connections; - if (!statuses) { - return POLL_INTERVAL_MS; - } - - return pollingTargets.some((target) => { - const status = statuses.find( - ({ connectionId }) => - connectionId === target.connectionId, - ); - if (!status || status.latestJob?.id !== target.jobId) { - return true; - } - - return status.latestJob.status === "PENDING" - || status.latestJob.status === "IN_PROGRESS"; - }) - ? POLL_INTERVAL_MS - : false; - }, - }); - const completedDuringPollingConnectionIds = useMemo(() => { - const pollingTargetsByConnectionId = new Map( - pollingTargets.map((target) => [target.connectionId, target]), - ); - return new Set( - polledStatuses?.connections.flatMap((status) => { - const target = pollingTargetsByConnectionId.get( - status.connectionId, - ); - return target - && status.latestJob?.id === target.jobId - && status.latestJob.status === "COMPLETED" - && status.syncedAt - ? [status.connectionId] - : []; - }) ?? [], - ); - }, [polledStatuses, pollingTargets]); - const displayedData = useMemo(() => { - const statusesByConnectionId = new Map( - polledStatuses?.connections.map((status) => [ - status.connectionId, - status, - ]) ?? [], - ); - - return syncAwareData.map((connection): DisplayedConnection => { - const showCompleted = completedDuringPollingConnectionIds.has( - connection.id, - ); - const status = statusesByConnectionId.get(connection.id); - const scheduledJob = scheduledSyncJobs.get(connection.id); - if (!status) { - return { ...connection, showCompleted }; - } - if (scheduledJob && status.latestJob?.id !== scheduledJob.id) { - return { - ...connection, - syncedAt: status.syncedAt - ? new Date(status.syncedAt) - : connection.syncedAt, - showCompleted, - }; - } - - return { - ...connection, - syncedAt: status.syncedAt - ? new Date(status.syncedAt) - : null, - latestJob: status.latestJob, - showCompleted, - }; - }); - }, [ - completedDuringPollingConnectionIds, - polledStatuses, - scheduledSyncJobs, - syncAwareData, - ]); - - const onSortChange = useCallback((column: SortBy) => { - const params = new URLSearchParams(searchParams.toString()); - const nextSortOrder = sortBy === column && sortOrder === "asc" - ? "desc" - : "asc"; - params.delete("page"); - if (column === "name") { - params.delete("sortBy"); - } else { - params.set("sortBy", column); - } - if (nextSortOrder === "asc") { - params.delete("sortOrder"); - } else { - params.set("sortOrder", nextSortOrder); - } - - const query = params.toString(); - router.push(query ? `${pathname}?${query}` : pathname); - }, [pathname, router, searchParams, sortBy, sortOrder]); - const columns = useMemo( - () => getColumns({ - sortBy, - sortOrder, - onSortChange, - onSyncScheduled, - }), - [onSortChange, onSyncScheduled, sortBy, sortOrder], - ); - const table = useReactTable({ - data: displayedData, - columns, - getCoreRowModel: getCoreRowModel(), - manualPagination: true, - manualSorting: true, - rowCount: totalCount, - state: { - pagination: { - pageIndex: currentPage - 1, - pageSize, - }, - sorting: [{ id: sortBy, desc: sortOrder === "desc" }], - }, - }); - const totalPages = Math.max(1, table.getPageCount()); - const firstVisibleConnection = totalCount === 0 - ? 0 - : (currentPage - 1) * pageSize + 1; - const lastVisibleConnection = Math.min(currentPage * pageSize, totalCount); - const hasActiveSearch = urlSearchValue.trim().length > 0; - - const goToPage = (page: number) => { - const params = new URLSearchParams(searchParams.toString()); - if (page === 1) { - params.delete("page"); - } else { - params.set("page", page.toString()); - } - const query = params.toString(); - router.push(query ? `${pathname}?${query}` : pathname); - }; - - const clearSearch = () => { - const params = new URLSearchParams(searchParamsString); - params.delete("search"); - params.delete("page"); - - const nextSearchParamsString = params.toString(); - setSearchValue(""); - pendingSearchValuesRef.current.add(""); - startSearchTransition(() => { - router.replace( - `${pathname}${nextSearchParamsString ? `?${nextSearchParamsString}` : ""}`, - { scroll: false }, - ); - }); - }; - - return ( -
-
- - - - - setSearchValue(event.target.value)} - placeholder="Search connections..." - /> - {isSearchPending && ( - - - - )} - -
-
- - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - {header.isPlaceholder - ? null - : flexRender( - header.column.columnDef.header, - header.getContext(), - )} - - ))} - - ))} - - - {table.getRowModel().rows.length > 0 ? ( - table.getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - {flexRender( - cell.column.columnDef.cell, - cell.getContext(), - )} - - ))} - - )) - ) : ( - - -
-

No connections found.

- {hasActiveSearch && ( - - )} -
- - - )} - -
-
- {totalCount > 0 && ( -
-

- Showing {firstVisibleConnection}-{lastVisibleConnection} of {totalCount} -

-
-

- Page {currentPage} of {totalPages} -

-
- - -
-
-
- )} -
- ); -}; diff --git a/packages/web/src/app/(app)/settings/connectionsv2/layout.tsx b/packages/web/src/app/(app)/settings/connectionsv2/layout.tsx deleted file mode 100644 index feb4a09a8..000000000 --- a/packages/web/src/app/(app)/settings/connectionsv2/layout.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { authenticatedPage } from "@/middleware/authenticatedPage"; -import { OrgRole } from "@sourcebot/db"; -import { SettingsContainer } from "../components/settingsContainer"; - -export default authenticatedPage<{ children: React.ReactNode }>( - async (_auth, { children }) => ( - {children} - ), - { minRole: OrgRole.OWNER, redirectTo: "/settings" }, -); diff --git a/packages/web/src/app/(app)/settings/connectionsv2/page.tsx b/packages/web/src/app/(app)/settings/connectionsv2/page.tsx deleted file mode 100644 index 4a918adc8..000000000 --- a/packages/web/src/app/(app)/settings/connectionsv2/page.tsx +++ /dev/null @@ -1,121 +0,0 @@ -import { getBullMQClient } from "@/lib/bullmqClient"; -import { authenticatedPage } from "@/middleware/authenticatedPage"; -import { OrgRole, type Prisma } from "@sourcebot/db"; -import { - CONNECTION_QUEUE, - type WorkloadJob, -} from "@sourcebot/shared"; -import Link from "next/link"; -import { z } from "zod"; -import { ConnectionsTable } from "./components/connectionsTable"; - -const DOCS_URL = "https://docs.sourcebot.dev/docs/connections/indexing-your-code"; -const DEFAULT_PAGE_SIZE = 20; -const pageSchema = z.coerce.number().int().positive(); -const sortBySchema = z.enum(["name", "syncedAt"]); -const sortOrderSchema = z.enum(["asc", "desc"]); - -type ConnectionsPageProps = { - searchParams: Promise<{ - page?: string; - search?: string; - sortBy?: string; - sortOrder?: string; - }>; -}; - -export default authenticatedPage(async ( - { org, prisma }, - { searchParams }, -) => { - const params = await searchParams; - const page = pageSchema.safeParse(params.page).data ?? 1; - const search = z.string().optional().safeParse(params.search).data?.trim() - ?? ""; - const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; - const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; - const skip = (page - 1) * DEFAULT_PAGE_SIZE; - const where: Prisma.ConnectionWhereInput = { - orgId: org.id, - ...(search - ? { - name: { - contains: search, - mode: "insensitive" as const, - }, - } - : {}), - }; - const orderBy: Prisma.ConnectionOrderByWithRelationInput[] = sortBy === "syncedAt" - ? [{ syncedAt: sortOrder }, { id: "asc" }] - : [{ name: sortOrder }, { id: "asc" }]; - - const [connections, totalCount] = await Promise.all([ - prisma.connection.findMany({ - where, - orderBy, - skip, - take: DEFAULT_PAGE_SIZE, - select: { - id: true, - name: true, - connectionType: true, - syncedAt: true, - latestSyncJobId: true, - }, - }), - prisma.connection.count({ where }), - ]); - const latestJobIds = connections.flatMap((connection) => - connection.latestSyncJobId ? [connection.latestSyncJobId] : [] - ); - let latestJobs = new Map< - string, - WorkloadJob<"connection-sync"> | null - >(); - try { - latestJobs = await getBullMQClient().getJobs( - CONNECTION_QUEUE, - latestJobIds, - ); - } catch (error) { - console.error("Failed to load latest connection sync jobs", error); - } - - return ( -
-
-

Code Host Connections

-

- Manage connections to external code hosts.{" "} - - Learn more - -

-
- ({ - id: connection.id, - name: connection.name, - connectionType: connection.connectionType, - syncedAt: connection.syncedAt, - latestJob: connection.latestSyncJobId - ? latestJobs.get(connection.latestSyncJobId) ?? null - : null, - }))} - currentPage={page} - pageSize={DEFAULT_PAGE_SIZE} - totalCount={totalCount} - sortBy={sortBy} - sortOrder={sortOrder} - /> -
- ); -}, { - minRole: OrgRole.OWNER, - redirectTo: "/settings", -}); diff --git a/packages/web/src/app/api/(server)/connection-sync-status/route.ts b/packages/web/src/app/api/(server)/connection-sync-status/route.ts index cb77ef583..16448c50e 100644 --- a/packages/web/src/app/api/(server)/connection-sync-status/route.ts +++ b/packages/web/src/app/api/(server)/connection-sync-status/route.ts @@ -1,4 +1,4 @@ -import type { ConnectionSyncStatusesResponse } from "@/app/(app)/settings/connectionsv2/types"; +import type { ConnectionSyncStatusesResponse } from "@/app/(app)/settings/connections/types"; import { apiHandler } from "@/lib/apiHandler"; import { getBullMQClient } from "@/lib/bullmqClient"; import { From 663325d4963f1ba86ebfbbfbf2733c2d3a3abe8e Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 18:59:02 -0700 Subject: [PATCH 17/34] improve first time syncing banner --- .../backend/src/repoIndexWorkload.test.ts | 47 +++++++- packages/backend/src/repoIndexWorkload.ts | 25 ++++ .../migration.sql | 11 ++ packages/db/prisma/schema.prisma | 2 + .../components/banners/bannerResolver.test.ts | 24 ++-- .../components/banners/bannerResolver.tsx | 8 +- .../repositoryFirstSyncBanner.test.tsx | 113 ++++++++++++++++++ .../banners/repositoryFirstSyncBanner.tsx | 46 ++++++- packages/web/src/app/(app)/layout.tsx | 55 ++------- packages/web/src/app/api/(client)/client.ts | 12 ++ .../(server)/repository-sync-counts/route.ts | 25 ++++ .../repos/repositorySyncCounts.server.ts | 53 ++++++++ 12 files changed, 355 insertions(+), 66 deletions(-) create mode 100644 packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql create mode 100644 packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx create mode 100644 packages/web/src/app/api/(server)/repository-sync-counts/route.ts create mode 100644 packages/web/src/features/repos/repositorySyncCounts.server.ts diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 46c290e74..bed614533 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -33,6 +33,7 @@ import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; const repoFindUnique = vi.fn(); const repoDeleteMany = vi.fn(); const repoUpdate = vi.fn(); +const repoUpdateMany = vi.fn(); const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => callback({ @@ -47,6 +48,7 @@ const db = { $transaction: transaction, repo: { deleteMany: repoDeleteMany, + updateMany: repoUpdateMany, }, } as unknown as PrismaClient; @@ -95,6 +97,7 @@ describe("repoIndexWorkload", () => { repoFindUnique.mockResolvedValue(eligibleRepo); repoDeleteMany.mockResolvedValue({ count: 1 }); repoUpdate.mockResolvedValue(undefined); + repoUpdateMany.mockResolvedValue({ count: 1 }); }); test("uses the same repository execution lock for INDEX and CLEANUP", () => { @@ -108,8 +111,48 @@ describe("repoIndexWorkload", () => { expect(workload.executionLock?.durationMs).toBe(60_000); expect(workload.queueSpec.dedupKey).toBeUndefined(); expect(workload.onStarted).toBeUndefined(); - expect(workload.onCompleted).toBeUndefined(); - expect(workload.onTerminalFailure).toBeUndefined(); + expect(workload.onCompleted).toBeTypeOf("function"); + expect(workload.onTerminalFailure).toBeTypeOf("function"); + }); + + test("records the first successful indexing job terminal state", async () => { + await workload.onCompleted?.(lifecycleContext, undefined); + + expect(repoUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + firstIndexingJobFinishedAt: null, + }, + data: { + firstIndexingJobFinishedAt: expect.any(Date), + }, + }); + }); + + test("records the first failed indexing job terminal state", async () => { + await workload.onTerminalFailure?.( + lifecycleContext, + new Error("indexing failed"), + ); + + expect(repoUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + firstIndexingJobFinishedAt: null, + }, + data: { + firstIndexingJobFinishedAt: expect.any(Date), + }, + }); + }); + + test("does not mark cleanup jobs as an initial indexing attempt", async () => { + await workload.onCompleted?.({ + ...lifecycleContext, + data: { repoId: 42, type: "CLEANUP" }, + }, undefined); + + expect(repoUpdateMany).not.toHaveBeenCalled(); }); test("validates state and records the latest job inside process", async () => { diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 258b6ff38..824da1c5d 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -117,8 +117,33 @@ export const createRepoIndexWorkload = ({ } } }, + onCompleted: async ({ data }) => { + await markFirstIndexingJobFinished(db, data); + }, + onTerminalFailure: async ({ data }) => { + await markFirstIndexingJobFinished(db, data); + }, }); +const markFirstIndexingJobFinished = async ( + db: PrismaClient, + data: { repoId: number; type: "INDEX" | "CLEANUP" }, +) => { + if (data.type !== "INDEX") { + return; + } + + await db.repo.updateMany({ + where: { + id: data.repoId, + firstIndexingJobFinishedAt: null, + }, + data: { + firstIndexingJobFinishedAt: new Date(), + }, + }); +}; + type RepoIndexStartDecision = | { action: "run"; diff --git a/packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql b/packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql new file mode 100644 index 000000000..c89ae953a --- /dev/null +++ b/packages/db/prisma/migrations/20260818183000_add_first_indexing_job_finished_at/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "Repo" ADD COLUMN "firstIndexingJobFinishedAt" TIMESTAMP(3); + +-- Existing successful indexes have already reached a terminal state. +UPDATE "Repo" +SET "firstIndexingJobFinishedAt" = "indexedAt" +WHERE "indexedAt" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "Repo_orgId_firstIndexingJobFinishedAt_idx" +ON "Repo"("orgId", "firstIndexingJobFinishedAt"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 9fb098353..5a441da7c 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -78,6 +78,7 @@ model Repo { indexedAt DateTime? /// When the repo was last indexed successfully. indexedCommitHash String? /// The commit hash of the last indexed commit (on HEAD). latestIndexingJobId String? /// The latest indexing job in the workload queue. + firstIndexingJobFinishedAt DateTime? /// When the first indexing job reached a terminal state, whether successful or failed. pushedAt DateTime? /// The timestamp of the most recent commit across all branches. external_id String /// The id of the repo in the external service @@ -92,6 +93,7 @@ model Repo { @@unique([external_id, external_codeHostUrl, orgId]) @@index([orgId]) + @@index([orgId, firstIndexingJobFinishedAt]) @@index([indexedAt]) } diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts index b38503475..a80bc9fa0 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts @@ -81,7 +81,7 @@ const makeContext = (overrides: Partial = {}): BannerContext => ( hasPermissionSyncEntitlement: false, hasPendingFirstSync: false, permissionSyncIssues: [], - repositorySyncCounts: { syncingCount: 0, failedCount: 0, warningCount: 0 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }, dismissals: {}, today: TODAY, now: NOW, @@ -142,7 +142,7 @@ describe('resolveActiveBanner', () => { const result = resolveActiveBanner(makeContext({ hasPermissionSyncEntitlement: true, hasPendingFirstSync: true, - repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 0 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, })); expect(result?.id).toBe('permissionSync'); }); @@ -153,7 +153,7 @@ describe('resolveActiveBanner', () => { status: 'trialing', trialEnd: daysFromNow(7), }), - repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 0 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, })); expect(result?.id).toBe('repositorySyncFailed'); }); @@ -164,7 +164,7 @@ describe('resolveActiveBanner', () => { status: 'trialing', trialEnd: daysFromNow(7), }), - repositorySyncCounts: { syncingCount: 0, failedCount: 0, warningCount: 1 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 1 }, })); expect(result?.id).toBe('trial'); }); @@ -173,7 +173,7 @@ describe('resolveActiveBanner', () => { describe('repository sync issues', () => { test('shows failures to owners', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncCounts: { syncingCount: 0, failedCount: 2, warningCount: 1 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 2, warningCount: 1 }, })); expect(result?.id).toBe('repositorySyncFailed'); expect(result?.dismissible).toBe(true); @@ -182,7 +182,7 @@ describe('resolveActiveBanner', () => { test('shows warnings when there are no failures', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncCounts: { syncingCount: 0, failedCount: 0, warningCount: 2 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 2 }, })); expect(result?.id).toBe('repositorySyncWarning'); expect(result?.dismissible).toBe(true); @@ -191,14 +191,14 @@ describe('resolveActiveBanner', () => { test('hides issues from members', () => { const result = resolveActiveBanner(makeContext({ role: OrgRole.MEMBER, - repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 1 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, })); expect(result).toBeNull(); }); test('does not let a warning dismissal suppress a later failure', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 1 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, dismissals: { repositorySyncWarning: TODAY }, })); expect(result?.id).toBe('repositorySyncFailed'); @@ -206,7 +206,7 @@ describe('resolveActiveBanner', () => { test('hides failures dismissed today', () => { const result = resolveActiveBanner(makeContext({ - repositorySyncCounts: { syncingCount: 0, failedCount: 1, warningCount: 1 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, dismissals: { repositorySyncFailed: TODAY }, })); expect(result).toBeNull(); @@ -217,7 +217,7 @@ describe('resolveActiveBanner', () => { test('shows first-time syncing repositories to owners', () => { const result = resolveActiveBanner(makeContext({ repositorySyncCounts: { - syncingCount: 3, + firstTimeSyncingCount: 3, failedCount: 0, warningCount: 0, }, @@ -232,7 +232,7 @@ describe('resolveActiveBanner', () => { const result = resolveActiveBanner(makeContext({ role: OrgRole.MEMBER, repositorySyncCounts: { - syncingCount: 3, + firstTimeSyncingCount: 3, failedCount: 0, warningCount: 0, }, @@ -244,7 +244,7 @@ describe('resolveActiveBanner', () => { test('repository warnings take priority over first-time syncing', () => { const result = resolveActiveBanner(makeContext({ repositorySyncCounts: { - syncingCount: 3, + firstTimeSyncingCount: 3, failedCount: 0, warningCount: 1, }, diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx index 1ff5ba0ec..ddd1abe98 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx @@ -33,7 +33,7 @@ export interface BannerContext { hasPendingFirstSync: boolean; permissionSyncIssues: PermissionSyncStatusResponse['issues']; repositorySyncCounts: { - syncingCount: number; + firstTimeSyncingCount: number; failedCount: number; warningCount: number; }; @@ -186,7 +186,7 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { } const { - syncingCount: repositorySyncingCount, + firstTimeSyncingCount: repositoryFirstTimeSyncingCount, failedCount: repositorySyncFailedCount, warningCount: repositorySyncWarningCount, } = ctx.repositorySyncCounts; @@ -219,7 +219,7 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { ), }); } - if (repositorySyncingCount > 0) { + if (repositoryFirstTimeSyncingCount > 0) { banners.push({ id: 'repositoryFirstSync', priority: BannerPriority.REPOSITORY_FIRST_SYNC, @@ -228,7 +228,7 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { render: (props) => ( ), }); diff --git a/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx new file mode 100644 index 000000000..caf639415 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.test.tsx @@ -0,0 +1,113 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ refresh: vi.fn() })); +const api = vi.hoisted(() => ({ getRepositorySyncCounts: vi.fn() })); + +vi.mock("next/navigation", () => ({ + useRouter: () => navigation, +})); + +vi.mock("@/app/api/(client)/client", () => ({ + getRepositorySyncCounts: api.getRepositorySyncCounts, +})); + +vi.mock("@/lib/utils", () => ({ + cn: (...classes: Array) => + classes.filter(Boolean).join(" "), + unwrapServiceError: (value: unknown) => value, +})); + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ title, description, action }: { + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+
{title}
+
{description}
+
{action}
+
+ ), +})); + +const { RepositoryFirstSyncBanner } = await import( + "./repositoryFirstSyncBanner" +); + +const renderBanner = (firstTimeSyncingCount: number) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + , + ); + return queryClient; +}; + +beforeEach(() => { + api.getRepositorySyncCounts.mockResolvedValue({ + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 0, + }); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("RepositoryFirstSyncBanner", () => { + test("updates the displayed count from the latest status", async () => { + api.getRepositorySyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }); + renderBanner(3); + + expect(screen.getByText(/3 repositories are syncing/)).toBeTruthy(); + await waitFor(() => { + expect(screen.getByText(/2 repositories are syncing/)).toBeTruthy(); + }); + }); + + test("hides and refreshes when the first sync count reaches zero", async () => { + const queryClient = renderBanner(3); + await waitFor(() => { + expect(api.getRepositorySyncCounts).toHaveBeenCalled(); + }); + api.getRepositorySyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 0, + failedCount: 1, + warningCount: 0, + }); + + await act(async () => { + await queryClient.refetchQueries({ + queryKey: ["repository-sync-counts"], + }); + }); + + await waitFor(() => { + expect(screen.queryByText(/syncing for the first time/)).toBeNull(); + expect(navigation.refresh).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx index 6e772771e..34970448e 100644 --- a/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx +++ b/packages/web/src/app/(app)/components/banners/repositoryFirstSyncBanner.tsx @@ -1,26 +1,64 @@ +"use client"; + +import { getRepositorySyncCounts } from "@/app/api/(client)/client"; import { Button } from "@/components/ui/button"; +import type { RepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import { unwrapServiceError } from "@/lib/utils"; +import { useQuery } from "@tanstack/react-query"; +import { usePrevious } from "@uidotdev/usehooks"; import { Loader2 } from "lucide-react"; import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect } from "react"; import { BannerShell } from "./bannerShell"; import type { BannerProps } from "./types"; +const POLL_INTERVAL_MS = 5_000; + interface RepositoryFirstSyncBannerProps extends BannerProps { - syncingCount: number; + initialCounts: RepositorySyncCounts; } export function RepositoryFirstSyncBanner({ id, dismissible, - syncingCount, + initialCounts, }: RepositoryFirstSyncBannerProps) { - const isSingular = syncingCount === 1; + const router = useRouter(); + const { data: counts, isError, isPending } = useQuery({ + queryKey: ["repository-sync-counts"], + queryFn: () => unwrapServiceError(getRepositorySyncCounts()), + refetchInterval: (query) => + query.state.data?.firstTimeSyncingCount + ? POLL_INTERVAL_MS + : false, + initialData: initialCounts, + }); + const previousCount = usePrevious(counts.firstTimeSyncingCount); + + useEffect(() => { + if ( + previousCount !== undefined + && previousCount > 0 + && counts.firstTimeSyncingCount === 0 + ) { + router.refresh(); + } + }, [counts.firstTimeSyncingCount, previousCount, router]); + + if (isError || isPending || counts.firstTimeSyncingCount === 0) { + return null; + } + + const firstTimeSyncingCount = counts.firstTimeSyncingCount; + const isSingular = firstTimeSyncingCount === 1; return ( } - title={`${syncingCount} ${isSingular ? "repository is" : "repositories are"} syncing for the first time`} + title={`${firstTimeSyncingCount} ${isSingular ? "repository is" : "repositories are"} syncing for the first time`} description={`${isSingular ? "It" : "They"} won't be available until syncing completes.`} action={(
From 191435d02a44aa1e91369a022ea5c61ccb3a527a Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 19:23:14 -0700 Subject: [PATCH 19/34] connection sync issue banner --- .../components/banners/bannerResolver.test.ts | 36 ++++++++ .../components/banners/bannerResolver.tsx | 36 ++++++++ .../(app)/components/banners/bannerSlot.tsx | 2 + .../connectionSyncIssuesBanner.test.tsx | 73 +++++++++++++++ .../banners/connectionSyncIssuesBanner.tsx | 50 +++++++++++ .../src/app/(app)/components/banners/types.ts | 4 + packages/web/src/app/(app)/layout.tsx | 8 ++ .../components/connectionsTable.test.tsx | 3 + .../components/syncIssuePopover.tsx | 2 +- .../connectionSyncCounts.server.test.ts | 90 +++++++++++++++++++ .../connectionSyncCounts.server.ts | 54 +++++++++++ 11 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx create mode 100644 packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx create mode 100644 packages/web/src/features/connections/connectionSyncCounts.server.test.ts create mode 100644 packages/web/src/features/connections/connectionSyncCounts.server.ts diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts index a80bc9fa0..1f9166eab 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts @@ -23,6 +23,7 @@ vi.mock('./trialBanner', () => ({ TrialBanner: () => null })); vi.mock('./upgradeAvailableBanner', () => ({ UpgradeAvailableBanner: () => null })); vi.mock('./repositorySyncIssuesBanner', () => ({ RepositorySyncIssuesBanner: () => null })); vi.mock('./repositoryFirstSyncBanner', () => ({ RepositoryFirstSyncBanner: () => null })); +vi.mock('./connectionSyncIssuesBanner', () => ({ ConnectionSyncIssuesBanner: () => null })); import { resolveActiveBanner, type BannerContext } from './bannerResolver'; @@ -81,6 +82,7 @@ const makeContext = (overrides: Partial = {}): BannerContext => ( hasPermissionSyncEntitlement: false, hasPendingFirstSync: false, permissionSyncIssues: [], + connectionSyncCounts: { failedCount: 0, warningCount: 0 }, repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }, dismissals: {}, today: TODAY, @@ -147,6 +149,14 @@ describe('resolveActiveBanner', () => { expect(result?.id).toBe('permissionSync'); }); + test('connection sync failures outrank repository sync failures', () => { + const result = resolveActiveBanner(makeContext({ + connectionSyncCounts: { failedCount: 1, warningCount: 0 }, + repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, + })); + expect(result?.id).toBe('connectionSyncFailed'); + }); + test('repository sync failures outrank trial notices', () => { const result = resolveActiveBanner(makeContext({ license: makeLicense({ @@ -170,6 +180,32 @@ describe('resolveActiveBanner', () => { }); }); + describe('connection sync issues', () => { + test('shows failures and warnings as separate banners', () => { + const failed = resolveActiveBanner(makeContext({ + connectionSyncCounts: { failedCount: 2, warningCount: 3 }, + })); + expect(failed?.id).toBe('connectionSyncFailed'); + expect(failed?.dismissible).toBe(true); + expect(failed?.audience).toBe('owner'); + + const warning = resolveActiveBanner(makeContext({ + connectionSyncCounts: { failedCount: 2, warningCount: 3 }, + dismissals: { connectionSyncFailed: TODAY }, + })); + expect(warning?.id).toBe('connectionSyncWarning'); + expect(warning?.dismissible).toBe(true); + }); + + test('hides connection sync issues from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + connectionSyncCounts: { failedCount: 1, warningCount: 1 }, + })); + expect(result).toBeNull(); + }); + }); + describe('repository sync issues', () => { test('shows failures to owners', () => { const result = resolveActiveBanner(makeContext({ diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx index ddd1abe98..107a9877a 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx @@ -17,6 +17,7 @@ import { TrialBanner } from "./trialBanner"; import { UpgradeAvailableBanner } from "./upgradeAvailableBanner"; import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; import { RepositoryFirstSyncBanner } from "./repositoryFirstSyncBanner"; +import { ConnectionSyncIssuesBanner } from "./connectionSyncIssuesBanner"; import type { PermissionSyncStatusResponse } from "@/app/api/(server)/ee/permissionSyncStatus/api"; // Mirrors the value in `lighthouse: lambda/serviceError.ts` and the gating @@ -32,6 +33,10 @@ export interface BannerContext { hasPermissionSyncEntitlement: boolean; hasPendingFirstSync: boolean; permissionSyncIssues: PermissionSyncStatusResponse['issues']; + connectionSyncCounts: { + failedCount: number; + warningCount: number; + }; repositorySyncCounts: { firstTimeSyncingCount: number; failedCount: number; @@ -185,6 +190,37 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { }); } + if (ctx.connectionSyncCounts.failedCount > 0) { + banners.push({ + id: 'connectionSyncFailed', + priority: BannerPriority.CONNECTION_SYNC_FAILED, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + if (ctx.connectionSyncCounts.warningCount > 0) { + banners.push({ + id: 'connectionSyncWarning', + priority: BannerPriority.CONNECTION_SYNC_WARNING, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } + const { firstTimeSyncingCount: repositoryFirstTimeSyncingCount, failedCount: repositorySyncFailedCount, diff --git a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx index 9cb70337b..93861fd8e 100644 --- a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx @@ -9,6 +9,8 @@ const KNOWN_BANNER_IDS: BannerId[] = [ 'licenseReboundElsewhere', 'invoicePastDue', 'permissionSync', + 'connectionSyncFailed', + 'connectionSyncWarning', 'repositorySyncFailed', 'repositorySyncWarning', 'repositoryFirstSync', diff --git a/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx new file mode 100644 index 000000000..195379c74 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.test.tsx @@ -0,0 +1,73 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { ConnectionSyncIssuesBanner } from "./connectionSyncIssuesBanner"; + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ + icon, + title, + description, + action, + }: { + icon?: ReactNode; + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+ {icon} +
{title}
+
{description}
+
{action}
+
+ ), +})); + +afterEach(cleanup); + +describe("ConnectionSyncIssuesBanner", () => { + test("renders a never-synced failure independently", () => { + render( + , + ); + + expect(screen.getByText("1 code host connection needs attention")).toBeTruthy(); + expect(screen.getByText( + "This connection failed to sync. Repositories are unavailable.", + )).toBeTruthy(); + expect( + screen.getByRole("link", { name: "View failed" }).getAttribute("href"), + ).toBe("/settings/connections?status=failed"); + expect(screen.queryByRole("link", { name: "View warnings" })).toBeNull(); + }); + + test("renders warnings independently", () => { + render( + , + ); + + expect(screen.getByText("2 code host connections need attention")).toBeTruthy(); + expect(screen.getByText( + "These connections have warnings. Repository discovery may be incomplete or out of date.", + )).toBeTruthy(); + expect( + screen.getByRole("link", { name: "View warnings" }).getAttribute("href"), + ).toBe("/settings/connections?status=warning"); + expect(screen.queryByRole("link", { name: "View failed" })).toBeNull(); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx new file mode 100644 index 000000000..6b01de68c --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionSyncIssuesBanner.tsx @@ -0,0 +1,50 @@ +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; +import { AlertTriangle } from "lucide-react"; +import Link from "next/link"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +interface ConnectionSyncIssuesBannerProps extends BannerProps { + count: number; + status: "failed" | "warning"; +} + +const pluralizeConnection = (count: number) => + count === 1 ? "connection" : "connections"; + +export function ConnectionSyncIssuesBanner({ + id, + dismissible, + count, + status, +}: ConnectionSyncIssuesBannerProps) { + const isFailed = status === "failed"; + const isSingular = count === 1; + + return ( + + )} + title={`${count} code host ${pluralizeConnection(count)} ${isSingular ? "needs" : "need"} attention`} + description={isFailed + ? `${isSingular ? "This connection" : "These connections"} failed to sync. Repositories are unavailable.` + : `${isSingular ? "This connection has a warning" : "These connections have warnings"}. Repository discovery may be incomplete or out of date.`} + action={( + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/types.ts b/packages/web/src/app/(app)/components/banners/types.ts index caaab2627..419739a0d 100644 --- a/packages/web/src/app/(app)/components/banners/types.ts +++ b/packages/web/src/app/(app)/components/banners/types.ts @@ -7,9 +7,11 @@ export const BannerPriority = { SERVICE_PING_ENFORCED: 95, INVOICE_PAST_DUE: 90, PERMISSION_SYNC: 50, + CONNECTION_SYNC_FAILED: 48, REPOSITORY_SYNC_FAILED: 45, TRIAL: 25, LICENSE_EXPIRY_HEADS_UP: 20, + CONNECTION_SYNC_WARNING: 18, REPOSITORY_SYNC_WARNING: 15, REPOSITORY_FIRST_SYNC: 12, SERVICE_PING_FAILED: 10, @@ -21,6 +23,8 @@ export type BannerId = | 'licenseReboundElsewhere' | 'invoicePastDue' | 'permissionSync' + | 'connectionSyncFailed' + | 'connectionSyncWarning' | 'repositorySyncFailed' | 'repositorySyncWarning' | 'repositoryFirstSync' diff --git a/packages/web/src/app/(app)/layout.tsx b/packages/web/src/app/(app)/layout.tsx index 4d1521808..3176e7209 100644 --- a/packages/web/src/app/(app)/layout.tsx +++ b/packages/web/src/app/(app)/layout.tsx @@ -37,6 +37,7 @@ import { LanguageModelProvider } from "@/features/chat/languageModelContext"; import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server"; import { NavigationGuardProvider } from "next-navigation-guard"; import { getRepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import { getConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; interface LayoutProps { children: React.ReactNode; @@ -180,6 +181,12 @@ export default async function Layout(props: LayoutProps) { }; }) : { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }; + const connectionSyncCounts = role === OrgRole.OWNER + ? await getConnectionSyncCounts(org.id).catch((error) => { + console.error("Failed to load connection sync counts", error); + return { failedCount: 0, warningCount: 0 }; + }) + : { failedCount: 0, warningCount: 0 }; const offlineLicense = getOfflineLicenseMetadata(); const license = offlineLicense @@ -214,6 +221,7 @@ export default async function Layout(props: LayoutProps) { hasPermissionSyncEntitlement={hasPermissionSyncEntitlement} hasPendingFirstSync={hasPendingFirstSync} permissionSyncIssues={permissionSyncIssues} + connectionSyncCounts={connectionSyncCounts} repositorySyncCounts={repositorySyncCounts} currentVersion={SOURCEBOT_VERSION} latestVersion={latestVersion} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx index 698a6301e..d053e30c9 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx @@ -436,6 +436,9 @@ describe("ConnectionsTable", () => { })); expect(screen.getByText("Connection sync failed")).toBeTruthy(); + expect(screen.getByText( + "This connection failed to sync. Its repositories are unavailable.", + )).toBeTruthy(); expect( screen.getByText( "Authentication failed while discovering repositories", diff --git a/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx b/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx index be9cae5be..3eb77d9ad 100644 --- a/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/syncIssuePopover.tsx @@ -196,7 +196,7 @@ export const SyncIssuePopover = ({ ? "Sourcebot could not honor the full configured discovery scope. Some repositories may be missing." : isWarning ? "Previously discovered repositories remain available, but they may be stale." - : "Sourcebot could not complete the latest sync for this connection."} + : "This connection failed to sync. Its repositories are unavailable."}

{reasons diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.test.ts b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts new file mode 100644 index 000000000..8f7f32f58 --- /dev/null +++ b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts @@ -0,0 +1,90 @@ +import type { WorkloadJob } from "@sourcebot/shared"; +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + findMany: vi.fn(), + getJobs: vi.fn(), +})); + +vi.mock("server-only", () => ({})); +vi.mock("@/prisma", () => ({ + __unsafePrisma: { + connection: { + findMany: mocks.findMany, + }, + }, +})); +vi.mock("@/lib/bullmqClient", () => ({ + getBullMQClient: () => ({ + getJobs: mocks.getJobs, + }), +})); + +const { getConnectionSyncCounts } = await import( + "./connectionSyncCounts.server" +); + +const job = ( + id: string, + connectionId: number, + status: WorkloadJob<"connection-sync">["status"], + result: WorkloadJob<"connection-sync">["result"] = null, +): WorkloadJob<"connection-sync"> => ({ + id, + data: { connectionId }, + status, + errorMessage: status === "FAILED" ? "sync failed" : null, + result, +}); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("getConnectionSyncCounts", () => { + test("classifies never-synced failures separately from warnings", async () => { + mocks.findMany.mockResolvedValue([ + { id: 1, syncedAt: null, latestSyncJobId: "failed-first-sync" }, + { id: 2, syncedAt: new Date(), latestSyncJobId: "failed-resync" }, + { id: 3, syncedAt: new Date(), latestSyncJobId: "partial-success" }, + { id: 4, syncedAt: new Date(), latestSyncJobId: "success" }, + { id: 5, syncedAt: null, latestSyncJobId: "mismatched-job" }, + { id: 6, syncedAt: null, latestSyncJobId: null }, + ]); + mocks.getJobs.mockResolvedValue(new Map([ + ["failed-first-sync", job("failed-first-sync", 1, "FAILED")], + ["failed-resync", job("failed-resync", 2, "FAILED")], + ["partial-success", job( + "partial-success", + 3, + "COMPLETED", + { outcome: "PARTIAL_SUCCESS", reasons: [] }, + )], + ["success", job("success", 4, "COMPLETED", { outcome: "SUCCESS" })], + ["mismatched-job", job("mismatched-job", 999, "FAILED")], + ])); + + await expect(getConnectionSyncCounts(42)).resolves.toEqual({ + failedCount: 1, + warningCount: 2, + }); + expect(mocks.findMany).toHaveBeenCalledWith({ + where: { orgId: 42 }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + }, + }); + expect(mocks.getJobs).toHaveBeenCalledWith( + expect.objectContaining({ name: "connection-sync" }), + [ + "failed-first-sync", + "failed-resync", + "partial-success", + "success", + "mismatched-job", + ], + ); + }); +}); diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.ts b/packages/web/src/features/connections/connectionSyncCounts.server.ts new file mode 100644 index 000000000..f614bbfc8 --- /dev/null +++ b/packages/web/src/features/connections/connectionSyncCounts.server.ts @@ -0,0 +1,54 @@ +import "server-only"; + +import { getBullMQClient } from "@/lib/bullmqClient"; +import { __unsafePrisma } from "@/prisma"; +import { CONNECTION_QUEUE } from "@sourcebot/shared"; + +export interface ConnectionSyncCounts { + failedCount: number; + warningCount: number; +} + +export const getConnectionSyncCounts = async ( + orgId: number, +): Promise => { + const connections = await __unsafePrisma.connection.findMany({ + where: { orgId }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + }, + }); + const latestJobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + const latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); + + return connections.reduce((counts, connection) => { + const latestJob = connection.latestSyncJobId + ? latestJobs.get(connection.latestSyncJobId) + : null; + if (!latestJob || latestJob.data.connectionId !== connection.id) { + return counts; + } + + if (latestJob.status === "FAILED") { + if (connection.syncedAt) { + counts.warningCount += 1; + } else { + counts.failedCount += 1; + } + } else if ( + latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ) { + counts.warningCount += 1; + } + + return counts; + }, { failedCount: 0, warningCount: 0 }); +}; From 51b92439aaf05bdb2d7ce61a0be0732aa7201ca4 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 19:31:27 -0700 Subject: [PATCH 20/34] remove connection job table --- .../src/connectionSyncWorkload.test.ts | 81 +------------------ .../backend/src/connectionSyncWorkload.ts | 51 +----------- .../migration.sql | 5 ++ packages/db/prisma/schema.prisma | 22 ----- 4 files changed, 13 insertions(+), 146 deletions(-) create mode 100644 packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index 46793d0cc..6a8eae457 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -6,8 +6,6 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), - connectionSyncJobUpsert: vi.fn(), - connectionSyncJobUpdate: vi.fn(), compileGithubConfig: vi.fn(), loadConfig: vi.fn(), syncSearchContexts: vi.fn(), @@ -99,28 +97,11 @@ import { import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { REPO_PERMISSION_SYNC_WHERE } from "./ee/permissionSyncEligibility.js"; -const transactionClient = { - connection: { - update: mocks.connectionUpdate, - }, - connectionSyncJob: { - upsert: mocks.connectionSyncJobUpsert, - }, -}; -const transaction = vi.fn( - (callback: (tx: typeof transactionClient) => Promise) => - callback(transactionClient), -); - const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, }, - connectionSyncJob: { - upsert: mocks.connectionSyncJobUpsert, - update: mocks.connectionSyncJobUpdate, - }, repo: { findMany: mocks.repoFindMany, upsert: mocks.repoUpsert, @@ -128,7 +109,6 @@ const db = { repoToConnection: { deleteMany: mocks.repoToConnectionDeleteMany, }, - $transaction: transaction, } as unknown as PrismaClient; const jobManager = { @@ -172,10 +152,10 @@ describe("connectionWorkload", () => { mocks.syncSearchContexts.mockResolvedValue(undefined); }); - test("declares database-backed lifecycle hooks", () => { + test("only records the latest job when the workload starts", () => { expect(connectionWorkload.onStarted).toBeTypeOf("function"); - expect(connectionWorkload.onCompleted).toBeTypeOf("function"); - expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); + expect(connectionWorkload.onCompleted).toBeUndefined(); + expect(connectionWorkload.onTerminalFailure).toBeUndefined(); }); test("uses a distinct execution lock for each connection", () => { @@ -204,26 +184,9 @@ describe("connectionWorkload", () => { expect(mocks.connectionFindUniqueOrThrow).not.toHaveBeenCalled(); }); - test("marks the connection sync job as in progress when started", async () => { + test("records the latest connection sync job ID when started", async () => { await connectionWorkload.onStarted?.(lifecycleContext); - expect(mocks.connectionSyncJobUpsert).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - update: { - status: "IN_PROGRESS", - completedAt: null, - errorMessage: null, - warningMessages: [], - }, - create: { - id: "job-1", - connectionId: 42, - status: "IN_PROGRESS", - warningMessages: [], - }, - }); expect(mocks.connectionUpdate).toHaveBeenCalledWith({ where: { id: 42, @@ -232,42 +195,6 @@ describe("connectionWorkload", () => { latestSyncJobId: "job-1", }, }); - expect(transaction).toHaveBeenCalledOnce(); - }); - - test("marks the connection sync job as completed", async () => { - await connectionWorkload.onCompleted?.(lifecycleContext, { - outcome: "SUCCESS", - }); - - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - status: "COMPLETED", - completedAt: expect.any(Date), - errorMessage: null, - }, - }); - }); - - test("marks the connection sync job as failed after terminal failure", async () => { - await connectionWorkload.onTerminalFailure?.( - lifecycleContext, - new Error("Connection credentials expired"), - ); - - expect(mocks.connectionSyncJobUpdate).toHaveBeenCalledWith({ - where: { - id: "job-1", - }, - data: { - status: "FAILED", - completedAt: expect.any(Date), - errorMessage: "Connection credentials expired", - }, - }); }); test("orchestrates discovery, persistence, and repo work reconciliation", async () => { diff --git a/packages/backend/src/connectionSyncWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts index f4aaf6b45..a7847b234 100644 --- a/packages/backend/src/connectionSyncWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -1,5 +1,5 @@ import * as Sentry from "@sentry/node"; -import { ConnectionSyncJobStatus, PrismaClient } from "@sourcebot/db"; +import { PrismaClient } from "@sourcebot/db"; import { ConnectionConfig } from "@sourcebot/schemas/v3/index.type"; import { CONNECTION_QUEUE, @@ -152,55 +152,12 @@ export const createConnectionSyncWorkload = ({ : { outcome: "PARTIAL_SUCCESS", reasons: issues }; }, onStarted: async ({ data: { connectionId }, jobId }) => { - await db.$transaction(async (tx) => { - await tx.connectionSyncJob.upsert({ - where: { - id: jobId, - }, - update: { - status: ConnectionSyncJobStatus.IN_PROGRESS, - completedAt: null, - errorMessage: null, - warningMessages: [], - }, - create: { - id: jobId, - connectionId, - status: ConnectionSyncJobStatus.IN_PROGRESS, - warningMessages: [], - }, - }); - await tx.connection.update({ - where: { - id: connectionId, - }, - data: { - latestSyncJobId: jobId, - }, - }); - }); - }, - onCompleted: async ({ jobId }) => { - await db.connectionSyncJob.update({ - where: { - id: jobId, - }, - data: { - status: ConnectionSyncJobStatus.COMPLETED, - completedAt: new Date(), - errorMessage: null, - }, - }); - }, - onTerminalFailure: async ({ jobId }, error) => { - await db.connectionSyncJob.update({ + await db.connection.update({ where: { - id: jobId, + id: connectionId, }, data: { - status: ConnectionSyncJobStatus.FAILED, - completedAt: new Date(), - errorMessage: error.message, + latestSyncJobId: jobId, }, }); }, diff --git a/packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql b/packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql new file mode 100644 index 000000000..ef37c453c --- /dev/null +++ b/packages/db/prisma/migrations/20260818193500_drop_connection_sync_job/migration.sql @@ -0,0 +1,5 @@ +-- DropTable +DROP TABLE "ConnectionSyncJob"; + +-- DropEnum +DROP TYPE "ConnectionSyncJobStatus"; diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 5a441da7c..f98b91f08 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -134,7 +134,6 @@ model Connection { // The type of connection (e.g., github, gitlab, etc.) connectionType ConnectionType - syncJobs ConnectionSyncJob[] /// When the connection was last synced successfully. syncedAt DateTime? latestSyncJobId String? /// The most recently started connection sync job. @@ -162,27 +161,6 @@ model Connection { @@unique([name, orgId]) } -enum ConnectionSyncJobStatus { - PENDING - IN_PROGRESS - COMPLETED - FAILED -} - -model ConnectionSyncJob { - id String @id @default(cuid()) - status ConnectionSyncJobStatus @default(PENDING) - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - completedAt DateTime? - - warningMessages String[] - errorMessage String? - - connection Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade) - connectionId Int -} - model RepoToConnection { addedAt DateTime @default(now()) From f9ae84b5736b61c4eb841bcb3508cc27a2a954b6 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 19:43:26 -0700 Subject: [PATCH 21/34] connection progress banner --- .../src/connectionSyncWorkload.test.ts | 42 ++++++- .../backend/src/connectionSyncWorkload.ts | 21 ++++ .../migration.sql | 11 ++ packages/db/prisma/schema.prisma | 2 + .../components/banners/bannerResolver.test.ts | 40 +++++- .../components/banners/bannerResolver.tsx | 16 +++ .../(app)/components/banners/bannerSlot.tsx | 1 + .../connectionFirstSyncBanner.test.tsx | 116 ++++++++++++++++++ .../banners/connectionFirstSyncBanner.tsx | 72 +++++++++++ .../src/app/(app)/components/banners/types.ts | 2 + packages/web/src/app/(app)/layout.tsx | 8 +- packages/web/src/app/api/(client)/client.ts | 12 ++ .../(server)/connection-sync-counts/route.ts | 25 ++++ .../connectionSyncCounts.server.test.ts | 17 ++- .../connectionSyncCounts.server.ts | 11 +- 15 files changed, 379 insertions(+), 17 deletions(-) create mode 100644 packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql create mode 100644 packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx create mode 100644 packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx create mode 100644 packages/web/src/app/api/(server)/connection-sync-counts/route.ts diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index 6a8eae457..60d1d610c 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -6,6 +6,7 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const mocks = vi.hoisted(() => ({ connectionFindUniqueOrThrow: vi.fn(), connectionUpdate: vi.fn(), + connectionUpdateMany: vi.fn(), compileGithubConfig: vi.fn(), loadConfig: vi.fn(), syncSearchContexts: vi.fn(), @@ -101,6 +102,7 @@ const db = { connection: { findUniqueOrThrow: mocks.connectionFindUniqueOrThrow, update: mocks.connectionUpdate, + updateMany: mocks.connectionUpdateMany, }, repo: { findMany: mocks.repoFindMany, @@ -143,6 +145,7 @@ describe("connectionWorkload", () => { vi.restoreAllMocks(); vi.clearAllMocks(); mocks.connectionUpdate.mockResolvedValue({}); + mocks.connectionUpdateMany.mockResolvedValue({ count: 1 }); mocks.repoFindMany.mockResolvedValue([]); mocks.getJobSchedulerIds.mockResolvedValue([]); mocks.upsertJobScheduler.mockResolvedValue("scheduled-job"); @@ -152,10 +155,10 @@ describe("connectionWorkload", () => { mocks.syncSearchContexts.mockResolvedValue(undefined); }); - test("only records the latest job when the workload starts", () => { + test("declares the connection lifecycle hooks", () => { expect(connectionWorkload.onStarted).toBeTypeOf("function"); - expect(connectionWorkload.onCompleted).toBeUndefined(); - expect(connectionWorkload.onTerminalFailure).toBeUndefined(); + expect(connectionWorkload.onCompleted).toBeTypeOf("function"); + expect(connectionWorkload.onTerminalFailure).toBeTypeOf("function"); }); test("uses a distinct execution lock for each connection", () => { @@ -197,6 +200,39 @@ describe("connectionWorkload", () => { }); }); + test("records the first successful sync job terminal state", async () => { + await connectionWorkload.onCompleted?.(lifecycleContext, { + outcome: "SUCCESS", + }); + + expect(mocks.connectionUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + firstSyncJobFinishedAt: null, + }, + data: { + firstSyncJobFinishedAt: expect.any(Date), + }, + }); + }); + + test("records the first failed sync job terminal state", async () => { + await connectionWorkload.onTerminalFailure?.( + lifecycleContext, + new Error("Connection credentials expired"), + ); + + expect(mocks.connectionUpdateMany).toHaveBeenCalledWith({ + where: { + id: 42, + firstSyncJobFinishedAt: null, + }, + data: { + firstSyncJobFinishedAt: expect.any(Date), + }, + }); + }); + test("orchestrates discovery, persistence, and repo work reconciliation", async () => { const config = { type: "github" as const, diff --git a/packages/backend/src/connectionSyncWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts index a7847b234..eaf05b059 100644 --- a/packages/backend/src/connectionSyncWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -161,8 +161,29 @@ export const createConnectionSyncWorkload = ({ }, }); }, + onCompleted: async ({ data: { connectionId } }) => { + await markFirstSyncJobFinished(db, connectionId); + }, + onTerminalFailure: async ({ data: { connectionId } }) => { + await markFirstSyncJobFinished(db, connectionId); + }, }); +const markFirstSyncJobFinished = async ( + db: PrismaClient, + connectionId: number, +) => { + await db.connection.updateMany({ + where: { + id: connectionId, + firstSyncJobFinishedAt: null, + }, + data: { + firstSyncJobFinishedAt: new Date(), + }, + }); +}; + export interface CurrentRepo { id: number; name: string; diff --git a/packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql b/packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql new file mode 100644 index 000000000..637c978a5 --- /dev/null +++ b/packages/db/prisma/migrations/20260818194000_add_first_connection_sync_job_finished_at/migration.sql @@ -0,0 +1,11 @@ +-- AlterTable +ALTER TABLE "Connection" ADD COLUMN "firstSyncJobFinishedAt" TIMESTAMP(3); + +-- Existing successful syncs have already reached a terminal state. +UPDATE "Connection" +SET "firstSyncJobFinishedAt" = "syncedAt" +WHERE "syncedAt" IS NOT NULL; + +-- CreateIndex +CREATE INDEX "Connection_orgId_firstSyncJobFinishedAt_idx" +ON "Connection"("orgId", "firstSyncJobFinishedAt"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index f98b91f08..5632d10ed 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -137,6 +137,7 @@ model Connection { /// When the connection was last synced successfully. syncedAt DateTime? latestSyncJobId String? /// The most recently started connection sync job. + firstSyncJobFinishedAt DateTime? /// When the first sync job reached a terminal state, whether successful or failed. /// Controls whether repository permissions are enforced for this connection. /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect. @@ -159,6 +160,7 @@ model Connection { orgId Int @@unique([name, orgId]) + @@index([orgId, firstSyncJobFinishedAt]) } model RepoToConnection { diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts index 1f9166eab..135bb5831 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.test.ts @@ -24,6 +24,7 @@ vi.mock('./upgradeAvailableBanner', () => ({ UpgradeAvailableBanner: () => null vi.mock('./repositorySyncIssuesBanner', () => ({ RepositorySyncIssuesBanner: () => null })); vi.mock('./repositoryFirstSyncBanner', () => ({ RepositoryFirstSyncBanner: () => null })); vi.mock('./connectionSyncIssuesBanner', () => ({ ConnectionSyncIssuesBanner: () => null })); +vi.mock('./connectionFirstSyncBanner', () => ({ ConnectionFirstSyncBanner: () => null })); import { resolveActiveBanner, type BannerContext } from './bannerResolver'; @@ -82,7 +83,7 @@ const makeContext = (overrides: Partial = {}): BannerContext => ( hasPermissionSyncEntitlement: false, hasPendingFirstSync: false, permissionSyncIssues: [], - connectionSyncCounts: { failedCount: 0, warningCount: 0 }, + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }, repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }, dismissals: {}, today: TODAY, @@ -151,7 +152,7 @@ describe('resolveActiveBanner', () => { test('connection sync failures outrank repository sync failures', () => { const result = resolveActiveBanner(makeContext({ - connectionSyncCounts: { failedCount: 1, warningCount: 0 }, + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, repositorySyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 0 }, })); expect(result?.id).toBe('connectionSyncFailed'); @@ -183,14 +184,14 @@ describe('resolveActiveBanner', () => { describe('connection sync issues', () => { test('shows failures and warnings as separate banners', () => { const failed = resolveActiveBanner(makeContext({ - connectionSyncCounts: { failedCount: 2, warningCount: 3 }, + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 2, warningCount: 3 }, })); expect(failed?.id).toBe('connectionSyncFailed'); expect(failed?.dismissible).toBe(true); expect(failed?.audience).toBe('owner'); const warning = resolveActiveBanner(makeContext({ - connectionSyncCounts: { failedCount: 2, warningCount: 3 }, + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 2, warningCount: 3 }, dismissals: { connectionSyncFailed: TODAY }, })); expect(warning?.id).toBe('connectionSyncWarning'); @@ -200,12 +201,41 @@ describe('resolveActiveBanner', () => { test('hides connection sync issues from members', () => { const result = resolveActiveBanner(makeContext({ role: OrgRole.MEMBER, - connectionSyncCounts: { failedCount: 1, warningCount: 1 }, + connectionSyncCounts: { firstTimeSyncingCount: 0, failedCount: 1, warningCount: 1 }, })); expect(result).toBeNull(); }); }); + describe('connection first sync', () => { + test('shows first-time syncing connections to owners', () => { + const result = resolveActiveBanner(makeContext({ + connectionSyncCounts: { + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result?.id).toBe('connectionFirstSync'); + expect(result?.dismissible).toBe(true); + expect(result?.audience).toBe('owner'); + }); + + test('hides first-time syncing connections from members', () => { + const result = resolveActiveBanner(makeContext({ + role: OrgRole.MEMBER, + connectionSyncCounts: { + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }, + })); + + expect(result).toBeNull(); + }); + }); + describe('repository sync issues', () => { test('shows failures to owners', () => { const result = resolveActiveBanner(makeContext({ diff --git a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx index 107a9877a..cdad2a1c4 100644 --- a/packages/web/src/app/(app)/components/banners/bannerResolver.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerResolver.tsx @@ -18,6 +18,7 @@ import { UpgradeAvailableBanner } from "./upgradeAvailableBanner"; import { RepositorySyncIssuesBanner } from "./repositorySyncIssuesBanner"; import { RepositoryFirstSyncBanner } from "./repositoryFirstSyncBanner"; import { ConnectionSyncIssuesBanner } from "./connectionSyncIssuesBanner"; +import { ConnectionFirstSyncBanner } from "./connectionFirstSyncBanner"; import type { PermissionSyncStatusResponse } from "@/app/api/(server)/ee/permissionSyncStatus/api"; // Mirrors the value in `lighthouse: lambda/serviceError.ts` and the gating @@ -34,6 +35,7 @@ export interface BannerContext { hasPendingFirstSync: boolean; permissionSyncIssues: PermissionSyncStatusResponse['issues']; connectionSyncCounts: { + firstTimeSyncingCount: number; failedCount: number; warningCount: number; }; @@ -220,6 +222,20 @@ function buildCandidates(ctx: BannerContext): BannerDescriptor[] { ), }); } + if (ctx.connectionSyncCounts.firstTimeSyncingCount > 0) { + banners.push({ + id: 'connectionFirstSync', + priority: BannerPriority.CONNECTION_FIRST_SYNC, + dismissible: true, + audience: 'owner', + render: (props) => ( + + ), + }); + } const { firstTimeSyncingCount: repositoryFirstTimeSyncingCount, diff --git a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx index 93861fd8e..83ffe4199 100644 --- a/packages/web/src/app/(app)/components/banners/bannerSlot.tsx +++ b/packages/web/src/app/(app)/components/banners/bannerSlot.tsx @@ -11,6 +11,7 @@ const KNOWN_BANNER_IDS: BannerId[] = [ 'permissionSync', 'connectionSyncFailed', 'connectionSyncWarning', + 'connectionFirstSync', 'repositorySyncFailed', 'repositorySyncWarning', 'repositoryFirstSync', diff --git a/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx new file mode 100644 index 000000000..884e3b931 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.test.tsx @@ -0,0 +1,116 @@ +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ refresh: vi.fn() })); +const api = vi.hoisted(() => ({ getConnectionSyncCounts: vi.fn() })); + +vi.mock("next/navigation", () => ({ + useRouter: () => navigation, +})); + +vi.mock("@/app/api/(client)/client", () => ({ + getConnectionSyncCounts: api.getConnectionSyncCounts, +})); + +vi.mock("@/lib/utils", () => ({ + cn: (...classes: Array) => + classes.filter(Boolean).join(" "), + unwrapServiceError: (value: unknown) => value, +})); + +vi.mock("./bannerShell", () => ({ + BannerShell: ({ title, description, action }: { + title: ReactNode; + description?: ReactNode; + action?: ReactNode; + }) => ( +
+
{title}
+
{description}
+
{action}
+
+ ), +})); + +const { ConnectionFirstSyncBanner } = await import( + "./connectionFirstSyncBanner" +); + +const renderBanner = (firstTimeSyncingCount: number) => { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false } }, + }); + render( + + + , + ); + return queryClient; +}; + +beforeEach(() => { + api.getConnectionSyncCounts.mockResolvedValue({ + firstTimeSyncingCount: 3, + failedCount: 0, + warningCount: 0, + }); +}); + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("ConnectionFirstSyncBanner", () => { + test("updates the displayed count from the latest status", async () => { + api.getConnectionSyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 2, + failedCount: 0, + warningCount: 0, + }); + renderBanner(3); + + expect(screen.getByText(/3 code host connections are syncing/)).toBeTruthy(); + await waitFor(() => { + expect(screen.getByText(/2 code host connections are syncing/)).toBeTruthy(); + }); + expect( + screen.getByRole("link", { name: "View connections" }).getAttribute("href"), + ).toBe("/settings/connections?sortBy=syncedAt&sortOrder=desc"); + }); + + test("hides and refreshes when the first sync count reaches zero", async () => { + const queryClient = renderBanner(3); + await waitFor(() => { + expect(api.getConnectionSyncCounts).toHaveBeenCalled(); + }); + api.getConnectionSyncCounts.mockResolvedValueOnce({ + firstTimeSyncingCount: 0, + failedCount: 1, + warningCount: 0, + }); + + await act(async () => { + await queryClient.refetchQueries({ + queryKey: ["connection-sync-counts"], + }); + }); + + await waitFor(() => { + expect(screen.queryByText(/syncing for the first time/)).toBeNull(); + expect(navigation.refresh).toHaveBeenCalledOnce(); + }); + }); +}); diff --git a/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx new file mode 100644 index 000000000..561fe0d64 --- /dev/null +++ b/packages/web/src/app/(app)/components/banners/connectionFirstSyncBanner.tsx @@ -0,0 +1,72 @@ +"use client"; + +import { getConnectionSyncCounts } from "@/app/api/(client)/client"; +import { Button } from "@/components/ui/button"; +import type { ConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; +import { unwrapServiceError } from "@/lib/utils"; +import { useQuery } from "@tanstack/react-query"; +import { usePrevious } from "@uidotdev/usehooks"; +import { Loader2 } from "lucide-react"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect } from "react"; +import { BannerShell } from "./bannerShell"; +import type { BannerProps } from "./types"; + +const POLL_INTERVAL_MS = 5_000; + +interface ConnectionFirstSyncBannerProps extends BannerProps { + initialCounts: ConnectionSyncCounts; +} + +export function ConnectionFirstSyncBanner({ + id, + dismissible, + initialCounts, +}: ConnectionFirstSyncBannerProps) { + const router = useRouter(); + const { data: counts, isError, isPending } = useQuery({ + queryKey: ["connection-sync-counts"], + queryFn: () => unwrapServiceError(getConnectionSyncCounts()), + refetchInterval: (query) => + query.state.data?.firstTimeSyncingCount + ? POLL_INTERVAL_MS + : false, + initialData: initialCounts, + }); + const previousCount = usePrevious(counts.firstTimeSyncingCount); + + useEffect(() => { + if ( + previousCount !== undefined + && previousCount > 0 + && counts.firstTimeSyncingCount === 0 + ) { + router.refresh(); + } + }, [counts.firstTimeSyncingCount, previousCount, router]); + + if (isError || isPending || counts.firstTimeSyncingCount === 0) { + return null; + } + + const firstTimeSyncingCount = counts.firstTimeSyncingCount; + const isSingular = firstTimeSyncingCount === 1; + + return ( + } + title={`${firstTimeSyncingCount} code host ${isSingular ? "connection is" : "connections are"} syncing for the first time`} + description={`Repositories from ${isSingular ? "this connection are" : "these connections are"} unavailable until syncing completes.`} + action={( + + )} + /> + ); +} diff --git a/packages/web/src/app/(app)/components/banners/types.ts b/packages/web/src/app/(app)/components/banners/types.ts index 419739a0d..38d6a11dd 100644 --- a/packages/web/src/app/(app)/components/banners/types.ts +++ b/packages/web/src/app/(app)/components/banners/types.ts @@ -13,6 +13,7 @@ export const BannerPriority = { LICENSE_EXPIRY_HEADS_UP: 20, CONNECTION_SYNC_WARNING: 18, REPOSITORY_SYNC_WARNING: 15, + CONNECTION_FIRST_SYNC: 14, REPOSITORY_FIRST_SYNC: 12, SERVICE_PING_FAILED: 10, UPGRADE_AVAILABLE: 5, @@ -25,6 +26,7 @@ export type BannerId = | 'permissionSync' | 'connectionSyncFailed' | 'connectionSyncWarning' + | 'connectionFirstSync' | 'repositorySyncFailed' | 'repositorySyncWarning' | 'repositoryFirstSync' diff --git a/packages/web/src/app/(app)/layout.tsx b/packages/web/src/app/(app)/layout.tsx index 3176e7209..2097e35d8 100644 --- a/packages/web/src/app/(app)/layout.tsx +++ b/packages/web/src/app/(app)/layout.tsx @@ -184,9 +184,13 @@ export default async function Layout(props: LayoutProps) { const connectionSyncCounts = role === OrgRole.OWNER ? await getConnectionSyncCounts(org.id).catch((error) => { console.error("Failed to load connection sync counts", error); - return { failedCount: 0, warningCount: 0 }; + return { + firstTimeSyncingCount: 0, + failedCount: 0, + warningCount: 0, + }; }) - : { failedCount: 0, warningCount: 0 }; + : { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }; const offlineLicense = getOfflineLicenseMetadata(); const license = offlineLicense diff --git a/packages/web/src/app/api/(client)/client.ts b/packages/web/src/app/api/(client)/client.ts index 0e71855a4..4f0a714bd 100644 --- a/packages/web/src/app/api/(client)/client.ts +++ b/packages/web/src/app/api/(client)/client.ts @@ -38,6 +38,7 @@ import type { } from "../(server)/ee/chat/[chatId]/searchMembers/route"; import type { JobLogs, QueueName } from "@sourcebot/shared"; import type { RepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import type { ConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; import type { OffersResponse } from "@sourcebot/shared/client"; import { ConnectMcpResponse } from "../(server)/ee/askmcp/connect/types"; import type { GetMcpServersResponse } from "../(server)/ee/askmcp/servers/route"; @@ -273,6 +274,17 @@ export const getRepositorySyncCounts = async (): Promise => { + const result = await fetch("/api/connection-sync-counts", { + method: "GET", + headers: { + "X-Sourcebot-Client-Source": "sourcebot-web-client", + }, + }).then(response => response.json()); + + return result as ConnectionSyncCounts | ServiceError; +} + export const getAccountSyncStatus = async (jobId: string): Promise => { const url = new URL("/api/ee/accountPermissionSyncJobStatus", window.location.origin); url.searchParams.set("jobId", jobId); diff --git a/packages/web/src/app/api/(server)/connection-sync-counts/route.ts b/packages/web/src/app/api/(server)/connection-sync-counts/route.ts new file mode 100644 index 000000000..264ecb1ac --- /dev/null +++ b/packages/web/src/app/api/(server)/connection-sync-counts/route.ts @@ -0,0 +1,25 @@ +import { getConnectionSyncCounts } from "@/features/connections/connectionSyncCounts.server"; +import { apiHandler } from "@/lib/apiHandler"; +import { serviceErrorResponse } from "@/lib/serviceError"; +import { isServiceError } from "@/lib/utils"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { sew } from "@/middleware/sew"; +import { OrgRole } from "@sourcebot/db"; +import { StatusCodes } from "http-status-codes"; + +export const GET = apiHandler(async () => { + const result = await sew(() => + withAuth(({ org, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, () => + getConnectionSyncCounts(org.id) + ) + ) + ); + + if (isServiceError(result)) { + return serviceErrorResponse(result); + } + + return Response.json(result, { status: StatusCodes.OK }); +}); diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.test.ts b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts index 8f7f32f58..ddf2616e6 100644 --- a/packages/web/src/features/connections/connectionSyncCounts.server.test.ts +++ b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts @@ -44,12 +44,13 @@ beforeEach(() => { describe("getConnectionSyncCounts", () => { test("classifies never-synced failures separately from warnings", async () => { mocks.findMany.mockResolvedValue([ - { id: 1, syncedAt: null, latestSyncJobId: "failed-first-sync" }, - { id: 2, syncedAt: new Date(), latestSyncJobId: "failed-resync" }, - { id: 3, syncedAt: new Date(), latestSyncJobId: "partial-success" }, - { id: 4, syncedAt: new Date(), latestSyncJobId: "success" }, - { id: 5, syncedAt: null, latestSyncJobId: "mismatched-job" }, - { id: 6, syncedAt: null, latestSyncJobId: null }, + { id: 1, syncedAt: null, latestSyncJobId: "failed-first-sync", firstSyncJobFinishedAt: new Date() }, + { id: 2, syncedAt: new Date(), latestSyncJobId: "failed-resync", firstSyncJobFinishedAt: new Date() }, + { id: 3, syncedAt: new Date(), latestSyncJobId: "partial-success", firstSyncJobFinishedAt: new Date() }, + { id: 4, syncedAt: new Date(), latestSyncJobId: "success", firstSyncJobFinishedAt: new Date() }, + { id: 5, syncedAt: null, latestSyncJobId: "mismatched-job", firstSyncJobFinishedAt: new Date() }, + { id: 6, syncedAt: null, latestSyncJobId: null, firstSyncJobFinishedAt: null }, + { id: 7, syncedAt: null, latestSyncJobId: "active-first-sync", firstSyncJobFinishedAt: null }, ]); mocks.getJobs.mockResolvedValue(new Map([ ["failed-first-sync", job("failed-first-sync", 1, "FAILED")], @@ -62,9 +63,11 @@ describe("getConnectionSyncCounts", () => { )], ["success", job("success", 4, "COMPLETED", { outcome: "SUCCESS" })], ["mismatched-job", job("mismatched-job", 999, "FAILED")], + ["active-first-sync", job("active-first-sync", 7, "IN_PROGRESS")], ])); await expect(getConnectionSyncCounts(42)).resolves.toEqual({ + firstTimeSyncingCount: 2, failedCount: 1, warningCount: 2, }); @@ -74,6 +77,7 @@ describe("getConnectionSyncCounts", () => { id: true, syncedAt: true, latestSyncJobId: true, + firstSyncJobFinishedAt: true, }, }); expect(mocks.getJobs).toHaveBeenCalledWith( @@ -84,6 +88,7 @@ describe("getConnectionSyncCounts", () => { "partial-success", "success", "mismatched-job", + "active-first-sync", ], ); }); diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.ts b/packages/web/src/features/connections/connectionSyncCounts.server.ts index f614bbfc8..dd85841a8 100644 --- a/packages/web/src/features/connections/connectionSyncCounts.server.ts +++ b/packages/web/src/features/connections/connectionSyncCounts.server.ts @@ -5,6 +5,7 @@ import { __unsafePrisma } from "@/prisma"; import { CONNECTION_QUEUE } from "@sourcebot/shared"; export interface ConnectionSyncCounts { + firstTimeSyncingCount: number; failedCount: number; warningCount: number; } @@ -18,6 +19,7 @@ export const getConnectionSyncCounts = async ( id: true, syncedAt: true, latestSyncJobId: true, + firstSyncJobFinishedAt: true, }, }); const latestJobIds = connections.flatMap((connection) => @@ -29,6 +31,13 @@ export const getConnectionSyncCounts = async ( ); return connections.reduce((counts, connection) => { + if ( + connection.syncedAt === null + && connection.firstSyncJobFinishedAt === null + ) { + counts.firstTimeSyncingCount += 1; + } + const latestJob = connection.latestSyncJobId ? latestJobs.get(connection.latestSyncJobId) : null; @@ -50,5 +59,5 @@ export const getConnectionSyncCounts = async ( } return counts; - }, { failedCount: 0, warningCount: 0 }); + }, { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }); }; From 7bab7c71377df467f6e548b83b6470b1f55bc1f8 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 19:46:31 -0700 Subject: [PATCH 22/34] add clear filter button --- .../repos/components/reposTable.test.tsx | 13 ++++++++++ .../app/(app)/repos/components/reposTable.tsx | 26 +++++++++---------- .../components/connectionsTable.test.tsx | 13 ++++++++++ .../components/connectionsTable.tsx | 26 +++++++++---------- 4 files changed, 52 insertions(+), 26 deletions(-) diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index 6b92c5ff8..8b1430ddf 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -133,6 +133,19 @@ describe("ReposTable", () => { ); }); + test.each(["search=first", "status=warning"])( + "shows clear filters in the toolbar for %s", + (searchParams) => { + navigation.searchParams = searchParams; + + renderTable([repos[1]]); + + expect( + screen.getByRole("button", { name: "Clear filters" }), + ).toBeTruthy(); + }, + ); + test("does not offer to clear filters in the unfiltered empty state", () => { renderTable([]); diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index a1988f8ca..2ca70a1f8 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -27,7 +27,7 @@ import { getCoreRowModel, useReactTable, } from "@tanstack/react-table"; -import { ArrowDown, ArrowUp, Check, Loader2, Search } from "lucide-react"; +import { ArrowDown, ArrowUp, Check, CircleX, Loader2, Search } from "lucide-react"; import Image from "next/image"; import Link from "next/link"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; @@ -842,6 +842,17 @@ export const ReposTable = ({ Warning + {hasActiveFilters && ( + + )}
@@ -897,18 +908,7 @@ export const ReposTable = ({ colSpan={columns.length} className="h-28 text-center text-sm text-muted-foreground" > -
-

{emptyMessage}

- {hasActiveFilters && ( - - )} -
+

{emptyMessage}

)} diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx index d053e30c9..a1fb02e92 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.test.tsx @@ -151,6 +151,19 @@ describe("ConnectionsTable", () => { ); }); + test.each(["search=github", "status=warning"])( + "shows clear filters in the toolbar for %s", + (searchParams) => { + navigation.searchParams = searchParams; + + renderTable({ data: [connections[0]] }); + + expect( + screen.getByRole("button", { name: "Clear filters" }), + ).toBeTruthy(); + }, + ); + test("uses URL-driven server pagination", () => { renderTable({ totalCount: 29 }); diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx index 6880a5fc2..5343b3a17 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx @@ -33,7 +33,7 @@ import { getCoreRowModel, useReactTable, } from "@tanstack/react-table"; -import { ArrowDown, ArrowUp, Loader2, Search } from "lucide-react"; +import { ArrowDown, ArrowUp, CircleX, Loader2, Search } from "lucide-react"; import Image from "next/image"; import { usePathname, useRouter, useSearchParams } from "next/navigation"; import { @@ -594,6 +594,17 @@ export const ConnectionsTable = ({ Warning + {hasActiveFilters && ( + + )}
@@ -649,18 +660,7 @@ export const ConnectionsTable = ({ colSpan={columns.length} className="h-28 text-center text-sm text-muted-foreground" > -
-

{emptyMessage}

- {hasActiveFilters && ( - - )} -
+

{emptyMessage}

)} From 3367852c9d6907b887de06588eeb7c117a79e2c2 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:10:35 -0700 Subject: [PATCH 23/34] migrate other hosts to using report function --- packages/backend/src/azuredevops.test.ts | 137 +++++++++ packages/backend/src/azuredevops.ts | 123 ++++---- packages/backend/src/bitbucket.test.ts | 165 ++++++++++- packages/backend/src/bitbucket.ts | 267 ++++++++++-------- packages/backend/src/connectionUtils.ts | 42 +-- packages/backend/src/gitea.test.ts | 132 +++++++++ packages/backend/src/gitea.ts | 115 ++++---- packages/backend/src/github.ts | 14 +- packages/backend/src/gitlab.test.ts | 114 +++++++- packages/backend/src/gitlab.ts | 86 +++--- packages/backend/src/repoCompileUtils.test.ts | 95 +++++++ packages/backend/src/repoCompileUtils.ts | 58 +++- 12 files changed, 1013 insertions(+), 335 deletions(-) create mode 100644 packages/backend/src/azuredevops.test.ts create mode 100644 packages/backend/src/gitea.test.ts diff --git a/packages/backend/src/azuredevops.test.ts b/packages/backend/src/azuredevops.test.ts new file mode 100644 index 000000000..234eaa55c --- /dev/null +++ b/packages/backend/src/azuredevops.test.ts @@ -0,0 +1,137 @@ +import type { AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/azuredevops.type'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + getProjects: vi.fn(), + getRepositories: vi.fn(), + getRepository: vi.fn(), +})); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...await importOriginal(), + getTokenFromConfig: vi.fn(async () => "token"), +})); + +vi.mock("azure-devops-node-api", () => ({ + getPersonalAccessTokenHandler: vi.fn(() => ({})), + WebApi: class { + getCoreApi = vi.fn(async () => ({ + getProjects: mocks.getProjects, + })); + getGitApi = vi.fn(async () => ({ + getRepositories: mocks.getRepositories, + getRepository: mocks.getRepository, + })); + }, +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { getAzureDevOpsReposFromConfig } from './azuredevops'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; + +const config = (overrides: Partial): AzureDevOpsConnectionConfig => ({ + type: "azuredevops", + deploymentType: "cloud", + token: { env: "AZURE_DEVOPS_TOKEN" }, + ...overrides, +}); + +beforeEach(() => { + vi.clearAllMocks(); + const notFound = Object.assign(new Error("Not Found"), { statusCode: 404 }); + mocks.getProjects.mockRejectedValue(notFound); + mocks.getRepositories.mockRejectedValue(notFound); + mocks.getRepository.mockRejectedValue(notFound); +}); + +describe("Azure DevOps repository discovery", () => { + test("reports inaccessible configured targets as partial successes", async () => { + const result = await collectRepositoryDiscoveryIssues(() => + getAzureDevOpsReposFromConfig(config({ + orgs: ["missing-org"], + projects: ["org/missing-project"], + repos: ["org/project/missing-repo"], + })) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: "missing-org", + }, + message: "Azure DevOps organization was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: "org/missing-project", + }, + message: "Azure DevOps project was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "org/project/missing-repo", + }, + message: "Azure DevOps repository was not found or is inaccessible.", + }, + ], + }); + }); + + test("reports incomplete project enumeration within an organization", async () => { + mocks.getProjects.mockResolvedValue([ + { name: "missing-id" }, + { id: "broken-project-id", name: "broken-project" }, + ]); + mocks.getRepositories.mockRejectedValue(new Error("Service unavailable")); + + const result = await collectRepositoryDiscoveryIssues(() => + getAzureDevOpsReposFromConfig(config({ orgs: ["my-org"] })) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: "my-org/missing-id", + }, + message: "Azure DevOps returned a project without an ID, so its repositories were skipped.", + }, + { + code: "ENUMERATION_FAILED", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: "my-org/broken-project", + }, + message: "Azure DevOps repository enumeration did not complete for this project.", + }, + ], + }); + }); +}); diff --git a/packages/backend/src/azuredevops.ts b/packages/backend/src/azuredevops.ts index 23b8edd62..864a80d86 100644 --- a/packages/backend/src/azuredevops.ts +++ b/packages/backend/src/azuredevops.ts @@ -7,6 +7,7 @@ import * as Sentry from "@sentry/node"; import * as azdev from "azure-devops-node-api"; import { GitRepository } from "azure-devops-node-api/interfaces/GitInterfaces.js"; import { getTokenFromConfig } from "@sourcebot/shared"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; const logger = createLogger('azuredevops'); const AZUREDEVOPS_CLOUD_HOSTNAME = "dev.azure.com"; @@ -42,39 +43,32 @@ export const getAzureDevOpsReposFromConfig = async ( const useTfsPath = config.useTfsPath || false; let allRepos: GitRepository[] = []; - let allWarnings: string[] = []; if (config.orgs) { - const { repos, warnings } = await getReposForOrganizations( + allRepos = allRepos.concat(await getReposForOrganizations( config.orgs, baseUrl, token, useTfsPath - ); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + )); } if (config.projects) { - const { repos, warnings } = await getReposForProjects( + allRepos = allRepos.concat(await getReposForProjects( config.projects, baseUrl, token, useTfsPath - ); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + )); } if (config.repos) { - const { repos, warnings } = await getRepos( + allRepos = allRepos.concat(await getRepos( config.repos, baseUrl, token, useTfsPath - ); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + )); } let repos = allRepos @@ -89,10 +83,7 @@ export const getAzureDevOpsReposFromConfig = async ( logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; }; export const shouldExcludeRepo = ({ @@ -180,6 +171,15 @@ async function getReposForOrganizations( for (const project of projects) { if (!project.id) { logger.warn(`Encountered project in org ${org} with no id: ${project.name}`); + reportRepositoryDiscoveryIssue({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: `${org}/${project.name ?? "unknown"}`, + }, + message: "Azure DevOps returned a project without an ID, so its repositories were skipped.", + }); continue; } @@ -188,6 +188,15 @@ async function getReposForOrganizations( allRepos.push(...repos); } catch (error) { logger.warn(`Failed to fetch repositories for project ${project.name}: ${error}`); + reportRepositoryDiscoveryIssue({ + code: "ENUMERATION_FAILED", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "project", + value: `${org}/${project.name ?? project.id}`, + }, + message: "Azure DevOps repository enumeration did not complete for this project.", + }); } } @@ -198,10 +207,7 @@ async function getReposForOrganizations( }); logger.debug(`Found ${data.length} repositories in organization ${org} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for organization ${org}.`, error); @@ -210,22 +216,23 @@ async function getReposForOrganizations( if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { const warning = `Organization ${org} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: org, + }, + message: "Azure DevOps organization was not found or is inaccessible.", + }); + return []; } throw error; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } async function getReposForProjects( @@ -253,10 +260,7 @@ async function getReposForProjects( }); logger.debug(`Found ${data.length} repositories in project ${project} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (error) { Sentry.captureException(error); logger.error(`Failed to fetch repositories for project ${project}.`, error); @@ -264,22 +268,23 @@ async function getReposForProjects( if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { const warning = `Project ${project} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Azure DevOps project was not found or is inaccessible.", + }); + return []; } throw error; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } async function getRepos( @@ -307,10 +312,7 @@ async function getRepos( }); logger.debug(`Found info for repository ${repo} in ${durationMs}ms`); - return { - type: 'valid' as const, - data: [result] - }; + return [result]; } catch (error) { Sentry.captureException(error); @@ -319,20 +321,21 @@ async function getRepos( if (error && typeof error === 'object' && 'statusCode' in error && error.statusCode === 404) { const warning = `Repository ${repo} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Azure DevOps repository was not found or is inaccessible.", + }); + return []; } throw error; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; -} \ No newline at end of file + return processPromiseResults(results); +} diff --git a/packages/backend/src/bitbucket.test.ts b/packages/backend/src/bitbucket.test.ts index 5d2fb55e4..2177e7d1b 100644 --- a/packages/backend/src/bitbucket.test.ts +++ b/packages/backend/src/bitbucket.test.ts @@ -1,8 +1,163 @@ -import { expect, test, describe } from 'vitest'; -import { cloudShouldExcludeRepo, serverShouldExcludeRepo, BitbucketRepository } from './bitbucket'; -import { BitbucketConnectionConfig } from '@sourcebot/schemas/v3/bitbucket.type'; -import { SchemaRepository as CloudRepository } from '@coderabbitai/bitbucket/cloud/openapi'; -import { SchemaRestRepository as ServerRepository } from '@coderabbitai/bitbucket/server/openapi'; +import { beforeEach, expect, test, describe, vi } from 'vitest'; +import type { BitbucketConnectionConfig } from '@sourcebot/schemas/v3/bitbucket.type'; +import type { SchemaRepository as CloudRepository } from '@coderabbitai/bitbucket/cloud/openapi'; +import type { SchemaRestRepository as ServerRepository } from '@coderabbitai/bitbucket/server/openapi'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { status: 404 }); + + return { + cloudGet: vi.fn(async () => { + throw notFound; + }), + serverGet: vi.fn(async () => { + throw notFound; + }), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@coderabbitai/bitbucket/cloud", () => ({ + createBitbucketCloudClient: () => ({ + use: vi.fn(), + GET: mocks.cloudGet, + }), +})); + +vi.mock("@coderabbitai/bitbucket/server", () => ({ + createBitbucketServerClient: () => ({ + use: vi.fn(), + GET: mocks.serverGet, + }), +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; +import { + cloudShouldExcludeRepo, + getBitbucketReposFromConfig, + serverShouldExcludeRepo, + type BitbucketRepository, +} from './bitbucket'; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Bitbucket repository discovery", () => { + test("reports Bitbucket Cloud discovery gaps", async () => { + const config = { + type: "bitbucket", + deploymentType: "cloud", + all: true, + workspaces: ["missing-workspace"], + projects: ["workspace/missing-project", "invalid-project"], + repos: ["workspace/missing-repo", "invalid-repo"], + exclude: { archived: true }, + } satisfies BitbucketConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getBitbucketReposFromConfig(config) + ); + + expect(result.value).toEqual([]); + expect(result.issues).toHaveLength(7); + expect(result.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { kind: "configuration", value: "all" }, + }), + expect.objectContaining({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "exclude.archived", + }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { kind: "workspace", value: "missing-workspace" }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { + kind: "project", + value: "workspace/missing-project", + }, + }), + expect.objectContaining({ + code: "INVALID_TARGET", + subject: { kind: "project", value: "invalid-project" }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { + kind: "repository", + value: "workspace/missing-repo", + }, + }), + expect.objectContaining({ + code: "INVALID_TARGET", + subject: { kind: "repository", value: "invalid-repo" }, + }), + ])); + }); + + test("reports Bitbucket Server discovery gaps", async () => { + const config = { + type: "bitbucket", + deploymentType: "server", + url: "https://bitbucket.example.com", + workspaces: ["unsupported-workspace"], + projects: ["missing-project"], + repos: ["PROJ/missing-repo", "invalid-repo"], + } satisfies BitbucketConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getBitbucketReposFromConfig(config) + ); + + expect(result.value).toEqual([]); + expect(result.issues).toHaveLength(4); + expect(result.issues).toEqual(expect.arrayContaining([ + expect.objectContaining({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "workspace", + value: "unsupported-workspace", + }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { kind: "project", value: "missing-project" }, + }), + expect.objectContaining({ + code: "NOT_FOUND_OR_INACCESSIBLE", + subject: { + kind: "repository", + value: "PROJ/missing-repo", + }, + }), + expect.objectContaining({ + code: "INVALID_TARGET", + subject: { kind: "repository", value: "invalid-repo" }, + }), + ])); + }); +}); const makeCloudRepo = (overrides: Partial = {}): BitbucketRepository => ({ type: 'repository', diff --git a/packages/backend/src/bitbucket.ts b/packages/backend/src/bitbucket.ts index 84f85d023..525eb336d 100644 --- a/packages/backend/src/bitbucket.ts +++ b/packages/backend/src/bitbucket.ts @@ -11,9 +11,9 @@ import { SchemaRepositoryUserPermission as CloudRepositoryUserPermission, } from "@coderabbitai/bitbucket/cloud/openapi"; import { SchemaRestRepository as ServerRepository } from "@coderabbitai/bitbucket/server/openapi"; -import { processPromiseResults } from "./connectionUtils.js"; -import { throwIfAnyFailed } from "./connectionUtils.js"; +import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; import { getTokenFromConfig } from "@sourcebot/shared"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; const logger = createLogger('bitbucket'); const BITBUCKET_CLOUD_GIT = 'https://bitbucket.org'; @@ -29,9 +29,9 @@ interface BitbucketClient { apiClient: any; baseUrl: string; gitUrl: string; - getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; - getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; - getRepos: (client: BitbucketClient, repos: string[]) => Promise<{repos: BitbucketRepository[], warnings: string[]}>; + getReposForWorkspace: (client: BitbucketClient, workspaces: string[]) => Promise; + getReposForProjects: (client: BitbucketClient, projects: string[]) => Promise; + getRepos: (client: BitbucketClient, repos: string[]) => Promise; shouldExcludeRepo: (repo: BitbucketRepository, config: BitbucketConnectionConfig) => boolean; } @@ -91,46 +91,63 @@ export const getBitbucketReposFromConfig = async (config: BitbucketConnectionCon createBitbucketCloudClient(config.user, token); let allRepos: BitbucketRepository[] = []; - let allWarnings: string[] = []; if (config.all === true) { if (client.deploymentType === BITBUCKET_SERVER) { - const { repos, warnings } = await serverGetAllRepos(client); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await serverGetAllRepos(client)); } else { const warning = `Ignoring option all:true in config: not supported for Bitbucket Cloud`; logger.warn(warning); - allWarnings = allWarnings.concat(warning); + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "all", + }, + message: "The all option is not supported for Bitbucket Cloud.", + }); } } if (config.workspaces) { - const { repos, warnings } = await client.getReposForWorkspace(client, config.workspaces); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat( + await client.getReposForWorkspace(client, config.workspaces), + ); } if (config.projects) { - const { repos, warnings } = await client.getReposForProjects(client, config.projects); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat( + await client.getReposForProjects(client, config.projects), + ); } if (config.repos) { - const { repos, warnings } = await client.getRepos(client, config.repos); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await client.getRepos(client, config.repos)); + } + + if ( + client.deploymentType === BITBUCKET_CLOUD + && config.exclude?.archived + ) { + const warning = "Bitbucket Cloud does not support filtering archived repositories. Ignoring exclude.archived."; + logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "exclude.archived", + }, + message: "The exclude.archived option is not supported for Bitbucket Cloud.", + }); } const filteredRepos = allRepos.filter((repo) => { return !client.shouldExcludeRepo(repo, config); }); - return { - repos: filteredRepos, - warnings: allWarnings, - }; + return filteredRepos; } export function createBitbucketCloudClient(user: string | undefined, token: string | undefined): BitbucketClient { @@ -210,7 +227,7 @@ function parseUrl(url: string): { path: string; query: Record; } } -async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { +async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise { const results = await Promise.allSettled(workspaces.map(async (workspace) => { try { logger.debug(`Fetching all repos for workspace ${workspace}...`); @@ -231,10 +248,7 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st }); logger.debug(`Found ${data.length} repos for workspace ${workspace} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: data, - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to get repos for workspace ${workspace}: ${e}`); @@ -242,33 +256,41 @@ async function cloudGetReposForWorkspace(client: BitbucketClient, workspaces: st if (e?.status === 404) { const warning = `Workspace ${workspace} not found or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - } + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "workspace", + value: workspace, + }, + message: "Bitbucket workspace was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings, - }; + return processPromiseResults(results); } -async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { +async function cloudGetReposForProjects(client: BitbucketClient, projects: string[]): Promise { const results = await Promise.allSettled(projects.map(async (project) => { const [workspace, project_name] = project.split('/'); if (!workspace || !project_name) { const warning = `Invalid project ${project}`; logger.warn(warning); - return { - type: 'warning' as const, - warning - } + reportRepositoryDiscoveryIssue({ + code: "INVALID_TARGET", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Bitbucket Cloud projects must use the workspace/project format.", + }); + return []; } logger.debug(`Fetching all repos for project ${project} for workspace ${workspace}...`); @@ -292,10 +314,7 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin }); logger.debug(`Found ${repos.length} repos for project ${project_name} for workspace ${workspace} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: repos - } + return repos; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch repos for project ${project_name}: ${e}`); @@ -303,33 +322,41 @@ async function cloudGetReposForProjects(client: BitbucketClient, projects: strin if (e?.status === 404) { const warning = `Project ${project_name} not found in ${workspace} or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - } + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Bitbucket Cloud project was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - } + return processPromiseResults(results); } -async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: CloudRepository[], warnings: string[]}> { +async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promise { const results = await Promise.allSettled(repoList.map(async (repo) => { const [workspace, repo_slug] = repo.split('/'); if (!workspace || !repo_slug) { const warning = `Invalid repo ${repo}`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "INVALID_TARGET", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Cloud repositories must use the workspace/repository format.", + }); + return []; } logger.debug(`Fetching repo ${repo_slug} for workspace ${workspace}...`); @@ -339,10 +366,7 @@ async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promi const { data } = await client.apiClient.GET(path); return data; }, `repo ${repo}`, logger); - return { - type: 'valid' as const, - data: [data] - }; + return [data]; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch repo ${repo}: ${e}`); @@ -350,21 +374,23 @@ async function cloudGetRepos(client: BitbucketClient, repoList: string[]): Promi if (e?.status === 404) { const warning = `Repo ${repo} not found in ${workspace} or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Cloud repository was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - }; + return processPromiseResults(results); } export function cloudShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean { @@ -381,10 +407,6 @@ export function cloudShouldExcludeRepo(repo: BitbucketRepository, config: Bitbuc } } - if (!!config.exclude?.archived) { - logger.warn(`Exclude archived repos flag provided in config but Bitbucket Cloud does not support archived repos. Ignoring...`); - } - if (!!config.exclude?.forks && cloudRepo.parent !== undefined) { reason = `\`exclude.forks\` is true`; return true; @@ -464,16 +486,23 @@ const getPaginatedServer = async ( return results; } -async function serverGetReposForWorkspace(client: BitbucketClient, workspaces: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { - const warnings = workspaces.map(workspace => `Workspaces are not supported in Bitbucket Server: ${workspace}`); +async function serverGetReposForWorkspace(_client: BitbucketClient, workspaces: string[]): Promise { logger.debug('Workspaces are not supported in Bitbucket Server'); - return { - repos: [], - warnings - }; + for (const workspace of workspaces) { + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "workspace", + value: workspace, + }, + message: "Workspaces are not supported for Bitbucket Server.", + }); + } + return []; } -async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { +async function serverGetReposForProjects(client: BitbucketClient, projects: string[]): Promise { const results = await Promise.allSettled(projects.map(async (project) => { try { logger.debug(`Fetching all repos for project ${project}...`); @@ -495,10 +524,7 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri }); logger.debug(`Found ${data.length} repos for project ${project} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: data, - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to get repos for project ${project}: ${e}`); @@ -506,33 +532,41 @@ async function serverGetReposForProjects(client: BitbucketClient, projects: stri if (e?.status === 404) { const warning = `Project ${project} not found or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "Bitbucket Server project was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - }; + return processPromiseResults(results); } -async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise<{repos: ServerRepository[], warnings: string[]}> { +async function serverGetRepos(client: BitbucketClient, repoList: string[]): Promise { const results = await Promise.allSettled(repoList.map(async (repo) => { const [project, repo_slug] = repo.split('/'); if (!project || !repo_slug) { const warning = `Invalid repo ${repo}`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "INVALID_TARGET", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Server repositories must use the project/repository format.", + }); + return []; } logger.debug(`Fetching repo ${repo_slug} for project ${project}...`); @@ -542,10 +576,7 @@ async function serverGetRepos(client: BitbucketClient, repoList: string[]): Prom const { data } = await client.apiClient.GET(path); return data; }, `repo ${repo}`, logger); - return { - type: 'valid' as const, - data: [data] - }; + return [data]; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch repo ${repo}: ${e}`); @@ -553,24 +584,26 @@ async function serverGetRepos(client: BitbucketClient, repoList: string[]): Prom if (e?.status === 404) { const warning = `Repo ${repo} not found in project ${project} or invalid access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Bitbucket Server repository was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - return { - repos, - warnings - }; + return processPromiseResults(results); } -async function serverGetAllRepos(client: BitbucketClient): Promise<{repos: ServerRepository[], warnings: string[]}> { +async function serverGetAllRepos(client: BitbucketClient): Promise { logger.debug(`Fetching all repos from Bitbucket Server...`); const path = `/rest/api/1.0/repos` as ServerGetRequestPath; const { durationMs, data } = await measure(async () => { @@ -583,7 +616,7 @@ async function serverGetAllRepos(client: BitbucketClient): Promise<{repos: Serve return fetchWithRetry(fetchFn, `all repos`, logger); }); logger.debug(`Found ${data.length} total repos in ${durationMs}ms.`); - return { repos: data, warnings: [] }; + return data; } export function serverShouldExcludeRepo(repo: BitbucketRepository, config: BitbucketConnectionConfig): boolean { diff --git a/packages/backend/src/connectionUtils.ts b/packages/backend/src/connectionUtils.ts index 074dfe7fe..69c1bd0eb 100644 --- a/packages/backend/src/connectionUtils.ts +++ b/packages/backend/src/connectionUtils.ts @@ -1,41 +1,11 @@ import * as Sentry from "@sentry/node"; -type ValidResult = { - type: 'valid'; - data: T[]; -}; - -type WarningResult = { - type: 'warning'; - warning: string; -}; - -type CustomResult = ValidResult | WarningResult; - export function processPromiseResults( - results: PromiseSettledResult>[], -): { - validItems: T[]; - warnings: string[]; -} { - const validItems: T[] = []; - const warnings: string[] = []; - - results.forEach(result => { - if (result.status === 'fulfilled') { - const value = result.value; - if (value.type === 'valid') { - validItems.push(...value.data); - } else { - warnings.push(value.warning); - } - } - }); - - return { - validItems, - warnings, - }; + results: PromiseSettledResult[], +): T[] { + return results.flatMap((result) => + result.status === "fulfilled" ? result.value : [] + ); } export function throwIfAnyFailed(results: PromiseSettledResult[]) { @@ -44,4 +14,4 @@ export function throwIfAnyFailed(results: PromiseSettledResult[]) { Sentry.captureException(failedResult.reason); throw failedResult.reason; } -} \ No newline at end of file +} diff --git a/packages/backend/src/gitea.test.ts b/packages/backend/src/gitea.test.ts new file mode 100644 index 000000000..7a669dfa6 --- /dev/null +++ b/packages/backend/src/gitea.test.ts @@ -0,0 +1,132 @@ +import type { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { status: 404 }); + + return { + userListRepos: vi.fn(async () => { + throw notFound; + }), + orgListRepos: vi.fn(async () => { + throw notFound; + }), + repoGet: vi.fn(async () => { + throw notFound; + }), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("gitea-js", () => ({ + giteaApi: () => ({ + users: { + userListRepos: mocks.userListRepos, + }, + orgs: { + orgListRepos: mocks.orgListRepos, + }, + repos: { + repoGet: mocks.repoGet, + }, + }), +})); + +vi.mock("./utils.js", () => ({ + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { getGiteaReposFromConfig } from './gitea'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("Gitea repository discovery", () => { + test("reports inaccessible configured targets as partial successes", async () => { + const config = { + type: "gitea", + orgs: ["missing-org"], + repos: ["missing-owner/missing-repo"], + users: ["missing-user"], + } satisfies GiteaConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getGiteaReposFromConfig(config) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: "missing-org", + }, + message: "Gitea organization was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: "missing-owner/missing-repo", + }, + message: "Gitea repository was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: "missing-user", + }, + message: "Gitea user was not found or is inaccessible.", + }, + ], + }); + }); + + test("reports malformed repositories returned by Gitea", async () => { + mocks.orgListRepos.mockResolvedValueOnce({ + data: [null, { id: 123 }], + headers: new Headers({ "x-total-count": "2" }), + } as never); + + const result = await collectRepositoryDiscoveryIssues(() => + getGiteaReposFromConfig({ + type: "gitea", + orgs: ["my-org"], + }) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "Gitea returned a null repository, so it was skipped.", + }, + { + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + subject: { + kind: "repository", + value: "123", + }, + message: "Gitea returned a repository without a full name, so it was skipped.", + }, + ], + }); + }); +}); diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts index 4ebdc2033..af7ed80ba 100644 --- a/packages/backend/src/gitea.ts +++ b/packages/backend/src/gitea.ts @@ -7,6 +7,7 @@ import fetch from 'cross-fetch'; import { Api, giteaApi, Repository as GiteaRepository, HttpResponse } from 'gitea-js'; import micromatch from 'micromatch'; import { processPromiseResults, throwIfAnyFailed } from './connectionUtils.js'; +import { reportRepositoryDiscoveryIssue } from './repositoryDiscoveryIssueContext.js'; import { measure } from './utils.js'; const logger = createLogger('gitea'); @@ -45,33 +46,42 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig) => }); let allRepos: GiteaRepository[] = []; - let allWarnings: string[] = []; if (config.orgs) { - const { repos, warnings } = await getReposForOrgs(config.orgs, api); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await getReposForOrgs(config.orgs, api)); } if (config.repos) { - const { repos, warnings } = await getRepos(config.repos, api); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await getRepos(config.repos, api)); } if (config.users) { - const { repos, warnings } = await getReposOwnedByUsers(config.users, api); - allRepos = allRepos.concat(repos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(await getReposOwnedByUsers(config.users, api)); } allRepos = allRepos.filter(repo => { if (repo === null || repo === undefined) { logger.warn(`Skipping null/undefined repository returned by the Gitea API`); + reportRepositoryDiscoveryIssue({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + message: "Gitea returned a null repository, so it was skipped.", + }); return false; } if (repo.full_name === undefined) { logger.warn(`Repository with undefined full_name found: repoId=${repo.id}`); + reportRepositoryDiscoveryIssue({ + code: "INVALID_PROVIDER_RESPONSE", + effect: "DISCOVERY_INCOMPLETE", + ...(repo.id !== undefined && repo.id !== null ? { + subject: { + kind: "repository" as const, + value: String(repo.id), + }, + } : {}), + message: "Gitea returned a repository without a full name, so it was skipped.", + }); return false; } return true; @@ -88,10 +98,7 @@ export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig) => }); logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; } const shouldExcludeRepo = ({ @@ -148,32 +155,30 @@ const getReposOwnedByUsers = async (users: string[], api: Api) => { ); logger.debug(`Found ${data.length} repos owned by user ${user} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); if (e?.status === 404) { const warning = `User ${user} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: user, + }, + message: "Gitea user was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } const getReposForOrgs = async (orgs: string[], api: Api) => { @@ -189,32 +194,30 @@ const getReposForOrgs = async (orgs: string[], api: Api) => { ); logger.debug(`Found ${data.length} repos for org ${org} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); if (e?.status === 404) { const warning = `Organization ${org} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "organization", + value: org, + }, + message: "Gitea organization was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } const getRepos = async (repoList: string[], api: Api) => { @@ -232,32 +235,30 @@ const getRepos = async (repoList: string[], api: Api) => { } logger.debug(`Found repo ${repo} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: [response.data] - }; + return [response.data]; } catch (e: any) { Sentry.captureException(e); if (e?.status === 404) { const warning = `Repository ${repo} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "repository", + value: repo, + }, + message: "Gitea repository was not found or is inaccessible.", + }); + return []; } throw e; } })); throwIfAnyFailed(results); - const { validItems: repos, warnings } = processPromiseResults(results); - - return { - repos, - warnings, - }; + return processPromiseResults(results); } // @see : https://docs.gitea.com/development/api-usage#pagination @@ -284,4 +285,4 @@ const paginate = async (request: (page: number) => Promise - result.status === "fulfilled" ? result.value : [] - ); + return processPromiseResults(results); } const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { @@ -428,9 +426,7 @@ const getReposForOrgs = async (orgs: string[], octokit: Octokit, signal: AbortSi }))); throwIfAnyFailed(results); - return results.flatMap((result) => - result.status === "fulfilled" ? result.value : [] - ); + return processPromiseResults(results); } const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSignal, url?: string) => { @@ -478,9 +474,7 @@ const getRepos = async (repoList: string[], octokit: Octokit, signal: AbortSigna }))); throwIfAnyFailed(results); - return results.flatMap((result) => - result.status === "fulfilled" ? result.value : [] - ); + return processPromiseResults(results); } export const shouldExcludeRepo = ({ diff --git a/packages/backend/src/gitlab.test.ts b/packages/backend/src/gitlab.test.ts index 2cfb4042a..31f75dcf1 100644 --- a/packages/backend/src/gitlab.test.ts +++ b/packages/backend/src/gitlab.test.ts @@ -1,6 +1,114 @@ -import { expect, test } from 'vitest'; -import { shouldExcludeProject } from './gitlab'; -import { ProjectSchema } from '@gitbeaker/rest'; +import type { ProjectSchema } from '@gitbeaker/rest'; +import type { GitlabConnectionConfig } from '@sourcebot/schemas/v3/gitlab.type'; +import { describe, expect, test, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const notFound = Object.assign(new Error("Not Found"), { + cause: { response: { status: 404 } }, + }); + + return { + groupsAllProjects: vi.fn(async () => { + throw notFound; + }), + usersAllProjects: vi.fn(async () => { + throw notFound; + }), + projectsShow: vi.fn(async () => { + throw notFound; + }), + projectsAll: vi.fn(async () => []), + }; +}); + +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + +vi.mock("@gitbeaker/rest", () => ({ + Gitlab: class { + Groups = { + allProjects: mocks.groupsAllProjects, + }; + Users = { + allProjects: mocks.usersAllProjects, + }; + Projects = { + all: mocks.projectsAll, + show: mocks.projectsShow, + }; + }, +})); + +vi.mock("./utils.js", () => ({ + fetchWithRetry: (routine: () => Promise) => routine(), + measure: async (routine: () => Promise) => ({ + durationMs: 1, + data: await routine(), + }), +})); + +import { collectRepositoryDiscoveryIssues } from "./repositoryDiscoveryIssueContext.js"; +import { getGitLabReposFromConfig, shouldExcludeProject } from './gitlab'; + +describe("GitLab repository discovery", () => { + test("reports unsupported configuration and inaccessible targets", async () => { + const config = { + type: "gitlab", + all: true, + groups: ["missing-group"], + users: ["missing-user"], + projects: ["missing-group/missing-project"], + } satisfies GitlabConnectionConfig; + + const result = await collectRepositoryDiscoveryIssues(() => + getGitLabReposFromConfig(config) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "all", + }, + message: "The all option is not supported for GitLab Cloud.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "group", + value: "missing-group", + }, + message: "GitLab group was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: "missing-user", + }, + message: "GitLab user was not found or is inaccessible.", + }, + { + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: "missing-group/missing-project", + }, + message: "GitLab project was not found or is inaccessible.", + }, + ], + }); + expect(mocks.projectsAll).not.toHaveBeenCalled(); + }); +}); test('shouldExcludeProject returns false when the project is not excluded.', () => { diff --git a/packages/backend/src/gitlab.ts b/packages/backend/src/gitlab.ts index 182657f21..6a0c57042 100644 --- a/packages/backend/src/gitlab.ts +++ b/packages/backend/src/gitlab.ts @@ -6,6 +6,7 @@ import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type"; import { env } from "@sourcebot/shared"; import micromatch from "micromatch"; import { processPromiseResults, throwIfAnyFailed } from "./connectionUtils.js"; +import { reportRepositoryDiscoveryIssue } from "./repositoryDiscoveryIssueContext.js"; import { fetchWithRetry, measure } from "./utils.js"; const logger = createLogger('gitlab'); @@ -50,7 +51,6 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = }); let allRepos: ProjectSchema[] = []; - let allWarnings: string[] = []; if (config.all === true) { if (hostname !== GITLAB_CLOUD_HOSTNAME) { @@ -72,7 +72,15 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = } else { const warning = `Ignoring option all:true in config : host is ${GITLAB_CLOUD_HOSTNAME}`; logger.warn(warning); - allWarnings = allWarnings.concat(warning); + reportRepositoryDiscoveryIssue({ + code: "UNSUPPORTED_CONFIGURATION", + effect: "CONFIGURATION_IGNORED", + subject: { + kind: "configuration", + value: "all", + }, + message: "The all option is not supported for GitLab Cloud.", + }); } } @@ -88,10 +96,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return fetchWithRetry(fetchFn, `group ${group}`, logger); }); logger.debug(`Found ${data.length} projects in group ${group} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch projects for group ${group}.`, e); @@ -99,10 +104,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = if (e?.cause?.response?.status === 404) { const warning = `Group ${group} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "group", + value: group, + }, + message: "GitLab group was not found or is inaccessible.", + }); + return []; } throw e; @@ -110,9 +121,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = })); throwIfAnyFailed(results); - const { validItems: validRepos, warnings } = processPromiseResults(results); - allRepos = allRepos.concat(validRepos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(processPromiseResults(results)); } if (config.users) { @@ -126,10 +135,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return fetchWithRetry(fetchFn, `user ${user}`, logger); }); logger.debug(`Found ${data.length} projects owned by user ${user} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data - }; + return data; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch projects for user ${user}.`, e); @@ -137,10 +143,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = if (e?.cause?.response?.status === 404) { const warning = `User ${user} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "user", + value: user, + }, + message: "GitLab user was not found or is inaccessible.", + }); + return []; } throw e; @@ -148,9 +160,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = })); throwIfAnyFailed(results); - const { validItems: validRepos, warnings } = processPromiseResults(results); - allRepos = allRepos.concat(validRepos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(processPromiseResults(results)); } if (config.projects) { @@ -162,10 +172,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = return fetchWithRetry(fetchFn, `project ${project}`, logger); }); logger.debug(`Found project ${project} in ${durationMs}ms.`); - return { - type: 'valid' as const, - data: [data] - }; + return [data]; } catch (e: any) { Sentry.captureException(e); logger.error(`Failed to fetch project ${project}.`, e); @@ -173,10 +180,16 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = if (e?.cause?.response?.status === 404) { const warning = `Project ${project} not found or no access`; logger.warn(warning); - return { - type: 'warning' as const, - warning - }; + reportRepositoryDiscoveryIssue({ + code: "NOT_FOUND_OR_INACCESSIBLE", + effect: "TARGET_SKIPPED", + subject: { + kind: "project", + value: project, + }, + message: "GitLab project was not found or is inaccessible.", + }); + return []; } throw e; @@ -184,9 +197,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = })); throwIfAnyFailed(results); - const { validItems: validRepos, warnings } = processPromiseResults(results); - allRepos = allRepos.concat(validRepos); - allWarnings = allWarnings.concat(warnings); + allRepos = allRepos.concat(processPromiseResults(results)); } let repos = allRepos @@ -204,10 +215,7 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig) = logger.debug(`Found ${repos.length} total repositories.`); - return { - repos, - warnings: allWarnings, - }; + return repos; } export const shouldExcludeProject = ({ diff --git a/packages/backend/src/repoCompileUtils.test.ts b/packages/backend/src/repoCompileUtils.test.ts index ee60d580c..df0d90f82 100644 --- a/packages/backend/src/repoCompileUtils.test.ts +++ b/packages/backend/src/repoCompileUtils.test.ts @@ -25,6 +25,7 @@ vi.mock('fs/promises', () => ({ import { isPathAValidGitRepoRoot, getOriginUrl, isUrlAValidGitRepo } from './git.js'; import { glob } from 'glob'; import fs from 'fs/promises'; +import { collectRepositoryDiscoveryIssues } from './repositoryDiscoveryIssueContext.js'; const mockedGlob = vi.mocked(glob); const mockedIsPathAValidGitRepoRoot = vi.mocked(isPathAValidGitRepoRoot); @@ -56,6 +57,30 @@ describe('compileGenericGitHostConfig_file', () => { expect(result).toHaveLength(0); }); + test('reports when a glob pattern matches no paths', async () => { + mockedGlob.mockResolvedValue([]); + + const result = await collectRepositoryDiscoveryIssues(() => + compileGenericGitHostConfig_file({ + type: 'git', + url: 'file:///path/to/**/repo', + }, 1) + ); + + expect(result).toEqual({ + value: [], + issues: [{ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: "/path/to/**/repo", + }, + message: "The configured path did not match any repository sources.", + }], + }); + }); + test('should return no repositories when path is a file, not a directory', async () => { mockedGlob.mockResolvedValue(['/path/to/a-file.txt']); mockedFsStat.mockResolvedValue({ isDirectory: () => false } as any); @@ -156,6 +181,52 @@ describe('compileGenericGitHostConfig_file', () => { expect(result).toHaveLength(1); }); + test('reports each invalid local repository source', async () => { + mockedGlob.mockResolvedValue([ + '/path/to/file', + '/path/to/not-a-repo', + '/path/to/no-origin', + ]); + mockedFsStat.mockImplementation(async (repoPath) => ({ + isDirectory: () => repoPath !== '/path/to/file', + }) as any); + mockedIsPathAValidGitRepoRoot.mockImplementation(async ({ path: repoPath }) => + repoPath !== '/path/to/not-a-repo' + ); + mockedGetOriginUrl.mockResolvedValue(null); + + const result = await collectRepositoryDiscoveryIssues(() => + compileGenericGitHostConfig_file({ + type: 'git', + url: 'file:///path/to/*', + }, 1) + ); + + expect(result).toEqual({ + value: [], + issues: [ + { + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { kind: "path", value: "/path/to/file" }, + message: "The configured path is not an accessible directory.", + }, + { + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { kind: "path", value: "/path/to/not-a-repo" }, + message: "The configured path is not a Git repository.", + }, + { + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { kind: "path", value: "/path/to/no-origin" }, + message: "The Git repository does not have a remote.origin.url.", + }, + ], + }); + }); + test('should decode URL-encoded characters in origin url pathname', async () => { mockedGlob.mockResolvedValue(['/path/to/repo-with-spaces']); mockedIsPathAValidGitRepoRoot.mockResolvedValue(true); @@ -198,6 +269,30 @@ describe('compileGenericGitHostConfig_url', () => { expect(result).toHaveLength(0); }); + test('reports an invalid remote repository source', async () => { + mockedIsUrlAValidGitRepo.mockResolvedValue(false); + + const result = await collectRepositoryDiscoveryIssues(() => + compileGenericGitHostConfig_url({ + type: 'git', + url: 'https://example.com/not-a-repo', + }, 1) + ); + + expect(result).toEqual({ + value: [], + issues: [{ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "url", + value: "https://example.com/not-a-repo", + }, + message: "The configured URL is not a Git repository.", + }], + }); + }); + test('should successfully compile with gitConfig when valid git repo url is found', async () => { mockedIsUrlAValidGitRepo.mockResolvedValue(true); diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts index ad8c728d2..cdc78d8e0 100644 --- a/packages/backend/src/repoCompileUtils.ts +++ b/packages/backend/src/repoCompileUtils.ts @@ -22,6 +22,7 @@ import GitUrlParse from 'git-url-parse'; import { RepoMetadata } from '@sourcebot/shared'; import { SINGLE_TENANT_ORG_ID } from './constants.js'; import pLimit from 'p-limit'; +import { reportRepositoryDiscoveryIssue } from './repositoryDiscoveryIssueContext.js'; export type RepoData = WithRequired; @@ -152,8 +153,7 @@ export const compileGitlabConfig = async ( config: GitlabConnectionConfig, connectionId: number): Promise => { - const gitlabReposResult = await getGitLabReposFromConfig(config); - const gitlabRepos = gitlabReposResult.repos; + const gitlabRepos = await getGitLabReposFromConfig(config); const hostUrl = (config.url ?? 'https://gitlab.com').replace(/\/+$/, ''); const webUrl = (config.webUrl ?? hostUrl).replace(/\/+$/, ''); @@ -236,8 +236,7 @@ export const compileGiteaConfig = async ( config: GiteaConnectionConfig, connectionId: number): Promise => { - const giteaReposResult = await getGiteaReposFromConfig(config); - const giteaRepos = giteaReposResult.repos; + const giteaRepos = await getGiteaReposFromConfig(config); const hostUrl = (config.url ?? 'https://gitea.com').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -383,8 +382,7 @@ export const compileBitbucketConfig = async ( config: BitbucketConnectionConfig, connectionId: number): Promise => { - const bitbucketReposResult = await getBitbucketReposFromConfig(config); - const bitbucketRepos = bitbucketReposResult.repos; + const bitbucketRepos = await getBitbucketReposFromConfig(config); const hostUrl = (config.url ?? 'https://bitbucket.org').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) @@ -576,6 +574,15 @@ export const compileGenericGitHostConfig_file = async ( if (repoPaths.length === 0) { const warning = `No paths matched the pattern '${configUrl.pathname}'. Please verify the path exists and is accessible.`; logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: configUrl.pathname, + }, + message: "The configured path did not match any repository sources.", + }); return repos; } @@ -586,6 +593,15 @@ export const compileGenericGitHostConfig_file = async ( if (!stat || !stat.isDirectory()) { const warning = `Skipping ${repoPath} - path is not a directory.`; logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: repoPath, + }, + message: "The configured path is not an accessible directory.", + }); return; } @@ -595,6 +611,15 @@ export const compileGenericGitHostConfig_file = async ( if (!isGitRepo) { const warning = `Skipping ${repoPath} - not a git repository.`; logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: repoPath, + }, + message: "The configured path is not a Git repository.", + }); return; } @@ -602,6 +627,15 @@ export const compileGenericGitHostConfig_file = async ( if (!origin) { const warning = `Skipping ${repoPath} - remote.origin.url not found in git config.`; logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "path", + value: repoPath, + }, + message: "The Git repository does not have a remote.origin.url.", + }); return; } @@ -677,6 +711,15 @@ export const compileGenericGitHostConfig_url = async ( if (!isGitRepo) { const warning = `Skipping ${remoteUrl.toString()} - not a git repository.`; logger.warn(warning); + reportRepositoryDiscoveryIssue({ + code: "INVALID_REPOSITORY_SOURCE", + effect: "TARGET_SKIPPED", + subject: { + kind: "url", + value: remoteUrl.toString(), + }, + message: "The configured URL is not a Git repository.", + }); return []; } @@ -728,8 +771,7 @@ export const compileAzureDevOpsConfig = async ( config: AzureDevOpsConnectionConfig, connectionId: number): Promise => { - const azureDevOpsReposResult = await getAzureDevOpsReposFromConfig(config); - const azureDevOpsRepos = azureDevOpsReposResult.repos; + const azureDevOpsRepos = await getAzureDevOpsReposFromConfig(config); const hostUrl = (config.url ?? 'https://dev.azure.com').replace(/\/+$/, ''); const repoNameRoot = new URL(hostUrl) From 3efad84eef46f8a5cc5895cecd00b4e9a56c1abd Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:12:36 -0700 Subject: [PATCH 24/34] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd9633879..14c8f2ca9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Migrated connection syncing, repository indexing, permission syncing, and background pruning from in-process managers and pollers to BullMQ workloads with retries and per-resource execution locking. [#1427](https://github.com/sourcebot-dev/sourcebot/pull/1427) +- Refactored the repository and connection list view to display useful debugging information to owners, such as failure reasons and logs. [#1608](https://github.com/sourcebot-dev/sourcebot/pull/1608) ### Fixed - Upgraded Next.js to 16.3.1 to bound memory retained by high-cardinality dynamic route cache entries. [#1594](https://github.com/sourcebot-dev/sourcebot/pull/1594) From b9805f077588b2a55674aa957e04dec32e6a850f Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:26:39 -0700 Subject: [PATCH 25/34] fix tests --- packages/shared/src/bullmqClient.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index d71b67bf1..424fc2a9e 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -168,7 +168,7 @@ describe("BullMQClient", () => { name: "connection-sync", data, opts: { - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, @@ -196,7 +196,7 @@ describe("BullMQClient", () => { { connectionId: 42 }, expect.objectContaining({ priority: 1, - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, @@ -237,7 +237,7 @@ describe("BullMQClient", () => { data: { connectionId: 42 }, opts: { priority: 10, - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, @@ -274,7 +274,7 @@ describe("BullMQClient", () => { template: { data: { connectionId: 42 }, opts: { - attempts: 4, + attempts: 2, backoff: { type: "exponential", delay: 30_000, From 36bc8e97a680d25ff7624752337f723d1a8fb260 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:36:32 -0700 Subject: [PATCH 26/34] feedback --- packages/web/src/app/(app)/layout.tsx | 16 ++- .../(server)/connection-sync-counts/route.ts | 12 +-- .../(server)/repository-sync-counts/route.ts | 12 +-- .../connectionSyncCounts.server.test.ts | 28 +++--- .../connectionSyncCounts.server.ts | 99 ++++++++++--------- .../repos/repositorySyncCounts.server.ts | 83 ++++++++-------- 6 files changed, 127 insertions(+), 123 deletions(-) diff --git a/packages/web/src/app/(app)/layout.tsx b/packages/web/src/app/(app)/layout.tsx index 2097e35d8..435678ef5 100644 --- a/packages/web/src/app/(app)/layout.tsx +++ b/packages/web/src/app/(app)/layout.tsx @@ -171,8 +171,8 @@ export default async function Layout(props: LayoutProps) { permissionSyncStatus !== null && !isServiceError(permissionSyncStatus) ? permissionSyncStatus.issues : []; - const repositorySyncCounts = role === OrgRole.OWNER - ? await getRepositorySyncCounts(org.id).catch((error) => { + const repositorySyncCountsResult = role === OrgRole.OWNER + ? await getRepositorySyncCounts().catch((error) => { console.error("Failed to load repository sync counts", error); return { firstTimeSyncingCount: 0, @@ -181,8 +181,12 @@ export default async function Layout(props: LayoutProps) { }; }) : { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }; - const connectionSyncCounts = role === OrgRole.OWNER - ? await getConnectionSyncCounts(org.id).catch((error) => { + if (isServiceError(repositorySyncCountsResult)) { + throw new ServiceErrorException(repositorySyncCountsResult); + } + const repositorySyncCounts = repositorySyncCountsResult; + const connectionSyncCountsResult = role === OrgRole.OWNER + ? await getConnectionSyncCounts().catch((error) => { console.error("Failed to load connection sync counts", error); return { firstTimeSyncingCount: 0, @@ -191,6 +195,10 @@ export default async function Layout(props: LayoutProps) { }; }) : { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }; + if (isServiceError(connectionSyncCountsResult)) { + throw new ServiceErrorException(connectionSyncCountsResult); + } + const connectionSyncCounts = connectionSyncCountsResult; const offlineLicense = getOfflineLicenseMetadata(); const license = offlineLicense diff --git a/packages/web/src/app/api/(server)/connection-sync-counts/route.ts b/packages/web/src/app/api/(server)/connection-sync-counts/route.ts index 264ecb1ac..55c6030e3 100644 --- a/packages/web/src/app/api/(server)/connection-sync-counts/route.ts +++ b/packages/web/src/app/api/(server)/connection-sync-counts/route.ts @@ -2,20 +2,12 @@ import { getConnectionSyncCounts } from "@/features/connections/connectionSyncCo import { apiHandler } from "@/lib/apiHandler"; import { serviceErrorResponse } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; -import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; import { sew } from "@/middleware/sew"; -import { OrgRole } from "@sourcebot/db"; import { StatusCodes } from "http-status-codes"; +// eslint-disable-next-line authz/require-auth-wrapper -- Authentication and owner authorization are enforced by getConnectionSyncCounts. export const GET = apiHandler(async () => { - const result = await sew(() => - withAuth(({ org, role }) => - withMinimumOrgRole(role, OrgRole.OWNER, () => - getConnectionSyncCounts(org.id) - ) - ) - ); + const result = await sew(() => getConnectionSyncCounts()); if (isServiceError(result)) { return serviceErrorResponse(result); diff --git a/packages/web/src/app/api/(server)/repository-sync-counts/route.ts b/packages/web/src/app/api/(server)/repository-sync-counts/route.ts index bdd73feac..b82e58063 100644 --- a/packages/web/src/app/api/(server)/repository-sync-counts/route.ts +++ b/packages/web/src/app/api/(server)/repository-sync-counts/route.ts @@ -1,21 +1,13 @@ import { apiHandler } from "@/lib/apiHandler"; import { serviceErrorResponse } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; -import { withAuth } from "@/middleware/withAuth"; -import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; import { sew } from "@/middleware/sew"; import { getRepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; -import { OrgRole } from "@sourcebot/db"; import { StatusCodes } from "http-status-codes"; +// eslint-disable-next-line authz/require-auth-wrapper -- Authentication and owner authorization are enforced by getRepositorySyncCounts. export const GET = apiHandler(async () => { - const result = await sew(() => - withAuth(({ org, role }) => - withMinimumOrgRole(role, OrgRole.OWNER, () => - getRepositorySyncCounts(org.id) - ) - ) - ); + const result = await sew(() => getRepositorySyncCounts()); if (isServiceError(result)) { return serviceErrorResponse(result); diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.test.ts b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts index ddf2616e6..c5c40bbc0 100644 --- a/packages/web/src/features/connections/connectionSyncCounts.server.test.ts +++ b/packages/web/src/features/connections/connectionSyncCounts.server.test.ts @@ -1,18 +1,24 @@ import type { WorkloadJob } from "@sourcebot/shared"; import { beforeEach, describe, expect, test, vi } from "vitest"; -const mocks = vi.hoisted(() => ({ - findMany: vi.fn(), - getJobs: vi.fn(), -})); +const mocks = vi.hoisted(() => { + const findMany = vi.fn(); + return { + findMany, + getJobs: vi.fn(), + prisma: { + connection: { findMany }, + }, + }; +}); vi.mock("server-only", () => ({})); -vi.mock("@/prisma", () => ({ - __unsafePrisma: { - connection: { - findMany: mocks.findMany, - }, - }, +vi.mock("@/middleware/withAuth", () => ({ + withAuth: (fn: (context: unknown) => unknown) => fn({ + org: { id: 42 }, + prisma: mocks.prisma, + role: "OWNER", + }), })); vi.mock("@/lib/bullmqClient", () => ({ getBullMQClient: () => ({ @@ -66,7 +72,7 @@ describe("getConnectionSyncCounts", () => { ["active-first-sync", job("active-first-sync", 7, "IN_PROGRESS")], ])); - await expect(getConnectionSyncCounts(42)).resolves.toEqual({ + await expect(getConnectionSyncCounts()).resolves.toEqual({ firstTimeSyncingCount: 2, failedCount: 1, warningCount: 2, diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.ts b/packages/web/src/features/connections/connectionSyncCounts.server.ts index dd85841a8..97511cd50 100644 --- a/packages/web/src/features/connections/connectionSyncCounts.server.ts +++ b/packages/web/src/features/connections/connectionSyncCounts.server.ts @@ -1,7 +1,9 @@ import "server-only"; import { getBullMQClient } from "@/lib/bullmqClient"; -import { __unsafePrisma } from "@/prisma"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { OrgRole } from "@sourcebot/db"; import { CONNECTION_QUEUE } from "@sourcebot/shared"; export interface ConnectionSyncCounts { @@ -10,54 +12,55 @@ export interface ConnectionSyncCounts { warningCount: number; } -export const getConnectionSyncCounts = async ( - orgId: number, -): Promise => { - const connections = await __unsafePrisma.connection.findMany({ - where: { orgId }, - select: { - id: true, - syncedAt: true, - latestSyncJobId: true, - firstSyncJobFinishedAt: true, - }, - }); - const latestJobIds = connections.flatMap((connection) => - connection.latestSyncJobId ? [connection.latestSyncJobId] : [] - ); - const latestJobs = await getBullMQClient().getJobs( - CONNECTION_QUEUE, - latestJobIds, - ); +export const getConnectionSyncCounts = async () => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const connections = await prisma.connection.findMany({ + where: { orgId: org.id }, + select: { + id: true, + syncedAt: true, + latestSyncJobId: true, + firstSyncJobFinishedAt: true, + }, + }); + const latestJobIds = connections.flatMap((connection) => + connection.latestSyncJobId ? [connection.latestSyncJobId] : [] + ); + const latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); - return connections.reduce((counts, connection) => { - if ( - connection.syncedAt === null - && connection.firstSyncJobFinishedAt === null - ) { - counts.firstTimeSyncingCount += 1; - } + return connections.reduce((counts, connection) => { + if ( + connection.syncedAt === null + && connection.firstSyncJobFinishedAt === null + ) { + counts.firstTimeSyncingCount += 1; + } - const latestJob = connection.latestSyncJobId - ? latestJobs.get(connection.latestSyncJobId) - : null; - if (!latestJob || latestJob.data.connectionId !== connection.id) { - return counts; - } + const latestJob = connection.latestSyncJobId + ? latestJobs.get(connection.latestSyncJobId) + : null; + if (!latestJob || latestJob.data.connectionId !== connection.id) { + return counts; + } - if (latestJob.status === "FAILED") { - if (connection.syncedAt) { - counts.warningCount += 1; - } else { - counts.failedCount += 1; - } - } else if ( - latestJob.status === "COMPLETED" - && latestJob.result?.outcome === "PARTIAL_SUCCESS" - ) { - counts.warningCount += 1; - } + if (latestJob.status === "FAILED") { + if (connection.syncedAt) { + counts.warningCount += 1; + } else { + counts.failedCount += 1; + } + } else if ( + latestJob.status === "COMPLETED" + && latestJob.result?.outcome === "PARTIAL_SUCCESS" + ) { + counts.warningCount += 1; + } - return counts; - }, { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }); -}; + return counts; + }, { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }); + }) + ); diff --git a/packages/web/src/features/repos/repositorySyncCounts.server.ts b/packages/web/src/features/repos/repositorySyncCounts.server.ts index e25bc8025..9a8c2affd 100644 --- a/packages/web/src/features/repos/repositorySyncCounts.server.ts +++ b/packages/web/src/features/repos/repositorySyncCounts.server.ts @@ -1,7 +1,9 @@ import "server-only"; import { getBullMQClient } from "@/lib/bullmqClient"; -import { __unsafePrisma } from "@/prisma"; +import { withAuth } from "@/middleware/withAuth"; +import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; +import { OrgRole } from "@sourcebot/db"; import { REPO_INDEX_QUEUE } from "@sourcebot/shared"; export interface RepositorySyncCounts { @@ -10,44 +12,45 @@ export interface RepositorySyncCounts { warningCount: number; } -export const getRepositorySyncCounts = async ( - orgId: number, -): Promise => { - const failedJobIds = await getBullMQClient().getFailedJobIds( - REPO_INDEX_QUEUE, - ); +export const getRepositorySyncCounts = async () => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + const failedJobIds = await getBullMQClient().getFailedJobIds( + REPO_INDEX_QUEUE, + ); - const [ - firstTimeSyncingCount, - failedCount, - warningCount, - ] = await Promise.all([ - __unsafePrisma.repo.count({ - where: { - orgId, - indexedAt: null, - firstIndexingJobFinishedAt: null, - }, - }), - __unsafePrisma.repo.count({ - where: { - orgId, - latestIndexingJobId: { in: failedJobIds }, - indexedAt: null, - }, - }), - __unsafePrisma.repo.count({ - where: { - orgId, - latestIndexingJobId: { in: failedJobIds }, - indexedAt: { not: null }, - }, - }), - ]); + const [ + firstTimeSyncingCount, + failedCount, + warningCount, + ] = await Promise.all([ + prisma.repo.count({ + where: { + orgId: org.id, + indexedAt: null, + firstIndexingJobFinishedAt: null, + }, + }), + prisma.repo.count({ + where: { + orgId: org.id, + latestIndexingJobId: { in: failedJobIds }, + indexedAt: null, + }, + }), + prisma.repo.count({ + where: { + orgId: org.id, + latestIndexingJobId: { in: failedJobIds }, + indexedAt: { not: null }, + }, + }), + ]); - return { - firstTimeSyncingCount, - failedCount, - warningCount, - }; -}; + return { + firstTimeSyncingCount, + failedCount, + warningCount, + }; + }) + ); From 81b8f03b1b96ad12dec1872a3ab3e00d7c6a83e1 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:36:53 -0700 Subject: [PATCH 27/34] feedback --- packages/web/src/app/api/(server)/job-logs/route.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/web/src/app/api/(server)/job-logs/route.ts b/packages/web/src/app/api/(server)/job-logs/route.ts index 38e9ace3e..5704bd493 100644 --- a/packages/web/src/app/api/(server)/job-logs/route.ts +++ b/packages/web/src/app/api/(server)/job-logs/route.ts @@ -50,6 +50,9 @@ export const POST = apiHandler(async (request) => { ); } + // TODO(SINGLE_TENANT_ORG_ID migration): Before supporting multiple organizations, + // map each queue's job data to an organization-scoped record and verify that the + // requested job belongs to the authenticated organization before returning logs. const result = await withAuth(({ role }) => withMinimumOrgRole(role, OrgRole.OWNER, () => getJobLogs( From 4a3b6b420befa5bf0e31a7f25423b60fdc09af7f Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:38:27 -0700 Subject: [PATCH 28/34] feedback --- .../web/src/app/(app)/settings/connections/page.tsx | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/web/src/app/(app)/settings/connections/page.tsx b/packages/web/src/app/(app)/settings/connections/page.tsx index a03faee0a..5b1a3988f 100644 --- a/packages/web/src/app/(app)/settings/connections/page.tsx +++ b/packages/web/src/app/(app)/settings/connections/page.tsx @@ -112,14 +112,10 @@ export default authenticatedPage(async ( connection.latestSyncJobId ? [connection.latestSyncJobId] : [] ); if (status === "all") { - try { - latestJobs = await getBullMQClient().getJobs( - CONNECTION_QUEUE, - latestJobIds, - ); - } catch (error) { - console.error("Failed to load latest connection sync jobs", error); - } + latestJobs = await getBullMQClient().getJobs( + CONNECTION_QUEUE, + latestJobIds, + ); } return ( From 17a3564b117d096ad670273cbc45532f12aea7a7 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:47:59 -0700 Subject: [PATCH 29/34] feedback --- packages/shared/src/queue.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 7036928f0..1ee263baf 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -46,6 +46,9 @@ export const JOB_PRIORITIES = { SCHEDULED: 10, } as const; +// BullMQ evaluates age-based cleanup only when another job reaches the same +// terminal state. A lone completed or failed job therefore remains available +// past this age; once a newer same-state job finishes, it replaces the old one. const TWO_WEEKS_IN_SECONDS = 14 * 24 * 60 * 60; export const DEFAULT_JOB_OPTIONS: JobOptions = { From e2a0e8f994f558247656855fb4bd14358c063fee Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 20:48:48 -0700 Subject: [PATCH 30/34] feedback --- .../(app)/settings/connections/components/connectionsTable.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx index 5343b3a17..f901f3065 100644 --- a/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx +++ b/packages/web/src/app/(app)/settings/connections/components/connectionsTable.tsx @@ -442,7 +442,7 @@ export const ConnectionsTable = ({ ...connection, syncedAt: status.syncedAt ? new Date(status.syncedAt) - : null, + : connection.syncedAt, latestJob: status.latestJob, showCompleted, }; From 03be1ee9edf975ba68c28b4fd2657064cdfcb4ff Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 21:25:52 -0700 Subject: [PATCH 31/34] move repo cleanup into sepreate queue with shared lock --- packages/backend/src/api.ts | 2 - packages/backend/src/configManager.test.ts | 8 +- .../src/connectionSyncWorkload.test.ts | 11 +- .../backend/src/connectionSyncWorkload.ts | 6 +- packages/backend/src/index.ts | 8 +- .../src/reconcileJobSchedulers.test.ts | 10 +- .../backend/src/reconcileJobSchedulers.ts | 6 +- .../backend/src/repoCleanupWorkload.test.ts | 185 +++++++++++++ packages/backend/src/repoCleanupWorkload.ts | 233 ++++++++++++++++ .../backend/src/repoIndexWorkload.test.ts | 210 ++------------- packages/backend/src/repoIndexWorkload.ts | 249 ++++-------------- packages/backend/src/repoLock.ts | 5 + packages/shared/src/bullmqClient.ts | 2 +- packages/shared/src/index.server.ts | 1 + packages/shared/src/queue.ts | 13 +- .../repos/components/reposTable.test.tsx | 16 +- .../app/(app)/repos/components/reposTable.tsx | 9 +- .../web/src/features/repos/actions.test.ts | 2 +- packages/web/src/features/repos/actions.ts | 2 +- 19 files changed, 536 insertions(+), 442 deletions(-) create mode 100644 packages/backend/src/repoCleanupWorkload.test.ts create mode 100644 packages/backend/src/repoCleanupWorkload.ts create mode 100644 packages/backend/src/repoLock.ts diff --git a/packages/backend/src/api.ts b/packages/backend/src/api.ts index 7e5fdc908..0e22f75e6 100644 --- a/packages/backend/src/api.ts +++ b/packages/backend/src/api.ts @@ -176,7 +176,6 @@ const scheduleAndTriggerRepoIndexing = async ({ reindexIntervalMs, { repoId, - type: "INDEX", }, { priority: JOB_PRIORITIES.SCHEDULED }, ); @@ -185,7 +184,6 @@ const scheduleAndTriggerRepoIndexing = async ({ "repo-index", { repoId, - type: "INDEX", }, { priority: JOB_PRIORITIES.INTERACTIVE }, ); diff --git a/packages/backend/src/configManager.test.ts b/packages/backend/src/configManager.test.ts index 9f28771b9..2d625ced0 100644 --- a/packages/backend/src/configManager.test.ts +++ b/packages/backend/src/configManager.test.ts @@ -217,13 +217,13 @@ describe("ConfigManager", () => { "repo-index-v1-2", ); expect(mocks.trigger).toHaveBeenCalledWith( - "repo-index", - { repoId: 1, type: "CLEANUP" }, + "repo-cleanup", + { repoId: 1 }, { priority: 10 }, ); expect(mocks.trigger).not.toHaveBeenCalledWith( - "repo-index", - { repoId: 2, type: "CLEANUP" }, + "repo-cleanup", + { repoId: 2 }, expect.anything(), ); expect(mocks.removeJobScheduler).toHaveBeenCalledWith( diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index 60d1d610c..21fda5d22 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -273,14 +273,13 @@ describe("connectionWorkload", () => { "repo-index", "repo-index-v1-4", 3_600_000, - { repoId: 4, type: "INDEX" }, + { repoId: 4 }, { priority: 10 }, ); expect(trigger).toHaveBeenCalledWith( "repo-index", { repoId: 4, - type: "INDEX", }, { priority: 5 }, ); @@ -502,7 +501,7 @@ describe("connectionWorkload repo sync helpers", () => { "repo-index", "repo-index-v1-1", 3_600_000, - { repoId: 1, type: "INDEX" }, + { repoId: 1 }, { priority: 10 }, ); expect(mocks.upsertJobScheduler).toHaveBeenNthCalledWith( @@ -510,7 +509,7 @@ describe("connectionWorkload repo sync helpers", () => { "repo-index", "repo-index-v1-4", 3_600_000, - { repoId: 4, type: "INDEX" }, + { repoId: 4 }, { priority: 10 }, ); expect(mocks.removeJobScheduler).toHaveBeenCalledWith( @@ -519,10 +518,9 @@ describe("connectionWorkload repo sync helpers", () => { ); expect(trigger).toHaveBeenNthCalledWith( 1, - "repo-index", + "repo-cleanup", { repoId: 2, - type: "CLEANUP", }, { priority: 10 }, ); @@ -531,7 +529,6 @@ describe("connectionWorkload repo sync helpers", () => { "repo-index", { repoId: 4, - type: "INDEX", }, { priority: 5 }, ); diff --git a/packages/backend/src/connectionSyncWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts index eaf05b059..c65e9abd2 100644 --- a/packages/backend/src/connectionSyncWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -332,7 +332,7 @@ export const reconcileRepoIndexWork = async ({ "repo-index", `repo-index-v1-${id}`, intervalMs, - { repoId: id, type: "INDEX" }, + { repoId: id }, { priority: JOB_PRIORITIES.SCHEDULED }, ), ), @@ -350,10 +350,9 @@ export const reconcileRepoIndexWork = async ({ await Promise.all( orphanedRepos.map(({ id }) => trigger( - "repo-index", + "repo-cleanup", { repoId: id, - type: "CLEANUP", }, { priority: JOB_PRIORITIES.SCHEDULED }, ), @@ -366,7 +365,6 @@ export const reconcileRepoIndexWork = async ({ "repo-index", { repoId: id, - type: "INDEX", }, { priority: JOB_PRIORITIES.INITIAL }, ), diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index efaa9447c..fc2ec78fc 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -13,7 +13,8 @@ import { prisma } from "./prisma.js"; import { PromClient } from './promClient.js'; import { redis } from "./redis.js"; import { createConnectionSyncWorkload } from "./connectionSyncWorkload.js"; -import { cleanupOrphanedRepoResources, createRepoIndexWorkload } from "./repoIndexWorkload.js"; +import { cleanupOrphanedRepoResources, createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; import { Api } from "./api.js"; import { createAccountPermissionSyncWorkload } from "./ee/accountPermissionSyncWorkload.js"; import { createRepoPermissionSyncWorkload } from "./ee/repoPermissionSyncWorkload.js"; @@ -59,6 +60,10 @@ const repoIndexWorkload = createRepoIndexWorkload({ db: prisma, settings, }); +const repoCleanupWorkload = createRepoCleanupWorkload({ + db: prisma, + settings, +}); const accountPermissionSyncWorkload = createAccountPermissionSyncWorkload({ db: prisma, settings, @@ -79,6 +84,7 @@ const auditLogPruneWorkload = createAuditLogPruneWorkload({ jobManager.register(connectionSyncWorkload); jobManager.register(repoIndexWorkload); +jobManager.register(repoCleanupWorkload); jobManager.register(accountPermissionSyncWorkload); jobManager.register(repoPermissionSyncWorkload); jobManager.register(attachmentPruneWorkload); diff --git a/packages/backend/src/reconcileJobSchedulers.test.ts b/packages/backend/src/reconcileJobSchedulers.test.ts index 223da8506..478daaec9 100644 --- a/packages/backend/src/reconcileJobSchedulers.test.ts +++ b/packages/backend/src/reconcileJobSchedulers.test.ts @@ -138,7 +138,7 @@ describe("reconcileJobSchedulers", () => { "repo-index", "repo-index-v1-42", 3_600_000, - { repoId: 42, type: "INDEX" }, + { repoId: 42 }, { priority: 10 }, ); expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( @@ -188,7 +188,7 @@ describe("reconcileJobSchedulers", () => { "repo-index", "repo-index-v1-42", 3_600_000, - { repoId: 42, type: "INDEX" }, + { repoId: 42 }, { priority: 10 }, ); expect(mocks.upsertJobScheduler).not.toHaveBeenCalledWith( @@ -203,8 +203,8 @@ describe("reconcileJobSchedulers", () => { "repo-index-v1-84", ); expect(mocks.trigger).toHaveBeenCalledWith( - "repo-index", - { repoId: 84, type: "CLEANUP" }, + "repo-cleanup", + { repoId: 84 }, { priority: 10 }, ); expect( @@ -241,7 +241,7 @@ describe("reconcileJobSchedulers", () => { "repo-index", "repo-index-v1-84", 3_600_000, - { repoId: 84, type: "INDEX" }, + { repoId: 84 }, { priority: 10 }, ); expect(mocks.removeJobScheduler).not.toHaveBeenCalledWith( diff --git a/packages/backend/src/reconcileJobSchedulers.ts b/packages/backend/src/reconcileJobSchedulers.ts index 9f21f1d57..d4e5f43e4 100644 --- a/packages/backend/src/reconcileJobSchedulers.ts +++ b/packages/backend/src/reconcileJobSchedulers.ts @@ -118,7 +118,7 @@ export const reconcileJobSchedulers = async ({ schedulerIdPrefix: "repo-index-v1-", targets: repos.map(({ id }) => ({ schedulerId: `repo-index-v1-${id}`, - data: { repoId: id, type: "INDEX" }, + data: { repoId: id }, })), schedule: settings.reindexIntervalMs, jobOptions: { priority: JOB_PRIORITIES.SCHEDULED }, @@ -150,8 +150,8 @@ export const reconcileJobSchedulers = async ({ await Promise.all( orphanedReposToCleanup.map(({ id }) => jobManager.trigger( - "repo-index", - { repoId: id, type: "CLEANUP" }, + "repo-cleanup", + { repoId: id }, { priority: JOB_PRIORITIES.SCHEDULED }, ), ), diff --git a/packages/backend/src/repoCleanupWorkload.test.ts b/packages/backend/src/repoCleanupWorkload.test.ts new file mode 100644 index 000000000..4a1e770e5 --- /dev/null +++ b/packages/backend/src/repoCleanupWorkload.test.ts @@ -0,0 +1,185 @@ +import type { PrismaClient } from "@sourcebot/db"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; + +const fsMocks = vi.hoisted(() => ({ + existsSync: vi.fn(), + readdir: vi.fn(), + rm: vi.fn(), +})); + +const lifecycleLogger = vi.hoisted(() => ({ + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), +})); + +vi.mock("@sourcebot/shared", async (importOriginal) => ({ + ...(await importOriginal()), + createLogger: vi.fn(() => lifecycleLogger), +})); + +vi.mock("fs", () => ({ + existsSync: fsMocks.existsSync, +})); + +vi.mock("fs/promises", () => ({ + readdir: fsMocks.readdir, + rm: fsMocks.rm, +})); + +const repoFindUnique = vi.fn(); +const repoDeleteMany = vi.fn(); +const repoUpdate = vi.fn(); + +const db = { + repo: { + findUnique: repoFindUnique, + deleteMany: repoDeleteMany, + update: repoUpdate, + }, +} as unknown as PrismaClient; + +const workload = createRepoCleanupWorkload({ + db, + settings: { + maxRepoIndexingJobConcurrency: 2, + } as never, +}); + +const processContext = { + data: { repoId: 42 }, + jobId: "job-1", + attemptsMade: 0, + maxAttempts: 2, + prisma: db, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger: vi.fn(), +}; + +const eligibleRepo = { + id: 42, + name: "github.com/acme/repo", + cloneUrl: "https://github.com/acme/repo.git", + external_codeHostType: "github", + orgId: 1, + indexedAt: null, + isAutoCleanupDisabled: false, + connections: [], +}; + +describe("repoCleanupWorkload", () => { + beforeEach(() => { + vi.clearAllMocks(); + fsMocks.existsSync.mockReturnValue(false); + fsMocks.readdir.mockResolvedValue([]); + fsMocks.rm.mockResolvedValue(undefined); + repoFindUnique.mockResolvedValue(eligibleRepo); + repoDeleteMany.mockResolvedValue({ count: 1 }); + repoUpdate.mockResolvedValue(undefined); + }); + + test("validates eligibility without updating indexing job state", async () => { + await workload.process(processContext); + + expect(repoFindUnique).toHaveBeenCalledWith({ + where: { id: 42 }, + include: { + connections: true, + }, + }); + expect(repoDeleteMany).toHaveBeenCalledWith({ + where: { + id: 42, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + expect(repoUpdate).not.toHaveBeenCalled(); + expect(workload.onStarted).toBeUndefined(); + expect(workload.onCompleted).toBeUndefined(); + expect(workload.onTerminalFailure).toBeUndefined(); + }); + + test("removes only shards belonging to the exact repository id", async () => { + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_420_v16.00000.zoekt", + ]); + + await workload.process(processContext); + + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_420_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test("finishes orphaned filesystem cleanup when a retry finds no repo", async () => { + repoFindUnique.mockResolvedValue(null); + fsMocks.existsSync.mockReturnValue(true); + fsMocks.readdir.mockResolvedValue([ + "1_42_v16.00000.zoekt", + "1_99_v16.00000.zoekt", + ]); + + await workload.process(processContext); + + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringMatching(/repos\/42$/), + { recursive: true, force: true }, + ); + expect(fsMocks.rm).toHaveBeenCalledWith( + expect.stringContaining("1_42_v16.00000.zoekt"), + { force: true }, + ); + expect(fsMocks.rm).not.toHaveBeenCalledWith( + expect.stringContaining("1_99_v16.00000.zoekt"), + expect.anything(), + ); + }); + + test.each([ + { + name: "automatic cleanup is disabled", + repo: { ...eligibleRepo, isAutoCleanupDisabled: true }, + reason: "automatic cleanup is disabled", + }, + { + name: "the repository was reattached", + repo: { ...eligibleRepo, connections: [{}] }, + reason: "repository has been reattached to a connection", + }, + ])("skips CLEANUP when $name", async ({ repo, reason }) => { + repoFindUnique.mockResolvedValue(repo); + + await workload.process(processContext); + + expect(repoDeleteMany).not.toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.debug).toHaveBeenCalledWith( + `Skipping CLEANUP job for repo 42: ${reason}`, + ); + }); + + test("revalidates cleanup eligibility when atomically deleting the repo", async () => { + repoDeleteMany.mockResolvedValue({ count: 0 }); + + await workload.process(processContext); + + expect(repoDeleteMany).toHaveBeenCalled(); + expect(fsMocks.readdir).not.toHaveBeenCalled(); + expect(lifecycleLogger.debug).toHaveBeenCalledWith( + "Skipping CLEANUP job for repo 42: repository is no longer eligible for cleanup", + ); + }); +}); diff --git a/packages/backend/src/repoCleanupWorkload.ts b/packages/backend/src/repoCleanupWorkload.ts new file mode 100644 index 000000000..d0fedbe81 --- /dev/null +++ b/packages/backend/src/repoCleanupWorkload.ts @@ -0,0 +1,233 @@ +import type { PrismaClient, Repo } from "@sourcebot/db"; +import { + createLogger, + getRepoIdFromPath, + getRepoPath, + REPO_CLEANUP_QUEUE, +} from "@sourcebot/shared"; +import { existsSync } from "fs"; +import { readdir, rm } from "fs/promises"; +import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from "./constants.js"; +import { REPOSITORY_EXECUTION_LOCK } from "./repoLock.js"; +import type { Settings, Workload } from "./types.js"; +import { getRepoIdFromShardFileName } from "./utils.js"; + +const logger = createLogger("repo-cleanup-workload"); + +interface Props { + db: PrismaClient; + settings: Settings; +} + +export const createRepoCleanupWorkload = ({ + db, + settings, +}: Props): Workload<"repo-cleanup"> => ({ + queueSpec: REPO_CLEANUP_QUEUE, + concurrency: settings.maxRepoIndexingJobConcurrency, + // Cleanup and indexing use separate queues, but must never operate on the + // same repository path or search index concurrently. + executionLock: REPOSITORY_EXECUTION_LOCK, + process: async ({ data: { repoId }, signal }) => { + signal.throwIfAborted(); + const start = await prepareRepoCleanupJob({ db, repoId }); + + if (start.action === "skip") { + logger.debug( + `Skipping CLEANUP job for repo ${repoId}: ${start.reason}`, + ); + + if (start.repoMissing) { + signal.throwIfAborted(); + await cleanupOrphanedRepoResourcesForRepoId(repoId); + } + return; + } + + signal.throwIfAborted(); + const { repo } = start; + logger.debug(`Running CLEANUP job for repo ${repo.name} (id: ${repo.id})`); + + const { count } = await db.repo.deleteMany({ + where: { + id: repo.id, + isAutoCleanupDisabled: false, + connections: { + none: {}, + }, + }, + }); + + if (count === 0) { + logger.debug( + `Skipping CLEANUP job for repo ${repo.id}: repository is no longer eligible for cleanup`, + ); + return; + } + + signal.throwIfAborted(); + await cleanupRepository(repo); + }, +}); + +type RepoCleanupStartDecision = + | { + action: "run"; + repo: Repo; + } + | { + action: "skip"; + reason: string; + repoMissing: boolean; + }; + +const prepareRepoCleanupJob = async ({ + db, + repoId, +}: { + db: PrismaClient; + repoId: number; +}): Promise => { + const repo = await db.repo.findUnique({ + where: { id: repoId }, + include: { + connections: true, + }, + }); + + if (!repo) { + return { + action: "skip", + reason: "repository no longer exists", + repoMissing: true, + }; + } + + if (repo.isAutoCleanupDisabled) { + return { + action: "skip", + reason: "automatic cleanup is disabled", + repoMissing: false, + }; + } + + if (repo.connections.length > 0) { + return { + action: "skip", + reason: "repository has been reattached to a connection", + repoMissing: false, + }; + } + + return { + action: "run", + repo, + }; +}; + +const cleanupRepository = async (repo: Repo) => { + const { path: repoPath, isReadOnly } = getRepoPath(repo); + if (existsSync(repoPath) && !isReadOnly) { + logger.debug(`Deleting repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + const files = (await readdir(INDEX_CACHE_DIR)).filter( + (file) => getRepoIdFromShardFileName(file) === repo.id, + ); + for (const file of files) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +const cleanupOrphanedRepoResourcesForRepoId = async (repoId: number) => { + const repoPath = `${REPOS_CACHE_DIR}/${repoId}`; + if (existsSync(repoPath)) { + logger.debug(`Deleting orphaned repo directory ${repoPath}`); + await rm(repoPath, { recursive: true, force: true }); + } + + if (!existsSync(INDEX_CACHE_DIR)) { + return; + } + + const shardFiles = (await readdir(INDEX_CACHE_DIR)).filter( + (file) => getRepoIdFromShardFileName(file) === repoId, + ); + for (const file of shardFiles) { + const filePath = `${INDEX_CACHE_DIR}/${file}`; + logger.debug(`Deleting orphaned shard file ${filePath}`); + await rm(filePath, { force: true }); + } +}; + +// Scans the repos and index directories on disk and removes any entries +// that have no corresponding Repo record in the database. This handles +// edge cases where the DB and disk resources are out of sync. +export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { + // --- Repo directories --- + // Dirs are named by repoId: DATA_CACHE_DIR/repos// + if (existsSync(REPOS_CACHE_DIR)) { + const entries = await readdir(REPOS_CACHE_DIR); + const repoIdToPath = new Map(); + for (const entry of entries) { + const repoPath = `${REPOS_CACHE_DIR}/${entry}`; + const repoId = getRepoIdFromPath(repoPath); + if (repoId !== undefined) { + repoIdToPath.set(repoId, repoPath); + } + } + + if (repoIdToPath.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToPath.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map((repo) => repo.id)); + for (const [repoId, repoPath] of repoIdToPath) { + if (!existingIds.has(repoId)) { + logger.debug( + `Removing orphaned repo directory with no DB record: ${repoPath}`, + ); + await rm(repoPath, { recursive: true, force: true }); + } + } + } + } + + // --- Index shards --- + // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt + if (existsSync(INDEX_CACHE_DIR)) { + const entries = await readdir(INDEX_CACHE_DIR); + const repoIdToShards = new Map(); + for (const entry of entries) { + const repoId = getRepoIdFromShardFileName(entry); + if (repoId !== undefined) { + const shards = repoIdToShards.get(repoId) ?? []; + shards.push(entry); + repoIdToShards.set(repoId, shards); + } + } + + if (repoIdToShards.size > 0) { + const existingRepos = await db.repo.findMany({ + where: { id: { in: [...repoIdToShards.keys()] } }, + select: { id: true }, + }); + const existingIds = new Set(existingRepos.map((repo) => repo.id)); + for (const [repoId, shards] of repoIdToShards) { + if (!existingIds.has(repoId)) { + for (const entry of shards) { + const shardPath = `${INDEX_CACHE_DIR}/${entry}`; + logger.debug( + `Removing orphaned index shard with no DB record: ${shardPath}`, + ); + await rm(shardPath, { force: true }); + } + } + } + } + } +}; diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index bed614533..91715f839 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -1,11 +1,7 @@ import type { PrismaClient } from "@sourcebot/db"; import { beforeEach, describe, expect, test, vi } from "vitest"; - -const fsMocks = vi.hoisted(() => ({ - existsSync: vi.fn(), - readdir: vi.fn(), - rm: vi.fn(), -})); +import { createRepoCleanupWorkload } from "./repoCleanupWorkload.js"; +import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; const lifecycleLogger = vi.hoisted(() => ({ debug: vi.fn(), @@ -19,19 +15,7 @@ vi.mock("@sourcebot/shared", async (importOriginal) => ({ createLogger: vi.fn(() => lifecycleLogger), })); -vi.mock("fs", () => ({ - existsSync: fsMocks.existsSync, -})); - -vi.mock("fs/promises", () => ({ - readdir: fsMocks.readdir, - rm: fsMocks.rm, -})); - -import { createRepoIndexWorkload } from "./repoIndexWorkload.js"; - const repoFindUnique = vi.fn(); -const repoDeleteMany = vi.fn(); const repoUpdate = vi.fn(); const repoUpdateMany = vi.fn(); @@ -47,22 +31,20 @@ const transaction = vi.fn(async (callback: (tx: unknown) => Promise) => const db = { $transaction: transaction, repo: { - deleteMany: repoDeleteMany, updateMany: repoUpdateMany, }, } as unknown as PrismaClient; -const workload = createRepoIndexWorkload({ - db, - settings: { - maxRepoIndexingJobConcurrency: 2, - } as never, -}); +const settings = { + maxRepoIndexingJobConcurrency: 2, +} as never; + +const workload = createRepoIndexWorkload({ db, settings }); +const cleanupWorkload = createRepoCleanupWorkload({ db, settings }); const lifecycleContext = { data: { repoId: 42, - type: "INDEX" as const, }, jobId: "job-1", attemptsMade: 0, @@ -77,42 +59,25 @@ const processContext = { trigger: vi.fn(), }; -const eligibleRepo = { - id: 42, - name: "github.com/acme/repo", - cloneUrl: "https://github.com/acme/repo.git", - external_codeHostType: "github", - orgId: 1, - indexedAt: null, - isAutoCleanupDisabled: false, - connections: [], -}; - describe("repoIndexWorkload", () => { beforeEach(() => { vi.clearAllMocks(); - fsMocks.existsSync.mockReturnValue(false); - fsMocks.readdir.mockResolvedValue([]); - fsMocks.rm.mockResolvedValue(undefined); - repoFindUnique.mockResolvedValue(eligibleRepo); - repoDeleteMany.mockResolvedValue({ count: 1 }); + repoFindUnique.mockResolvedValue(null); repoUpdate.mockResolvedValue(undefined); repoUpdateMany.mockResolvedValue({ count: 1 }); }); - test("uses the same repository execution lock for INDEX and CLEANUP", () => { - expect(workload.executionLock).toBeDefined(); + test("shares its repository execution lock with cleanup", () => { + expect(workload.executionLock).toBe(cleanupWorkload.executionLock); expect( - workload.executionLock?.resource({ repoId: 42, type: "INDEX" }), - ).toBe("sourcebot:lock:repo:42"); - expect( - workload.executionLock?.resource({ repoId: 42, type: "CLEANUP" }), + workload.executionLock?.resource({ repoId: 42 }), ).toBe("sourcebot:lock:repo:42"); + expect(cleanupWorkload.executionLock?.resource({ repoId: 42 })).toBe( + "sourcebot:lock:repo:42", + ); expect(workload.executionLock?.durationMs).toBe(60_000); - expect(workload.queueSpec.dedupKey).toBeUndefined(); - expect(workload.onStarted).toBeUndefined(); - expect(workload.onCompleted).toBeTypeOf("function"); - expect(workload.onTerminalFailure).toBeTypeOf("function"); + expect(workload.queueSpec.name).toBe("repo-index"); + expect(cleanupWorkload.queueSpec.name).toBe("repo-cleanup"); }); test("records the first successful indexing job terminal state", async () => { @@ -146,153 +111,12 @@ describe("repoIndexWorkload", () => { }); }); - test("does not mark cleanup jobs as an initial indexing attempt", async () => { - await workload.onCompleted?.({ - ...lifecycleContext, - data: { repoId: 42, type: "CLEANUP" }, - }, undefined); - - expect(repoUpdateMany).not.toHaveBeenCalled(); - }); - - test("validates state and records the latest job inside process", async () => { - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(repoFindUnique).toHaveBeenCalledWith({ - where: { id: 42 }, - include: { - connections: { - include: { - connection: true, - }, - }, - }, - }); - expect(repoUpdate).toHaveBeenCalledWith({ - where: { - id: 42, - }, - data: { - latestIndexingJobId: "job-1", - }, - }); - expect(repoDeleteMany).toHaveBeenCalledWith({ - where: { - id: 42, - isAutoCleanupDisabled: false, - connections: { - none: {}, - }, - }, - }); - }); - - test("cleanup removes only shards belonging to the exact repository id", async () => { - fsMocks.readdir.mockResolvedValue([ - "1_42_v16.00000.zoekt", - "1_420_v16.00000.zoekt", - ]); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(fsMocks.rm).toHaveBeenCalledWith( - expect.stringContaining("1_42_v16.00000.zoekt"), - { force: true }, - ); - expect(fsMocks.rm).not.toHaveBeenCalledWith( - expect.stringContaining("1_420_v16.00000.zoekt"), - expect.anything(), - ); - }); - test("skips an INDEX job when the repository no longer exists", async () => { - repoFindUnique.mockResolvedValue(null); - await workload.process(processContext); expect(repoUpdate).not.toHaveBeenCalled(); - expect(repoDeleteMany).not.toHaveBeenCalled(); - expect(fsMocks.readdir).not.toHaveBeenCalled(); expect(lifecycleLogger.debug).toHaveBeenCalledWith( "Skipping INDEX job for repo 42: repository no longer exists", ); }); - - test("finishes orphaned filesystem cleanup when a CLEANUP retry finds no repo", async () => { - repoFindUnique.mockResolvedValue(null); - fsMocks.existsSync.mockReturnValue(true); - fsMocks.readdir.mockResolvedValue([ - "1_42_v16.00000.zoekt", - "1_99_v16.00000.zoekt", - ]); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(repoUpdate).not.toHaveBeenCalled(); - expect(fsMocks.rm).toHaveBeenCalledWith( - expect.stringMatching(/repos\/42$/), - { recursive: true, force: true }, - ); - expect(fsMocks.rm).toHaveBeenCalledWith( - expect.stringContaining("1_42_v16.00000.zoekt"), - { force: true }, - ); - expect(fsMocks.rm).not.toHaveBeenCalledWith( - expect.stringContaining("1_99_v16.00000.zoekt"), - expect.anything(), - ); - }); - - test.each([ - { - name: "automatic cleanup is disabled", - repo: { ...eligibleRepo, isAutoCleanupDisabled: true }, - reason: "automatic cleanup is disabled", - }, - { - name: "the repository was reattached", - repo: { ...eligibleRepo, connections: [{}] }, - reason: "repository has been reattached to a connection", - }, - ])("skips CLEANUP when $name", async ({ repo, reason }) => { - repoFindUnique.mockResolvedValue(repo); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(repoUpdate).not.toHaveBeenCalled(); - expect(repoDeleteMany).not.toHaveBeenCalled(); - expect(fsMocks.readdir).not.toHaveBeenCalled(); - expect(lifecycleLogger.debug).toHaveBeenCalledWith( - `Skipping CLEANUP job for repo 42: ${reason}`, - ); - }); - - test("revalidates cleanup eligibility when atomically deleting the repo", async () => { - repoDeleteMany.mockResolvedValue({ count: 0 }); - - await workload.process({ - ...processContext, - data: { repoId: 42, type: "CLEANUP" }, - }); - - expect(repoUpdate).toHaveBeenCalled(); - expect(repoDeleteMany).toHaveBeenCalled(); - expect(fsMocks.readdir).not.toHaveBeenCalled(); - expect(lifecycleLogger.debug).toHaveBeenCalledWith( - "Skipping CLEANUP job for repo 42: repository is no longer eligible for cleanup", - ); - }); - }); diff --git a/packages/backend/src/repoIndexWorkload.ts b/packages/backend/src/repoIndexWorkload.ts index 824da1c5d..2a15d90aa 100644 --- a/packages/backend/src/repoIndexWorkload.ts +++ b/packages/backend/src/repoIndexWorkload.ts @@ -1,18 +1,17 @@ -import { PrismaClient, Repo } from "@sourcebot/db"; -import { createLogger, getRepoPath, getRepoIdFromPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { PrismaClient } from "@sourcebot/db"; +import { createLogger, getRepoPath, RepoMetadata, repoMetadataSchema, REPO_INDEX_QUEUE } from "@sourcebot/shared"; import { existsSync } from 'fs'; -import { readdir, rm } from 'fs/promises'; +import { rm } from 'fs/promises'; import micromatch from 'micromatch'; -import { INDEX_CACHE_DIR, REPOS_CACHE_DIR } from './constants.js'; import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig, writeCommitGraph } from './git.js'; import { captureEvent } from './posthog.js'; +import { REPOSITORY_EXECUTION_LOCK } from "./repoLock.js"; import { RepoWithConnections, Settings, Workload } from "./types.js"; -import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, measure } from './utils.js'; +import { getAuthCredentialsForRepo, measure } from './utils.js'; import { cleanupTempShards, indexGitRepository } from './zoekt.js'; const LOG_TAG = 'repo-index-workload'; const logger = createLogger(LOG_TAG); -const REPO_INDEX_LOCK_DURATION_MS = 60_000; interface Props { db: PrismaClient; @@ -25,96 +24,63 @@ export const createRepoIndexWorkload = ({ }: Props): Workload<'repo-index'> => ({ queueSpec: REPO_INDEX_QUEUE, concurrency: settings.maxRepoIndexingJobConcurrency, - // Indexing and cleanup share this lock because both operate on the same - // repository path and search index. - executionLock: { - resource: ({ repoId }) => `sourcebot:lock:repo:${repoId}`, - durationMs: REPO_INDEX_LOCK_DURATION_MS, - }, + executionLock: REPOSITORY_EXECUTION_LOCK, process: async ({ data, jobId, signal }) => { signal.throwIfAborted(); + const start = await prepareRepoIndexJob({ db, repoId: data.repoId, - type: data.type, jobId, }); if (start.action === "skip") { logger.debug( - `Skipping ${data.type} job for repo ${data.repoId}: ${start.reason}`, + `Skipping INDEX job for repo ${data.repoId}: ${start.reason}`, ); - - if (data.type === "CLEANUP" && start.repoMissing) { - signal.throwIfAborted(); - await cleanupOrphanedRepoResourcesForRepoId(data.repoId); - } return; } signal.throwIfAborted(); const { repo } = start; - logger.debug(`Running ${data.type} job for repo ${repo.name} (id: ${repo.id})`); - - if (data.type === "CLEANUP") { - signal.throwIfAborted(); - const { count } = await db.repo.deleteMany({ - where: { - id: repo.id, - isAutoCleanupDisabled: false, - connections: { - none: {}, - }, - }, - }); + logger.debug(`Running INDEX job for repo ${repo.name} (id: ${repo.id})`); - if (count === 0) { - logger.debug( - `Skipping CLEANUP job for repo ${repo.id}: repository is no longer eligible for cleanup`, - ); - return; - } - - signal.throwIfAborted(); - await cleanupRepository(repo); - } else { - const isFirstIndex = repo.indexedAt === null; - const revisions = await indexRepository(db, settings, repo, signal); - signal.throwIfAborted(); - const { path: repoPath } = getRepoPath(repo); - const isEmpty = await isRepoEmpty({ path: repoPath }); - const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ - path: repoPath, - refName: 'HEAD', - }); - const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); - const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); - const currentRepo = await db.repo.findUniqueOrThrow({ - where: { id: repo.id }, - select: { metadata: true }, - }); + const isFirstIndex = repo.indexedAt === null; + const revisions = await indexRepository(db, settings, repo, signal); + signal.throwIfAborted(); + const { path: repoPath } = getRepoPath(repo); + const isEmpty = await isRepoEmpty({ path: repoPath }); + const commitHash = isEmpty ? undefined : await getCommitHashForRefName({ + path: repoPath, + refName: 'HEAD', + }); + const pushedAt = await getLatestCommitTimestamp({ path: repoPath }); + const defaultBranch = await getLocalDefaultBranch({ path: repoPath }); + const currentRepo = await db.repo.findUniqueOrThrow({ + where: { id: repo.id }, + select: { metadata: true }, + }); - signal.throwIfAborted(); - await db.repo.update({ - where: { id: repo.id }, - data: { - indexedAt: new Date(), - indexedCommitHash: commitHash, - pushedAt, - metadata: { - ...(currentRepo.metadata as RepoMetadata), - indexedRevisions: revisions, - } satisfies RepoMetadata, - defaultBranch, - }, - }); + signal.throwIfAborted(); + await db.repo.update({ + where: { id: repo.id }, + data: { + indexedAt: new Date(), + indexedCommitHash: commitHash, + pushedAt, + metadata: { + ...(currentRepo.metadata as RepoMetadata), + indexedRevisions: revisions, + } satisfies RepoMetadata, + defaultBranch, + }, + }); - if (isFirstIndex) { - captureEvent('backend_repo_first_indexed', { - repoId: repo.id, - type: repo.external_codeHostType, - }); - } + if (isFirstIndex) { + captureEvent('backend_repo_first_indexed', { + repoId: repo.id, + type: repo.external_codeHostType, + }); } }, onCompleted: async ({ data }) => { @@ -127,12 +93,8 @@ export const createRepoIndexWorkload = ({ const markFirstIndexingJobFinished = async ( db: PrismaClient, - data: { repoId: number; type: "INDEX" | "CLEANUP" }, + data: { repoId: number }, ) => { - if (data.type !== "INDEX") { - return; - } - await db.repo.updateMany({ where: { id: data.repoId, @@ -152,18 +114,15 @@ type RepoIndexStartDecision = | { action: "skip"; reason: string; - repoMissing: boolean; }; const prepareRepoIndexJob = async ({ db, repoId, - type, jobId, }: { db: PrismaClient; repoId: number; - type: "INDEX" | "CLEANUP"; jobId: string; }): Promise => db.$transaction(async (tx) => { @@ -182,23 +141,6 @@ const prepareRepoIndexJob = async ({ return { action: "skip", reason: "repository no longer exists", - repoMissing: true, - }; - } - - if (type === "CLEANUP" && repo.isAutoCleanupDisabled) { - return { - action: "skip", - reason: "automatic cleanup is disabled", - repoMissing: false, - }; - } - - if (type === "CLEANUP" && repo.connections.length > 0) { - return { - action: "skip", - reason: "repository has been reattached to a connection", - repoMissing: false, }; } @@ -407,106 +349,3 @@ const indexRepository = async ( return revisions; }; - -const cleanupRepository = async (repo: Repo) => { - const { path: repoPath, isReadOnly } = getRepoPath(repo); - if (existsSync(repoPath) && !isReadOnly) { - logger.debug(`Deleting repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - const files = (await readdir(INDEX_CACHE_DIR)).filter(file => getRepoIdFromShardFileName(file) === repo.id); - for (const file of files) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting shard file ${filePath}`); - await rm(filePath, { force: true }); - } -}; - -const cleanupOrphanedRepoResourcesForRepoId = async ( - repoId: number, -) => { - const repoPath = `${REPOS_CACHE_DIR}/${repoId}`; - if (existsSync(repoPath)) { - logger.debug(`Deleting orphaned repo directory ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - - if (!existsSync(INDEX_CACHE_DIR)) { - return; - } - - const shardFiles = (await readdir(INDEX_CACHE_DIR)).filter( - (file) => getRepoIdFromShardFileName(file) === repoId, - ); - for (const file of shardFiles) { - const filePath = `${INDEX_CACHE_DIR}/${file}`; - logger.debug(`Deleting orphaned shard file ${filePath}`); - await rm(filePath, { force: true }); - } -}; - -// Scans the repos and index directories on disk and removes any entries -// that have no corresponding Repo record in the database. This handles -// edge cases where the DB and disk resources are out of sync. -export const cleanupOrphanedRepoResources = async (db: PrismaClient) => { - // --- Repo directories --- - // Dirs are named by repoId: DATA_CACHE_DIR/repos// - if (existsSync(REPOS_CACHE_DIR)) { - const entries = await readdir(REPOS_CACHE_DIR); - const repoIdToPath = new Map(); - for (const entry of entries) { - const repoPath = `${REPOS_CACHE_DIR}/${entry}`; - const repoId = getRepoIdFromPath(repoPath); - if (repoId !== undefined) { - repoIdToPath.set(repoId, repoPath); - } - } - - if (repoIdToPath.size > 0) { - const existingRepos = await db.repo.findMany({ - where: { id: { in: [...repoIdToPath.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, repoPath] of repoIdToPath) { - if (!existingIds.has(repoId)) { - logger.debug(`Removing orphaned repo directory with no DB record: ${repoPath}`); - await rm(repoPath, { recursive: true, force: true }); - } - } - } - } - - // --- Index shards --- - // Shard files are prefixed with _: DATA_CACHE_DIR/index/__*.zoekt - if (existsSync(INDEX_CACHE_DIR)) { - const entries = await readdir(INDEX_CACHE_DIR); - const repoIdToShards = new Map(); - for (const entry of entries) { - const repoId = getRepoIdFromShardFileName(entry); - if (repoId !== undefined) { - const shards = repoIdToShards.get(repoId) ?? []; - shards.push(entry); - repoIdToShards.set(repoId, shards); - } - } - - if (repoIdToShards.size > 0) { - const existingRepos = await db.repo.findMany({ - where: { id: { in: [...repoIdToShards.keys()] } }, - select: { id: true }, - }); - const existingIds = new Set(existingRepos.map(r => r.id)); - for (const [repoId, shards] of repoIdToShards) { - if (!existingIds.has(repoId)) { - for (const entry of shards) { - const shardPath = `${INDEX_CACHE_DIR}/${entry}`; - logger.debug(`Removing orphaned index shard with no DB record: ${shardPath}`); - await rm(shardPath, { force: true }); - } - } - } - } - } -}; diff --git a/packages/backend/src/repoLock.ts b/packages/backend/src/repoLock.ts new file mode 100644 index 000000000..c55c30585 --- /dev/null +++ b/packages/backend/src/repoLock.ts @@ -0,0 +1,5 @@ +export const REPOSITORY_EXECUTION_LOCK = { + resource: ({ repoId }: { repoId: number }) => + `sourcebot:lock:repo:${repoId}`, + durationMs: 60_000, +}; diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index f3cfbcfe5..24f2c6a34 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -151,7 +151,7 @@ export class BullMQClient { const { dedupKey: getDedupKey }: { dedupKey?(data: DataOf): string; } = spec; - const dedupKey = getDedupKey?.(data); + const deduplication = getDeduplication?.(data); const queue = this.getQueue(spec); const requestedJobId = randomUUID(); diff --git a/packages/shared/src/index.server.ts b/packages/shared/src/index.server.ts index 0c16dd456..73b08c5e0 100644 --- a/packages/shared/src/index.server.ts +++ b/packages/shared/src/index.server.ts @@ -126,6 +126,7 @@ export { DEFAULT_JOB_OPTIONS, JOB_PRIORITIES, QUEUE_SPECS, + REPO_CLEANUP_QUEUE, REPO_INDEX_QUEUE, REPO_PERMISSION_SYNC_QUEUE, } from "./queue.js"; diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 1ee263baf..4f9cd184e 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -101,7 +101,12 @@ interface QueueRegistry { "repo-index": { data: { repoId: number; - type: "INDEX" | "CLEANUP"; + }; + result: void; + }; + "repo-cleanup": { + data: { + repoId: number; }; result: void; }; @@ -145,6 +150,11 @@ export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { jobOptions: DEFAULT_JOB_OPTIONS, }; +export const REPO_CLEANUP_QUEUE: QueueSpec<"repo-cleanup"> = { + name: "repo-cleanup", + jobOptions: DEFAULT_JOB_OPTIONS, +}; + export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = { name: "account-permission-sync", jobOptions: DEFAULT_JOB_OPTIONS, @@ -163,6 +173,7 @@ export const QUEUE_SPECS = { [AUDIT_LOG_PRUNE_QUEUE.name]: AUDIT_LOG_PRUNE_QUEUE, [CONNECTION_QUEUE.name]: CONNECTION_QUEUE, [REPO_INDEX_QUEUE.name]: REPO_INDEX_QUEUE, + [REPO_CLEANUP_QUEUE.name]: REPO_CLEANUP_QUEUE, [ACCOUNT_PERMISSION_SYNC_QUEUE.name]: ACCOUNT_PERMISSION_SYNC_QUEUE, [REPO_PERMISSION_SYNC_QUEUE.name]: REPO_PERMISSION_SYNC_QUEUE, } as const satisfies { [TName in QueueName]: QueueSpec }; diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index 8b1430ddf..fce94c0b9 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -38,7 +38,7 @@ const repos: Repo[] = [ indexedCommitHash: null, latestJob: { id: "job-1", - data: { repoId: 1, type: "INDEX" }, + data: { repoId: 1 }, status: "IN_PROGRESS", errorMessage: null, result: null, @@ -187,7 +187,7 @@ describe("ReposTable", () => { ...repos[1], latestJob: { id: "completed-job", - data: { repoId: 2, type: "INDEX" }, + data: { repoId: 2 }, status: "COMPLETED", errorMessage: null, result: null, @@ -276,7 +276,7 @@ describe("ReposTable", () => { ...repos[1], latestJob: { id: "active-reindex-job", - data: { repoId: repos[1].id, type: "INDEX" }, + data: { repoId: repos[1].id }, status: "IN_PROGRESS", errorMessage: null, result: null, @@ -313,7 +313,7 @@ describe("ReposTable", () => { indexedCommitHash: "3333333333333333333333333333333333333333", latestJob: { id: "first-interactive-job", - data: { repoId: 1, type: "INDEX" }, + data: { repoId: 1 }, status: "COMPLETED", errorMessage: null, result: null, @@ -357,7 +357,7 @@ describe("ReposTable", () => { ...repos[1], latestJob: { id: "warning-job", - data: { repoId: 2, type: "INDEX" }, + data: { repoId: 2 }, status: "FAILED", errorMessage: "The remote repository could not be reached", result: null, @@ -483,7 +483,7 @@ describe("ReposTable", () => { indexedCommitHash: null, latestJob: { id: "job-1", - data: { repoId: 1, type: "INDEX" }, + data: { repoId: 1 }, status: "FAILED", errorMessage: "Authentication failed while cloning", result: null, @@ -535,7 +535,7 @@ describe("ReposTable", () => { indexedCommitHash: "1111111111111111111111111111111111111111", latestJob: { id: "job-1", - data: { repoId: 1, type: "INDEX" }, + data: { repoId: 1 }, status: "COMPLETED", errorMessage: null, result: null, @@ -575,7 +575,7 @@ describe("ReposTable", () => { indexedCommitHash: null, latestJob: { id: "job-1", - data: { repoId: 1, type: "INDEX" }, + data: { repoId: 1 }, status: "FAILED", errorMessage: "Indexing failed", result: null, diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index 2ca70a1f8..f53777433 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -104,8 +104,7 @@ const getRepoIndexingStatuses = async ( const getSyncAnnotation = (repo: Repo): SyncAnnotation => { const latestJob = repo.latestJob; - const isLatestIndexJob = latestJob?.data.repoId === repo.id - && latestJob.data.type === "INDEX"; + const isLatestIndexJob = latestJob?.data.repoId === repo.id; if (isLatestIndexJob && latestJob.status === "FAILED") { return repo.indexedAt ? "WARNING" : "FAILED"; @@ -134,7 +133,6 @@ const getSyncAnnotation = (repo: Repo): SyncAnnotation => { const hasActiveIndexingJob = (repo: Repo) => { const latestJob = repo.latestJob; return latestJob?.data.repoId === repo.id - && latestJob.data.type === "INDEX" && ( latestJob.status === "PENDING" || latestJob.status === "IN_PROGRESS" @@ -555,7 +553,7 @@ export const ReposTable = ({ const nextJobs = new Map(currentJobs); nextJobs.set(repoId, { id: jobId, - data: { repoId, type: "INDEX" }, + data: { repoId }, status: "PENDING", errorMessage: null, result: null, @@ -609,8 +607,7 @@ export const ReposTable = ({ } const latestJob = status.latestJob; - const isLatestIndexJob = latestJob?.data.repoId === status.repoId - && latestJob.data.type === "INDEX"; + const isLatestIndexJob = latestJob?.data.repoId === status.repoId; const expectedJobId = target.jobId; if (expectedJobId && latestJob?.id !== expectedJobId) { diff --git a/packages/web/src/features/repos/actions.test.ts b/packages/web/src/features/repos/actions.test.ts index c64d5c0ac..9ccfef21a 100644 --- a/packages/web/src/features/repos/actions.test.ts +++ b/packages/web/src/features/repos/actions.test.ts @@ -67,7 +67,7 @@ describe('indexRepo', () => { }); expect(mocks.enqueue).toHaveBeenCalledWith( repoIndexQueue, - { repoId: 42, type: 'INDEX' }, + { repoId: 42 }, { priority: 1 }, ); }); diff --git a/packages/web/src/features/repos/actions.ts b/packages/web/src/features/repos/actions.ts index 286fb1ab8..eceaec2a4 100644 --- a/packages/web/src/features/repos/actions.ts +++ b/packages/web/src/features/repos/actions.ts @@ -27,7 +27,7 @@ export const indexRepo = async (repoId: number) => sew(() => const jobId = await getBullMQClient().enqueue( REPO_INDEX_QUEUE, - { repoId: repo.id, type: 'INDEX' }, + { repoId: repo.id }, { priority: JOB_PRIORITIES.INTERACTIVE }, ); From 3924d57fb44aeca09ff06fd688077317120adb49 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 21:26:35 -0700 Subject: [PATCH 32/34] added additional deduplication behaviour --- .../src/connectionSyncWorkload.test.ts | 5 ++- packages/backend/src/jobManager.test.ts | 4 +- .../backend/src/repoIndexWorkload.test.ts | 10 +++++ packages/shared/src/bullmqClient.test.ts | 40 ++++++++++++++++++- packages/shared/src/bullmqClient.ts | 8 ++-- packages/shared/src/queue.ts | 24 +++++++---- 6 files changed, 77 insertions(+), 14 deletions(-) diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index 21fda5d22..b8b616c09 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -36,8 +36,9 @@ vi.mock("./entitlements.js", () => ({ vi.mock("@sourcebot/shared", () => ({ CONNECTION_QUEUE: { name: "connection-sync", - dedupKey: ({ connectionId }: { connectionId: number }) => - `connection:${connectionId}`, + deduplication: ({ connectionId }: { connectionId: number }) => ({ + id: `connection:${connectionId}`, + }), jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, diff --git a/packages/backend/src/jobManager.test.ts b/packages/backend/src/jobManager.test.ts index 4c2c0cf8f..161e993ef 100644 --- a/packages/backend/src/jobManager.test.ts +++ b/packages/backend/src/jobManager.test.ts @@ -107,7 +107,9 @@ const createWorkload = ( queueSpec: { name: "connection-sync", resultSchema: z.unknown() as ZodType, - dedupKey: ({ connectionId }) => `connection:${connectionId}`, + deduplication: ({ connectionId }) => ({ + id: `connection:${connectionId}`, + }), jobOptions: { attempts: 2, backoff: { type: "exponential", delayMs: 5000 }, diff --git a/packages/backend/src/repoIndexWorkload.test.ts b/packages/backend/src/repoIndexWorkload.test.ts index 91715f839..ae3d84603 100644 --- a/packages/backend/src/repoIndexWorkload.test.ts +++ b/packages/backend/src/repoIndexWorkload.test.ts @@ -78,6 +78,16 @@ describe("repoIndexWorkload", () => { expect(workload.executionLock?.durationMs).toBe(60_000); expect(workload.queueSpec.name).toBe("repo-index"); expect(cleanupWorkload.queueSpec.name).toBe("repo-cleanup"); + expect(workload.queueSpec.deduplication?.({ repoId: 42 })).toEqual({ + id: "repo:42", + keepLastIfActive: true, + }); + expect( + cleanupWorkload.queueSpec.deduplication?.({ repoId: 42 }), + ).toEqual({ + id: "repo:42", + keepLastIfActive: true, + }); }); test("records the first successful indexing job terminal state", async () => { diff --git a/packages/shared/src/bullmqClient.test.ts b/packages/shared/src/bullmqClient.test.ts index 424fc2a9e..72ed1918e 100644 --- a/packages/shared/src/bullmqClient.test.ts +++ b/packages/shared/src/bullmqClient.test.ts @@ -32,7 +32,7 @@ vi.mock("./jobLogger.js", () => ({ })); import { BullMQClient } from "./bullmqClient.js"; -import { CONNECTION_QUEUE } from "./queue.js"; +import { CONNECTION_QUEUE, type QueueSpec } from "./queue.js"; describe("BullMQClient", () => { beforeEach(() => { @@ -206,6 +206,44 @@ describe("BullMQClient", () => { ); }); + test("adds simple deduplication to immediate jobs", async () => { + const client = new BullMQClient({} as Redis); + + await client.enqueue(CONNECTION_QUEUE, { connectionId: 42 }); + + expect(mocks.add).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + expect.objectContaining({ + deduplication: { id: "connection:42" }, + }), + ); + }); + + test("passes keepLastIfActive to BullMQ deduplication", async () => { + const queueSpec = { + ...CONNECTION_QUEUE, + deduplication: ({ connectionId }) => ({ + id: `connection:${connectionId}`, + keepLastIfActive: true, + }), + } satisfies QueueSpec<"connection-sync">; + const client = new BullMQClient({} as Redis); + + await client.enqueue(queueSpec, { connectionId: 42 }); + + expect(mocks.add).toHaveBeenCalledWith( + "connection-sync", + { connectionId: 42 }, + expect.objectContaining({ + deduplication: { + id: "connection:42", + keepLastIfActive: true, + }, + }), + ); + }); + test("adds enqueue priority to scheduled jobs", async () => { const client = new BullMQClient({} as Redis); diff --git a/packages/shared/src/bullmqClient.ts b/packages/shared/src/bullmqClient.ts index 24f2c6a34..0562543ad 100644 --- a/packages/shared/src/bullmqClient.ts +++ b/packages/shared/src/bullmqClient.ts @@ -148,8 +148,10 @@ export class BullMQClient { // QueueSpec is distributive so queue names, data, and result schemas stay // correlated when TName is a union. Re-establish the shared generic here // before invoking the optional method. - const { dedupKey: getDedupKey }: { - dedupKey?(data: DataOf): string; + const { deduplication: getDeduplication }: { + deduplication?( + data: DataOf, + ): { id: string; keepLastIfActive?: boolean }; } = spec; const deduplication = getDeduplication?.(data); const queue = this.getQueue(spec); @@ -157,7 +159,7 @@ export class BullMQClient { const requestedJobId = randomUUID(); const job = await queue.add(spec.name, data, { jobId: requestedJobId, - ...(dedupKey ? { deduplication: { id: dedupKey } } : {}), + ...(deduplication ? { deduplication } : {}), ...(options.priority !== undefined ? { priority: options.priority } : {}), diff --git a/packages/shared/src/queue.ts b/packages/shared/src/queue.ts index 4f9cd184e..a80dddcac 100644 --- a/packages/shared/src/queue.ts +++ b/packages/shared/src/queue.ts @@ -1,4 +1,4 @@ -import type { KeepJobs } from "bullmq"; +import type { DeduplicationOptions, KeepJobs } from "bullmq"; import { z, type ZodType } from "zod"; import { connectionSyncResultSchema, @@ -8,7 +8,9 @@ import { DEFAULT_JOB_LOGS_MAX_ENTRIES } from "./jobLogger.js"; interface BaseQueueSpec { name: TName; - dedupKey?(data: DataOf): string; + deduplication?( + data: DataOf, + ): Pick; jobOptions: JobOptions; } @@ -128,44 +130,52 @@ export const ATTACHMENT_PRUNE_QUEUE: QueueSpec<"attachment-prune"> = { name: "attachment-prune", resultSchema: attachmentPruneResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: () => "global", + deduplication: () => ({ id: "global" }), }; export const AUDIT_LOG_PRUNE_QUEUE: QueueSpec<"audit-log-prune"> = { name: "audit-log-prune", resultSchema: auditLogPruneResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: () => "global", + deduplication: () => ({ id: "global" }), }; export const CONNECTION_QUEUE: QueueSpec<"connection-sync"> = { name: "connection-sync", resultSchema: connectionSyncResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `connection:${data.connectionId}`, + deduplication: (data) => ({ id: `connection:${data.connectionId}` }), }; export const REPO_INDEX_QUEUE: QueueSpec<"repo-index"> = { name: "repo-index", jobOptions: DEFAULT_JOB_OPTIONS, + deduplication: ({ repoId }) => ({ + id: `repo:${repoId}`, + keepLastIfActive: true, + }), }; export const REPO_CLEANUP_QUEUE: QueueSpec<"repo-cleanup"> = { name: "repo-cleanup", jobOptions: DEFAULT_JOB_OPTIONS, + deduplication: ({ repoId }) => ({ + id: `repo:${repoId}`, + keepLastIfActive: true, + }), }; export const ACCOUNT_PERMISSION_SYNC_QUEUE: QueueSpec<"account-permission-sync"> = { name: "account-permission-sync", jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `account:${data.accountId}`, + deduplication: (data) => ({ id: `account:${data.accountId}` }), }; export const REPO_PERMISSION_SYNC_QUEUE: QueueSpec<"repo-permission-sync"> = { name: "repo-permission-sync", resultSchema: repoPermissionSyncResultSchema, jobOptions: DEFAULT_JOB_OPTIONS, - dedupKey: (data) => `repo:${data.repoId}`, + deduplication: (data) => ({ id: `repo:${data.repoId}` }), }; export const QUEUE_SPECS = { From 61d904421a6f9d8b5a34f7fc0b639f8384ceceb6 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 21:51:27 -0700 Subject: [PATCH 33/34] add retry all button to repository table --- .../repos/components/reposTable.test.tsx | 35 +++++- .../app/(app)/repos/components/reposTable.tsx | 103 +++++++++++++++++- packages/web/src/app/(app)/repos/page.tsx | 13 ++- .../connectionSyncCounts.server.ts | 5 +- .../web/src/features/repos/actions.test.ts | 70 +++++++++++- packages/web/src/features/repos/actions.ts | 64 +++++++++++ .../repos/repositorySyncCounts.server.ts | 5 +- 7 files changed, 283 insertions(+), 12 deletions(-) diff --git a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx index fce94c0b9..5ed70c23c 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.test.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.test.tsx @@ -15,10 +15,12 @@ const navigation = vi.hoisted(() => ({ })); const reposActions = vi.hoisted(() => ({ indexRepo: vi.fn(), + retryReposWithSyncIssues: vi.fn(), })); vi.mock("@/features/repos/actions", () => ({ indexRepo: reposActions.indexRepo, + retryReposWithSyncIssues: reposActions.retryReposWithSyncIssues, })); vi.mock("next/navigation", () => ({ @@ -60,7 +62,11 @@ const repos: Repo[] = [ }, ]; -const renderTable = (data: Repo[] = repos, canRetry = true) => { +const renderTable = ( + data: Repo[] = repos, + canRetry = true, + retryableCount = 0, +) => { const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } }, }); @@ -74,6 +80,7 @@ const renderTable = (data: Repo[] = repos, canRetry = true) => { pageSize={20} totalCount={data.length} canRetry={canRetry} + retryableCount={retryableCount} sortBy="indexedAt" sortOrder="asc" /> @@ -90,6 +97,32 @@ afterEach(() => { }); describe("ReposTable", () => { + test("retries all repository sync issues without refreshing", async () => { + reposActions.retryReposWithSyncIssues.mockResolvedValue({ + jobs: [{ repoId: 999, jobId: "retry-999" }], + failedCount: 0, + }); + renderTable([repos[1]], true, 2); + + fireEvent.click(screen.getByRole("button", { name: "Retry all (2)" })); + + await waitFor(() => { + expect(reposActions.retryReposWithSyncIssues).toHaveBeenCalledOnce(); + }); + await waitFor(() => { + expect( + screen.queryByRole("button", { name: /Retry all/ }), + ).toBeNull(); + }); + expect(navigation.refresh).not.toHaveBeenCalled(); + }); + + test("hides retry all from non-owners", () => { + renderTable([repos[1]], false, 2); + + expect(screen.queryByRole("button", { name: /Retry all/ })).toBeNull(); + }); + test("reflects the status filter from the URL", () => { navigation.searchParams = "status=failed"; diff --git a/packages/web/src/app/(app)/repos/components/reposTable.tsx b/packages/web/src/app/(app)/repos/components/reposTable.tsx index f53777433..3a0cfb358 100644 --- a/packages/web/src/app/(app)/repos/components/reposTable.tsx +++ b/packages/web/src/app/(app)/repos/components/reposTable.tsx @@ -2,6 +2,7 @@ import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { useToast } from "@/components/hooks/use-toast"; import { InputGroup, InputGroupAddon, @@ -16,7 +17,11 @@ import { } from "@/components/ui/select"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { cn, getCodeHostIcon, getRepoImageSrc } from "@/lib/utils"; +import { + retryReposWithSyncIssues, + type ScheduledRepoIndexJob, +} from "@/features/repos/actions"; +import { cn, getCodeHostIcon, getRepoImageSrc, isServiceError } from "@/lib/utils"; import type { CodeHostType } from "@sourcebot/db"; import type { WorkloadJob } from "@sourcebot/shared"; import { useQuery } from "@tanstack/react-query"; @@ -27,7 +32,7 @@ import { getCoreRowModel, useReactTable, } from "@tanstack/react-table"; -import { ArrowDown, ArrowUp, Check, CircleX, Loader2, Search } from "lucide-react"; +import { ArrowDown, ArrowUp, Check, CircleX, Loader2, RotateCw, Search } from "lucide-react"; import Image from "next/image"; import Link from "next/link"; import { AnimatePresence, motion, useReducedMotion } from "motion/react"; @@ -466,6 +471,7 @@ type ReposTableProps = { pageSize: number; totalCount: number; canRetry: boolean; + retryableCount: number; sortBy: SortBy; sortOrder: SortOrder; }; @@ -476,6 +482,7 @@ export const ReposTable = ({ pageSize, totalCount, canRetry, + retryableCount, sortBy, sortOrder, }: ReposTableProps) => { @@ -489,6 +496,11 @@ export const ReposTable = ({ const [scheduledRetryJobs, setScheduledRetryJobs] = useState< Map> >(() => new Map()); + const [displayedRetryableCount, setDisplayedRetryableCount] = useState( + retryableCount, + ); + const [isRetryingAll, setIsRetryingAll] = useState(false); + const { toast } = useToast(); const debouncedSearchValue = useDebounce(searchValue, 300); const [isSearchNavigationPending, startSearchTransition] = useTransition(); const pendingSearchValuesRef = useRef>(new Set()); @@ -509,6 +521,10 @@ export const ReposTable = ({ setSearchValue(urlSearchValue); }, [urlSearchValue]); + useEffect(() => { + setDisplayedRetryableCount(retryableCount); + }, [retryableCount]); + useEffect(() => { if (debouncedSearchValue !== searchValue) { return; @@ -561,6 +577,65 @@ export const ReposTable = ({ return nextJobs; }); }, []); + const onRetriesScheduled = useCallback((jobs: ScheduledRepoIndexJob[]) => { + setScheduledRetryJobs((currentJobs) => { + const nextJobs = new Map(currentJobs); + for (const { repoId, jobId } of jobs) { + nextJobs.set(repoId, { + id: jobId, + data: { repoId }, + status: "PENDING", + errorMessage: null, + result: null, + }); + } + return nextJobs; + }); + }, []); + const retryAll = async () => { + setIsRetryingAll(true); + + try { + const response = await retryReposWithSyncIssues(); + if (isServiceError(response)) { + toast({ + variant: "destructive", + title: "Failed to retry repository syncs", + description: response.message, + }); + return; + } + + onRetriesScheduled(response.jobs); + setDisplayedRetryableCount(response.failedCount); + + if (response.failedCount > 0) { + toast({ + variant: "destructive", + title: "Some repository syncs could not be retried", + description: `${response.jobs.length} scheduled; ${response.failedCount} failed to schedule.`, + }); + } else if (response.jobs.length > 0) { + toast({ + title: "Syncs scheduled", + description: `${response.jobs.length} ${response.jobs.length === 1 ? "repository was" : "repositories were"} queued for indexing.`, + }); + } else { + toast({ + title: "No retries needed", + description: "No repositories currently have failed syncs.", + }); + } + } catch { + toast({ + variant: "destructive", + title: "Failed to retry repository syncs", + description: "An unexpected error occurred while scheduling the syncs.", + }); + } finally { + setIsRetryingAll(false); + } + }; const retryAwareData = useMemo( () => data.map((repo) => { const scheduledJob = scheduledRetryJobs.get(repo.id); @@ -850,6 +925,30 @@ export const ReposTable = ({ Clear filters )} + {canRetry && displayedRetryableCount > 0 && ( + + + + + + Retry all repositories whose latest sync failed. + + + )}
diff --git a/packages/web/src/app/(app)/repos/page.tsx b/packages/web/src/app/(app)/repos/page.tsx index b4e1a65d5..4b23e012e 100644 --- a/packages/web/src/app/(app)/repos/page.tsx +++ b/packages/web/src/app/(app)/repos/page.tsx @@ -7,6 +7,8 @@ import { REPO_INDEX_QUEUE, type WorkloadJob, } from "@sourcebot/shared"; +import { getRepositorySyncCounts } from "@/features/repos/repositorySyncCounts.server"; +import { isServiceError } from "@/lib/utils"; import { OrgRole, type Prisma } from "@sourcebot/db"; import { z } from "zod"; import { ReposTable } from "./components/reposTable"; @@ -37,6 +39,7 @@ export default authenticatedPage< const status = statusSchema.safeParse(params.status).data ?? "all"; const sortBy = sortBySchema.safeParse(params.sortBy).data ?? "name"; const sortOrder = sortOrderSchema.safeParse(params.sortOrder).data ?? "asc"; + const canRetry = role === OrgRole.OWNER; const skip = (page - 1) * DEFAULT_PAGE_SIZE; const orderBy = sortBy === "indexedAt" ? [{ indexedAt: sortOrder }, { id: "asc" as const }] @@ -44,6 +47,13 @@ export default authenticatedPage< const failedJobIds = status === "all" ? [] : await getBullMQClient().getFailedJobIds(REPO_INDEX_QUEUE); + const repositorySyncCounts = canRetry + ? await getRepositorySyncCounts() + : null; + const retryableCount = repositorySyncCounts + && !isServiceError(repositorySyncCounts) + ? repositorySyncCounts.failedCount + repositorySyncCounts.warningCount + : 0; const where: Prisma.RepoWhereInput = { orgId: org.id, ...(search @@ -120,7 +130,8 @@ export default authenticatedPage< currentPage={page} pageSize={DEFAULT_PAGE_SIZE} totalCount={totalCount} - canRetry={role === OrgRole.OWNER} + canRetry={canRetry} + retryableCount={retryableCount} sortBy={sortBy} sortOrder={sortOrder} /> diff --git a/packages/web/src/features/connections/connectionSyncCounts.server.ts b/packages/web/src/features/connections/connectionSyncCounts.server.ts index 97511cd50..43239d377 100644 --- a/packages/web/src/features/connections/connectionSyncCounts.server.ts +++ b/packages/web/src/features/connections/connectionSyncCounts.server.ts @@ -5,6 +5,7 @@ import { withAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; import { OrgRole } from "@sourcebot/db"; import { CONNECTION_QUEUE } from "@sourcebot/shared"; +import { cache } from "react"; export interface ConnectionSyncCounts { firstTimeSyncingCount: number; @@ -12,7 +13,7 @@ export interface ConnectionSyncCounts { warningCount: number; } -export const getConnectionSyncCounts = async () => +export const getConnectionSyncCounts = cache(async () => withAuth(({ org, prisma, role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { const connections = await prisma.connection.findMany({ @@ -63,4 +64,4 @@ export const getConnectionSyncCounts = async () => return counts; }, { firstTimeSyncingCount: 0, failedCount: 0, warningCount: 0 }); }) - ); + )); diff --git a/packages/web/src/features/repos/actions.test.ts b/packages/web/src/features/repos/actions.test.ts index 9ccfef21a..02452ba12 100644 --- a/packages/web/src/features/repos/actions.test.ts +++ b/packages/web/src/features/repos/actions.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ authContext: undefined as unknown, enqueue: vi.fn(), + getFailedJobIds: vi.fn(), })); const repoIndexQueue = { @@ -10,7 +11,10 @@ const repoIndexQueue = { }; vi.mock('@/lib/bullmqClient', () => ({ - getBullMQClient: () => ({ enqueue: mocks.enqueue }), + getBullMQClient: () => ({ + enqueue: mocks.enqueue, + getFailedJobIds: mocks.getFailedJobIds, + }), })); vi.mock('@/lib/serviceError', () => ({ unexpectedError: (message: string) => ({ error: message }), @@ -34,17 +38,21 @@ vi.mock('@sourcebot/shared', () => ({ REPO_INDEX_QUEUE: repoIndexQueue, })); -const { indexRepo } = await import('./actions'); +const { indexRepo, retryReposWithSyncIssues } = await import('./actions'); beforeEach(() => { vi.clearAllMocks(); mocks.enqueue.mockResolvedValue('job-1'); + mocks.getFailedJobIds.mockResolvedValue([]); }); -const setAuthContext = (findFirst: ReturnType) => { +const setAuthContext = ( + findFirst: ReturnType, + findMany = vi.fn().mockResolvedValue([]), +) => { mocks.authContext = { org: { id: 1 }, - prisma: { repo: { findFirst } }, + prisma: { repo: { findFirst, findMany } }, role: 'OWNER', }; }; @@ -90,3 +98,57 @@ describe('indexRepo', () => { }); }); }); + +describe('retryReposWithSyncIssues', () => { + test('retries every repository whose latest job failed', async () => { + const findMany = vi.fn().mockResolvedValue([{ id: 10 }, { id: 20 }]); + setAuthContext(vi.fn(), findMany); + mocks.getFailedJobIds.mockResolvedValue(['failed-1', 'failed-2']); + mocks.enqueue + .mockResolvedValueOnce('retry-10') + .mockResolvedValueOnce('retry-20'); + + await expect(retryReposWithSyncIssues()).resolves.toEqual({ + jobs: [ + { repoId: 10, jobId: 'retry-10' }, + { repoId: 20, jobId: 'retry-20' }, + ], + failedCount: 0, + }); + + expect(findMany).toHaveBeenCalledWith({ + where: { + orgId: 1, + latestIndexingJobId: { in: ['failed-1', 'failed-2'] }, + }, + orderBy: { id: 'asc' }, + select: { id: true }, + }); + expect(mocks.enqueue).toHaveBeenNthCalledWith( + 1, + repoIndexQueue, + { repoId: 10 }, + { priority: 1 }, + ); + expect(mocks.enqueue).toHaveBeenNthCalledWith( + 2, + repoIndexQueue, + { repoId: 20 }, + { priority: 1 }, + ); + }); + + test('returns successful jobs when part of the batch fails', async () => { + const findMany = vi.fn().mockResolvedValue([{ id: 10 }, { id: 20 }]); + setAuthContext(vi.fn(), findMany); + mocks.getFailedJobIds.mockResolvedValue(['failed-1']); + mocks.enqueue + .mockResolvedValueOnce('retry-10') + .mockRejectedValueOnce(new Error('Redis unavailable')); + + await expect(retryReposWithSyncIssues()).resolves.toEqual({ + jobs: [{ repoId: 10, jobId: 'retry-10' }], + failedCount: 1, + }); + }); +}); diff --git a/packages/web/src/features/repos/actions.ts b/packages/web/src/features/repos/actions.ts index eceaec2a4..29d2753a5 100644 --- a/packages/web/src/features/repos/actions.ts +++ b/packages/web/src/features/repos/actions.ts @@ -8,6 +8,13 @@ import { withMinimumOrgRole } from '@/middleware/withMinimumOrgRole'; import { OrgRole } from '@sourcebot/db'; import { JOB_PRIORITIES, REPO_INDEX_QUEUE } from '@sourcebot/shared'; +const MAX_CONCURRENT_REPO_RETRIES = 10; + +export type ScheduledRepoIndexJob = { + repoId: number; + jobId: string; +}; + export const indexRepo = async (repoId: number) => sew(() => withAuth(({ org, prisma, role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { @@ -38,3 +45,60 @@ export const indexRepo = async (repoId: number) => sew(() => }) ) ); + +export const retryReposWithSyncIssues = async () => sew(() => + withAuth(({ org, prisma, role }) => + withMinimumOrgRole(role, OrgRole.OWNER, async () => { + try { + const client = getBullMQClient(); + const failedJobIds = await client.getFailedJobIds( + REPO_INDEX_QUEUE, + ); + const repos = await prisma.repo.findMany({ + where: { + orgId: org.id, + latestIndexingJobId: { in: failedJobIds }, + }, + orderBy: { id: 'asc' }, + select: { id: true }, + }); + + const jobs: ScheduledRepoIndexJob[] = []; + let failedCount = 0; + + for ( + let offset = 0; + offset < repos.length; + offset += MAX_CONCURRENT_REPO_RETRIES + ) { + const batch = repos.slice( + offset, + offset + MAX_CONCURRENT_REPO_RETRIES, + ); + const results = await Promise.allSettled( + batch.map(async ({ id: repoId }) => ({ + repoId, + jobId: await client.enqueue( + REPO_INDEX_QUEUE, + { repoId }, + { priority: JOB_PRIORITIES.INTERACTIVE }, + ), + })), + ); + + for (const result of results) { + if (result.status === 'fulfilled') { + jobs.push(result.value); + } else { + failedCount += 1; + } + } + } + + return { jobs, failedCount }; + } catch { + return unexpectedError('Failed to retry repository syncs'); + } + }) + ) +); diff --git a/packages/web/src/features/repos/repositorySyncCounts.server.ts b/packages/web/src/features/repos/repositorySyncCounts.server.ts index 9a8c2affd..772ef1c1d 100644 --- a/packages/web/src/features/repos/repositorySyncCounts.server.ts +++ b/packages/web/src/features/repos/repositorySyncCounts.server.ts @@ -5,6 +5,7 @@ import { withAuth } from "@/middleware/withAuth"; import { withMinimumOrgRole } from "@/middleware/withMinimumOrgRole"; import { OrgRole } from "@sourcebot/db"; import { REPO_INDEX_QUEUE } from "@sourcebot/shared"; +import { cache } from "react"; export interface RepositorySyncCounts { firstTimeSyncingCount: number; @@ -12,7 +13,7 @@ export interface RepositorySyncCounts { warningCount: number; } -export const getRepositorySyncCounts = async () => +export const getRepositorySyncCounts = cache(async () => withAuth(({ org, prisma, role }) => withMinimumOrgRole(role, OrgRole.OWNER, async () => { const failedJobIds = await getBullMQClient().getFailedJobIds( @@ -53,4 +54,4 @@ export const getRepositorySyncCounts = async () => warningCount, }; }) - ); + )); From 310c5c1bb063957335fabdce15830dc25eb5d7d1 Mon Sep 17 00:00:00 2001 From: Brendan Kellam Date: Tue, 18 Aug 2026 22:18:01 -0700 Subject: [PATCH 34/34] improve connection sync repo removal behaviour --- .../src/connectionSyncWorkload.test.ts | 81 +++++++++++++++++++ .../backend/src/connectionSyncWorkload.ts | 40 +++++++-- 2 files changed, 116 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/connectionSyncWorkload.test.ts b/packages/backend/src/connectionSyncWorkload.test.ts index b8b616c09..3b78b780a 100644 --- a/packages/backend/src/connectionSyncWorkload.test.ts +++ b/packages/backend/src/connectionSyncWorkload.test.ts @@ -330,6 +330,87 @@ describe("connectionWorkload", () => { }); }); + test("preserves missing repositories when discovery is incomplete", async () => { + const reason = { + code: "ENUMERATION_FAILED" as const, + effect: "DISCOVERY_INCOMPLETE" as const, + subject: { + kind: "organization" as const, + value: "sourcebot-dev", + }, + message: "Repository enumeration did not complete.", + }; + const indexedAt = new Date("2026-07-30T12:00:00.000Z"); + mocks.connectionFindUniqueOrThrow.mockResolvedValue({ + id: 42, + name: "github", + orgId: 7, + config: { type: "github" }, + }); + mocks.compileGithubConfig.mockImplementation(async () => { + reportRepositoryDiscoveryIssue(reason); + return [ + { + external_id: "repo-1", + external_codeHostUrl: "https://github.com", + }, + ]; + }); + mocks.repoFindMany + .mockResolvedValueOnce([ + { + id: 2, + name: "github.com/sourcebot/repo-2", + indexedAt, + }, + ]) + .mockResolvedValueOnce([]); + mocks.repoUpsert.mockResolvedValue({ + id: 1, + name: "github.com/sourcebot/repo-1", + indexedAt, + }); + const trigger = vi.fn(); + + await expect( + connectionWorkload.process({ + ...lifecycleContext, + signal: new AbortController().signal, + updateProgress: vi.fn(), + trigger, + }), + ).resolves.toEqual({ + outcome: "PARTIAL_SUCCESS", + reasons: [reason], + }); + + expect(mocks.repoToConnectionDeleteMany).not.toHaveBeenCalled(); + expect(mocks.logger.debug).toHaveBeenCalledWith( + "Preserving repositories omitted from a DISCOVERY_INCOMPLETE result", + { + connectionId: 42, + repositories: [ + { + id: 2, + name: "github.com/sourcebot/repo-2", + }, + ], + }, + ); + expect(mocks.upsertJobScheduler).toHaveBeenCalledWith( + "repo-index", + "repo-index-v1-2", + 3_600_000, + { repoId: 2 }, + { priority: 10 }, + ); + expect(trigger).not.toHaveBeenCalledWith( + "repo-cleanup", + { repoId: 2 }, + expect.anything(), + ); + }); + test("does not mark the connection synced when repo work reconciliation fails", async () => { mocks.connectionFindUniqueOrThrow.mockResolvedValue({ id: 42, diff --git a/packages/backend/src/connectionSyncWorkload.ts b/packages/backend/src/connectionSyncWorkload.ts index c65e9abd2..3eb896b2e 100644 --- a/packages/backend/src/connectionSyncWorkload.ts +++ b/packages/backend/src/connectionSyncWorkload.ts @@ -78,12 +78,17 @@ export const createConnectionSyncWorkload = ({ repositoryCount: repoData.length, }); + const discoveryIsComplete = !issues.some( + ({ effect }) => effect === "DISCOVERY_INCOMPLETE", + ); + signal.throwIfAborted(); const repoChanges = await replaceConnectionRepositories({ db, connectionId, orgId, discoveredRepos: repoData, + removeMissing: discoveryIsComplete, }); signal.throwIfAborted(); @@ -216,11 +221,13 @@ export const replaceConnectionRepositories = async ({ connectionId, orgId, discoveredRepos, + removeMissing = true, }: { db: PrismaClient; connectionId: number; orgId: number; discoveredRepos: RepoData[]; + removeMissing?: boolean; }): Promise => { const previouslyAssociatedRepos = await db.repo.findMany({ where: { @@ -232,12 +239,14 @@ export const replaceConnectionRepositories = async ({ }, select: { id: true, + name: true, + indexedAt: true, }, }); - const currentRepos: CurrentRepo[] = []; + const discoveredCurrentRepos: CurrentRepo[] = []; for (const repo of deduplicateRepos(discoveredRepos)) { - currentRepos.push( + discoveredCurrentRepos.push( await db.repo.upsert({ where: { external_id_external_codeHostUrl_orgId: { @@ -267,10 +276,31 @@ export const replaceConnectionRepositories = async ({ ); } + const discoveredRepoIds = new Set( + discoveredCurrentRepos.map(({ id }) => id), + ); + const missingRepos = previouslyAssociatedRepos.filter( + ({ id }) => !discoveredRepoIds.has(id), + ); + if (!removeMissing && missingRepos.length > 0) { + logger.debug( + "Preserving repositories omitted from a DISCOVERY_INCOMPLETE result", + { + connectionId, + repositories: missingRepos.map(({ id, name }) => ({ + id, + name, + })), + }, + ); + } + const currentRepos = removeMissing + ? discoveredCurrentRepos + : [...discoveredCurrentRepos, ...missingRepos]; const currentRepoIds = new Set(currentRepos.map(({ id }) => id)); - const staleRepoIds = previouslyAssociatedRepos - .map(({ id }) => id) - .filter((id) => !currentRepoIds.has(id)); + const staleRepoIds = removeMissing + ? missingRepos.map(({ id }) => id) + : []; if (staleRepoIds.length > 0) { await db.repoToConnection.deleteMany({