From 4d2a672c5e51a32368c455bf3ceb44b475dbb8c0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Mar 2026 16:22:42 +0000 Subject: [PATCH 01/32] Implement organization-scoped ClickHouse instances The only way to get a ClickHouse client now is through the factory. Refactored all existing code to use that and pass in an org. The runReplication and otlpExporter are the hot paths here which need special attention in reviews. --- .cursor/mcp.json | 6 +- .../organization-scoped-clickhouse.md | 6 + .../v3/ApiRunListPresenter.server.ts | 5 +- .../v3/CreateBulkActionPresenter.server.ts | 5 +- .../v3/RunTagListPresenter.server.ts | 5 +- .../presenters/v3/TaskListPresenter.server.ts | 25 +- .../presenters/v3/UsagePresenter.server.ts | 5 +- .../v3/ViewSchedulePresenter.server.ts | 5 +- .../v3/WaitpointPresenter.server.ts | 5 +- .../route.tsx | 8 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../route.tsx | 7 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../route.tsx | 5 +- .../webapp/app/routes/api.v1.prompts.$slug.ts | 15 +- .../routes/api.v1.prompts.$slug.versions.ts | 12 +- .../app/routes/api.v1.prompts._index.ts | 5 +- apps/webapp/app/routes/otel.v1.logs.ts | 5 +- apps/webapp/app/routes/otel.v1.metrics.ts | 5 +- apps/webapp/app/routes/otel.v1.traces.ts | 5 +- ...projectParam.env.$envParam.logs.$logId.tsx | 5 +- ...ojects.$projectParam.env.$envParam.logs.ts | 5 +- ...nvParam.prompts.$promptSlug.generations.ts | 5 +- .../services/admin/missingLlmModels.server.ts | 10 +- .../clickhouseCredentialsService.server.ts | 109 +++++ .../clickhouse/clickhouseFactory.server.ts | 422 ++++++++++++++++++ .../clickhouse/clickhouseFactory.test.ts | 155 +++++++ .../clickhouseSecretSchemas.server.ts | 11 + .../app/services/clickhouseInstance.server.ts | 130 ------ .../app/services/queryService.server.ts | 5 +- .../services/runsReplicationService.server.ts | 107 ++++- apps/webapp/app/v3/otlpExporter.server.ts | 47 +- .../v3/services/bulk/BulkActionV2.server.ts | 8 +- 39 files changed, 959 insertions(+), 234 deletions(-) create mode 100644 .server-changes/organization-scoped-clickhouse.md create mode 100644 apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts create mode 100644 apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts create mode 100644 apps/webapp/app/services/clickhouse/clickhouseFactory.test.ts create mode 100644 apps/webapp/app/services/clickhouse/clickhouseSecretSchemas.server.ts delete mode 100644 apps/webapp/app/services/clickhouseInstance.server.ts diff --git a/.cursor/mcp.json b/.cursor/mcp.json index da39e4ffafe..c4b06a67630 100644 --- a/.cursor/mcp.json +++ b/.cursor/mcp.json @@ -1,3 +1,7 @@ { - "mcpServers": {} + "mcpServers": { + "linear": { + "url": "https://mcp.linear.app/mcp" + } + } } diff --git a/.server-changes/organization-scoped-clickhouse.md b/.server-changes/organization-scoped-clickhouse.md new file mode 100644 index 00000000000..874b9dc6026 --- /dev/null +++ b/.server-changes/organization-scoped-clickhouse.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Organization-scoped ClickHouse routing enables customers with HIPAA and other data security requirements to use dedicated database instances diff --git a/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts index 254ec18d1c0..b0ba01b9dc6 100644 --- a/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts @@ -9,7 +9,7 @@ import { type Project, type RuntimeEnvironment, type TaskRunStatus } from "@trig import assertNever from "assert-never"; import { z } from "zod"; import { API_VERSIONS, RunStatusUnspecifiedApiVersion } from "~/api/versions"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { logger } from "~/services/logger.server"; import { CoercedDate } from "~/utils/zod"; import { ServiceValidationError } from "~/v3/services/baseService.server"; @@ -259,7 +259,8 @@ export class ApiRunListPresenter extends BasePresenter { options.machines = searchParams["filter[machine]"]; } - const presenter = new NextRunListPresenter(this._replica, clickhouseClient); + const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); + const presenter = new NextRunListPresenter(this._replica, clickhouse); logger.debug("Calling RunListPresenter", { options }); diff --git a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts index acf511f0f5e..5e8bfc405b8 100644 --- a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts @@ -1,6 +1,6 @@ import { type PrismaClient } from "@trigger.dev/database"; import { CreateBulkActionSearchParams } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; import { getRunFiltersFromRequest } from "../RunFilters.server"; import { BasePresenter } from "./basePresenter.server"; @@ -24,8 +24,9 @@ export class CreateBulkActionPresenter extends BasePresenter { Object.fromEntries(new URL(request.url).searchParams) ); + const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); const runsRepository = new RunsRepository({ - clickhouse: clickhouseClient, + clickhouse, prisma: this._replica as PrismaClient, }); diff --git a/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts b/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts index e9de368eceb..89b9c8b41fa 100644 --- a/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts @@ -1,6 +1,6 @@ import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; import { BasePresenter } from "./basePresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { type PrismaClient } from "@trigger.dev/database"; import { timeFilters } from "~/components/runs/v3/SharedFilters"; @@ -37,8 +37,9 @@ export class RunTagListPresenter extends BasePresenter { }: TagListOptions) { const hasFilters = Boolean(name?.trim()); + const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); const runsRepository = new RunsRepository({ - clickhouse: clickhouseClient, + clickhouse, prisma: this._replica as PrismaClient, }); diff --git a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts index f1635f23375..a6471c30c60 100644 --- a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts @@ -4,7 +4,7 @@ import { type TaskTriggerSource, } from "@trigger.dev/database"; import { $replica } from "~/db.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { type AverageDurations, ClickHouseEnvironmentMetricsRepository, @@ -25,10 +25,7 @@ export type TaskListItem = { export type TaskActivity = DailyTaskActivity[string]; export class TaskListPresenter { - constructor( - private readonly environmentMetricsRepository: EnvironmentMetricsRepository, - private readonly _replica: PrismaClientOrTransaction - ) {} + constructor(private readonly _replica: PrismaClientOrTransaction) {} public async call({ organizationId, @@ -76,9 +73,15 @@ export class TaskListPresenter { const slugs = tasks.map((t) => t.slug); + // Create org-specific environment metrics repository + const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); + const environmentMetricsRepository = new ClickHouseEnvironmentMetricsRepository({ + clickhouse, + }); + // IMPORTANT: Don't await these, we want to return the promises // so we can defer the loading of the data - const activity = this.environmentMetricsRepository.getDailyTaskActivity({ + const activity = environmentMetricsRepository.getDailyTaskActivity({ organizationId, projectId, environmentId, @@ -86,7 +89,7 @@ export class TaskListPresenter { tasks: slugs, }); - const runningStats = this.environmentMetricsRepository.getCurrentRunningStats({ + const runningStats = environmentMetricsRepository.getCurrentRunningStats({ organizationId, projectId, environmentId, @@ -94,7 +97,7 @@ export class TaskListPresenter { tasks: slugs, }); - const durations = this.environmentMetricsRepository.getAverageDurations({ + const durations = environmentMetricsRepository.getAverageDurations({ organizationId, projectId, environmentId, @@ -109,9 +112,5 @@ export class TaskListPresenter { export const taskListPresenter = singleton("taskListPresenter", setupTaskListPresenter); function setupTaskListPresenter() { - const environmentMetricsRepository = new ClickHouseEnvironmentMetricsRepository({ - clickhouse: clickhouseClient, - }); - - return new TaskListPresenter(environmentMetricsRepository, $replica); + return new TaskListPresenter($replica); } diff --git a/apps/webapp/app/presenters/v3/UsagePresenter.server.ts b/apps/webapp/app/presenters/v3/UsagePresenter.server.ts index 2fac95617a6..c4654e870ed 100644 --- a/apps/webapp/app/presenters/v3/UsagePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/UsagePresenter.server.ts @@ -4,7 +4,7 @@ import { getUsage, getUsageSeries } from "~/services/platform.v3.server"; import { createTimeSeriesData } from "~/utils/graphs"; import { BasePresenter } from "./basePresenter.server"; import { DataPoint, linear } from "regression"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; type Options = { organizationId: string; @@ -124,7 +124,8 @@ async function getTaskUsageByOrganization( endOfMonth: Date, replica: PrismaClientOrTransaction ) { - const [queryError, tasks] = await clickhouseClient.taskRuns.getTaskUsageByOrganization({ + const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); + const [queryError, tasks] = await clickhouse.taskRuns.getTaskUsageByOrganization({ startTime: startOfMonth.getTime(), endTime: endOfMonth.getTime(), organizationId, diff --git a/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts b/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts index f0e955fd04d..52ebad96b4e 100644 --- a/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts @@ -1,7 +1,7 @@ import { ScheduleObject } from "@trigger.dev/core/v3"; import { PrismaClient, prisma } from "~/db.server"; import { displayableEnvironment } from "~/models/runtimeEnvironment.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { nextScheduledTimestamps } from "~/v3/utils/calculateNextSchedule.server"; import { NextRunListPresenter } from "./NextRunListPresenter.server"; import { scheduleWhereClause } from "~/models/schedules.server"; @@ -75,7 +75,8 @@ export class ViewSchedulePresenter { ? nextScheduledTimestamps(schedule.generatorExpression, schedule.timezone, new Date(), 5) : []; - const runPresenter = new NextRunListPresenter(this.#prismaClient, clickhouseClient); + const clickhouse = await getClickhouseForOrganization(schedule.project.organizationId, "standard"); + const runPresenter = new NextRunListPresenter(this.#prismaClient, clickhouse); const { runs } = await runPresenter.call(schedule.project.organizationId, environmentId, { projectId: schedule.project.id, scheduleId: schedule.id, diff --git a/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts b/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts index 9abcdf32215..15eaef0d13d 100644 --- a/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts @@ -1,5 +1,5 @@ import { isWaitpointOutputTimeout, prettyPrintPacket } from "@trigger.dev/core/v3"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; import { logger } from "~/services/logger.server"; import { BasePresenter } from "./basePresenter.server"; @@ -79,7 +79,8 @@ export class WaitpointPresenter extends BasePresenter { const connectedRuns: NextRunListItem[] = []; if (connectedRunIds.length > 0) { - const runPresenter = new NextRunListPresenter(this._prisma, clickhouseClient); + const clickhouse = await getClickhouseForOrganization(waitpoint.environment.organizationId, "standard"); + const runPresenter = new NextRunListPresenter(this._prisma, clickhouse); const { runs } = await runPresenter.call( waitpoint.environment.organizationId, environmentId, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx index cd358b7e67d..9bbd1c04b5c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx @@ -31,7 +31,7 @@ import { MetricDashboardPresenter, } from "~/presenters/v3/MetricDashboardPresenter.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { requireUser } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; @@ -74,10 +74,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const filters = dashboard.filters ?? ["tasks", "queues"]; + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + // Load distinct models from ClickHouse if the dashboard has a models filter let possibleModels: { model: string; system: string }[] = []; if (filters.includes("models")) { - const queryFn = clickhouseClient.reader.query({ + const queryFn = clickhouse.reader.query({ name: "getDistinctModels", query: `SELECT response_model, any(gen_ai_system) AS gen_ai_system FROM trigger_dev.llm_metrics_v1 WHERE organization_id = {organizationId: String} AND project_id = {projectId: String} AND environment_id = {environmentId: String} AND response_model != '' GROUP BY response_model ORDER BY response_model`, params: z.object({ @@ -97,7 +99,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } } - const promptPresenter = new PromptPresenter(clickhouseClient); + const promptPresenter = new PromptPresenter(clickhouse); const [possiblePrompts, possibleOperations, possibleProviders] = await Promise.all([ filters.includes("prompts") ? promptPresenter.getDistinctPromptSlugs(project.organizationId, project.id, environment.id) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx index e92b5b34644..35f8131189b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx @@ -70,7 +70,7 @@ import { type ErrorOccurrences, type ErrorsList as ErrorsListData, } from "~/presenters/v3/ErrorsListPresenter.server"; -import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; import { requireUser } from "~/services/session.server"; import { formatNumberCompact } from "~/utils/numberFormatter"; @@ -123,7 +123,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const plan = await getCurrentPlan(project.organizationId); const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30; - const presenter = new ErrorsListPresenter($replica, logsClickhouseClient); + const logsClickhouse = await getClickhouseForOrganization(project.organizationId, "logs"); + const presenter = new ErrorsListPresenter($replica, logsClickhouse); const listPromise = presenter .call(project.organizationId, environment.id, { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index af3cc30a246..bccb7125daf 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -16,7 +16,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { LogsListPresenter, LogEntry } from "~/presenters/v3/LogsListPresenter.server"; import type { LogLevel } from "~/utils/logUtils"; import { $replica, prisma } from "~/db.server"; -import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { Suspense, useCallback, useEffect, useMemo, useRef, useState, useTransition } from "react"; @@ -137,7 +137,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const plan = await getCurrentPlan(project.organizationId); const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30; - const presenter = new LogsListPresenter($replica, logsClickhouseClient); + const logsClickhouse = await getClickhouseForOrganization(project.organizationId, "logs"); + const presenter = new LogsListPresenter($replica, logsClickhouse); const listPromise = presenter .call(project.organizationId, environment.id, { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx index 7a25f996d4d..4256c64d455 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx @@ -28,7 +28,7 @@ import type { QueryWidgetConfig } from "~/components/metrics/QueryWidget"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { ModelRegistryPresenter } from "~/presenters/v3/ModelRegistryPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { requireUserId } from "~/services/session.server"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; @@ -68,7 +68,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Environment not found", { status: 404 }); } - const presenter = new ModelRegistryPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new ModelRegistryPresenter(clickhouse); const model = await presenter.getModelDetail(modelId); if (!model) { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx index 7bf257f987b..dca4ca9483b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx @@ -71,7 +71,7 @@ import { type PopularModel, ModelRegistryPresenter, } from "~/presenters/v3/ModelRegistryPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { requireUserId } from "~/services/session.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; @@ -109,7 +109,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Environment not found", { status: 404 }); } - const presenter = new ModelRegistryPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new ModelRegistryPresenter(clickhouse); const catalog = await presenter.getModelCatalog(); const now = new Date(); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx index 661fb294268..879dcf47e6b 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx @@ -20,7 +20,7 @@ import { type ModelComparisonItem, ModelRegistryPresenter, } from "~/presenters/v3/ModelRegistryPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { requireUserId } from "~/services/session.server"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; @@ -55,7 +55,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { return typedjson({ comparison: [] as ModelComparisonItem[], models: responseModels }); } - const presenter = new ModelRegistryPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new ModelRegistryPresenter(clickhouse); const now = new Date(); const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx index 5a953c0199b..f37e8d3fed9 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx @@ -70,7 +70,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { type GenerationRow, PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; import { SpanView } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { getResizableSnapshot } from "~/services/resizablePanel.server"; import { requireUserId } from "~/services/session.server"; import { PromptService } from "~/v3/services/promptService.server"; @@ -242,7 +242,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const startTime = fromTime ? new Date(fromTime) : new Date(Date.now() - periodMs); const endTime = toTime ? new Date(toTime) : new Date(); - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); let generations: Awaited>["generations"] = []; let generationsPagination: { next?: string } = {}; try { @@ -273,7 +274,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { // Load distinct filter values and resizable snapshots in parallel const distinctQuery = (col: string, name: string) => - clickhouseClient.reader.query({ + clickhouse.reader.query({ name, query: `SELECT DISTINCT ${col} AS val FROM trigger_dev.llm_metrics_v1 WHERE environment_id = {environmentId: String} AND prompt_slug = {promptSlug: String} AND ${col} != '' ORDER BY val`, params: z.object({ environmentId: z.string(), promptSlug: z.string() }), diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx index 02c7cc444b7..4e229a48f74 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx @@ -22,7 +22,7 @@ import { useProject } from "~/hooks/useProject"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { requireUserId } from "~/services/session.server"; import { docsPath, EnvironmentParamSchema, v3PromptsPath } from "~/utils/pathBuilder"; import { LinkButton } from "~/components/primitives/Buttons"; @@ -46,7 +46,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Environment not found", { status: 404 }); } - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); const prompts = await presenter.listPrompts(project.id, environment.id); const sparklines = await presenter.getUsageSparklines( diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx index f35376a4275..422f5367e4c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx @@ -92,7 +92,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server"; import { RunEnvironmentMismatchError, RunPresenter } from "~/presenters/v3/RunPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { getImpersonationId } from "~/services/impersonation.server"; import { logger } from "~/services/logger.server"; import { getResizableSnapshot } from "~/services/resizablePanel.server"; @@ -182,7 +182,8 @@ async function getRunsListFromTableState({ return null; } - const runsListPresenter = new NextRunListPresenter($replica, clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const runsListPresenter = new NextRunListPresenter($replica, clickhouse); const currentPageResult = await runsListPresenter.call(project.organizationId, environment.id, { userId, projectId: project.id, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx index ca7e8b7b0c1..ba3cd4b83ae 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx @@ -44,7 +44,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { setRootOnlyFilterPreference, uiPreferencesStorage, @@ -87,7 +87,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const filters = await getRunFiltersFromRequest(request); - const presenter = new NextRunListPresenter($replica, clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new NextRunListPresenter($replica, clickhouse); const list = presenter.call(project.organizationId, environment.id, { userId, projectId: project.id, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx index ee69419e1b7..38356c6a247 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx @@ -74,7 +74,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components import { DialogClose, DialogDescription } from "@radix-ui/react-dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { $replica } from "~/db.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { RegionsPresenter, type Region } from "~/presenters/v3/RegionsPresenter.server"; import { TestSidebarTabs } from "./TestSidebarTabs"; import { AIPayloadTabContent } from "./AIPayloadTabContent"; @@ -102,7 +102,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }); } - const presenter = new TestTaskPresenter($replica, clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new TestTaskPresenter($replica, clickhouse); try { const [result, regionsResult] = await Promise.all([ presenter.call({ diff --git a/apps/webapp/app/routes/api.v1.prompts.$slug.ts b/apps/webapp/app/routes/api.v1.prompts.$slug.ts index 32ea1525c14..230ceb1277f 100644 --- a/apps/webapp/app/routes/api.v1.prompts.$slug.ts +++ b/apps/webapp/app/routes/api.v1.prompts.$slug.ts @@ -2,7 +2,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { createActionApiRoute, createLoaderApiRoute, @@ -33,6 +33,13 @@ export const loader = createLoaderApiRoute( slug: params.slug, }, }, + include: { + project: { + select: { + organizationId: true, + }, + }, + }, }); }, authorization: { @@ -46,7 +53,8 @@ export const loader = createLoaderApiRoute( return json({ error: "Prompt not found" }, { status: 404 }); } - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(prompt.project.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); const version = await presenter.resolveVersion(prompt.id, { version: searchParams.version, label: searchParams.label, @@ -117,7 +125,8 @@ const { action } = createActionApiRoute( return json({ error: "Prompt not found" }, { status: 404 }); } - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(authentication.environment.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); const version = await presenter.resolveVersion(prompt.id, { version: body.version, label: body.label, diff --git a/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts b/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts index c40b3e62dbf..17b88b12c42 100644 --- a/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts +++ b/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts @@ -2,7 +2,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; const ParamsSchema = z.object({ @@ -23,6 +23,13 @@ export const loader = createLoaderApiRoute( slug: params.slug, }, }, + include: { + project: { + select: { + organizationId: true, + }, + }, + }, }); }, authorization: { @@ -36,7 +43,8 @@ export const loader = createLoaderApiRoute( return json({ error: "Prompt not found" }, { status: 404 }); } - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(prompt.project.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); const versions = await presenter.listVersions(prompt.id); return json({ diff --git a/apps/webapp/app/routes/api.v1.prompts._index.ts b/apps/webapp/app/routes/api.v1.prompts._index.ts index ccbc0ec38d0..44f2f86d0cb 100644 --- a/apps/webapp/app/routes/api.v1.prompts._index.ts +++ b/apps/webapp/app/routes/api.v1.prompts._index.ts @@ -1,6 +1,6 @@ import { json } from "@remix-run/server-runtime"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; export const loader = createLoaderApiRoute( @@ -15,7 +15,8 @@ export const loader = createLoaderApiRoute( }, }, async ({ authentication }) => { - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(authentication.environment.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); const prompts = await presenter.listPrompts( authentication.environment.projectId, authentication.environment.id diff --git a/apps/webapp/app/routes/otel.v1.logs.ts b/apps/webapp/app/routes/otel.v1.logs.ts index a05ddd24cf2..1dc7c07c16c 100644 --- a/apps/webapp/app/routes/otel.v1.logs.ts +++ b/apps/webapp/app/routes/otel.v1.logs.ts @@ -4,12 +4,13 @@ import { otlpExporter } from "~/v3/otlpExporter.server"; export async function action({ request }: ActionFunctionArgs) { try { + const exporter = await otlpExporter; const contentType = request.headers.get("content-type")?.toLowerCase() ?? ""; if (contentType.startsWith("application/json")) { const body = await request.json(); - const exportResponse = await otlpExporter.exportLogs(body as ExportLogsServiceRequest); + const exportResponse = await exporter.exportLogs(body as ExportLogsServiceRequest); return json(exportResponse, { status: 200 }); } else if (contentType.startsWith("application/x-protobuf")) { @@ -17,7 +18,7 @@ export async function action({ request }: ActionFunctionArgs) { const exportRequest = ExportLogsServiceRequest.decode(new Uint8Array(buffer)); - const exportResponse = await otlpExporter.exportLogs(exportRequest); + const exportResponse = await exporter.exportLogs(exportRequest); return new Response(ExportLogsServiceResponse.encode(exportResponse).finish(), { status: 200, diff --git a/apps/webapp/app/routes/otel.v1.metrics.ts b/apps/webapp/app/routes/otel.v1.metrics.ts index 5529f9310ec..9a09cb18233 100644 --- a/apps/webapp/app/routes/otel.v1.metrics.ts +++ b/apps/webapp/app/routes/otel.v1.metrics.ts @@ -7,12 +7,13 @@ import { otlpExporter } from "~/v3/otlpExporter.server"; export async function action({ request }: ActionFunctionArgs) { try { + const exporter = await otlpExporter; const contentType = request.headers.get("content-type")?.toLowerCase() ?? ""; if (contentType.startsWith("application/json")) { const body = await request.json(); - const exportResponse = await otlpExporter.exportMetrics( + const exportResponse = await exporter.exportMetrics( body as ExportMetricsServiceRequest ); @@ -22,7 +23,7 @@ export async function action({ request }: ActionFunctionArgs) { const exportRequest = ExportMetricsServiceRequest.decode(new Uint8Array(buffer)); - const exportResponse = await otlpExporter.exportMetrics(exportRequest); + const exportResponse = await exporter.exportMetrics(exportRequest); return new Response(ExportMetricsServiceResponse.encode(exportResponse).finish(), { status: 200, diff --git a/apps/webapp/app/routes/otel.v1.traces.ts b/apps/webapp/app/routes/otel.v1.traces.ts index 609b72c0465..8e974c7b1dd 100644 --- a/apps/webapp/app/routes/otel.v1.traces.ts +++ b/apps/webapp/app/routes/otel.v1.traces.ts @@ -4,12 +4,13 @@ import { otlpExporter } from "~/v3/otlpExporter.server"; export async function action({ request }: ActionFunctionArgs) { try { + const exporter = await otlpExporter; const contentType = request.headers.get("content-type")?.toLowerCase() ?? ""; if (contentType.startsWith("application/json")) { const body = await request.json(); - const exportResponse = await otlpExporter.exportTraces(body as ExportTraceServiceRequest); + const exportResponse = await exporter.exportTraces(body as ExportTraceServiceRequest); return json(exportResponse, { status: 200 }); } else if (contentType.startsWith("application/x-protobuf")) { @@ -17,7 +18,7 @@ export async function action({ request }: ActionFunctionArgs) { const exportRequest = ExportTraceServiceRequest.decode(new Uint8Array(buffer)); - const exportResponse = await otlpExporter.exportTraces(exportRequest); + const exportResponse = await exporter.exportTraces(exportRequest); return new Response(ExportTraceServiceResponse.encode(exportResponse).finish(), { status: 200, diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx index f862ced6b05..0e0469bcd1b 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx @@ -1,7 +1,7 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { typedjson } from "remix-typedjson"; import { z } from "zod"; -import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { requireUserId } from "~/services/session.server"; import { LogDetailPresenter } from "~/presenters/v3/LogDetailPresenter.server"; import { findProjectBySlug } from "~/models/project.server"; @@ -43,7 +43,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const [traceId, spanId, , startTime] = parts; - const presenter = new LogDetailPresenter($replica, logsClickhouseClient); + const logsClickhouse = await getClickhouseForOrganization(project.organizationId, "logs"); + const presenter = new LogDetailPresenter($replica, logsClickhouse); let result; try { diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts index 66ddebe4e2a..d55c7496258 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts @@ -6,7 +6,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server"; import { $replica } from "~/db.server"; -import { logsClickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; // Valid log levels for filtering @@ -69,7 +69,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { retentionLimitDays, }) as any; // Validated by LogsListOptionsSchema at runtime - const presenter = new LogsListPresenter($replica, logsClickhouseClient); + const logsClickhouse = await getClickhouseForOrganization(project.organizationId, "logs"); + const presenter = new LogsListPresenter($replica, logsClickhouse); const result = await presenter.call(project.organizationId, environment.id, options); return json({ diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts index 77a55ec3f0b..17a11e05837 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts @@ -6,7 +6,7 @@ import { EnvironmentParamSchema } from "~/utils/pathBuilder"; import { parsePeriodToMs } from "~/utils/periods"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { PromptPresenter, type GenerationRow, @@ -59,7 +59,8 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const operations = url.searchParams.getAll("operations").filter(Boolean); const providers = url.searchParams.getAll("providers").filter(Boolean); - const presenter = new PromptPresenter(clickhouseClient); + const clickhouse = await getClickhouseForOrganization(project.organizationId, "standard"); + const presenter = new PromptPresenter(clickhouse); const result = await presenter.listGenerations({ environmentId: environment.id, promptSlug, diff --git a/apps/webapp/app/services/admin/missingLlmModels.server.ts b/apps/webapp/app/services/admin/missingLlmModels.server.ts index 7ce6bc2ab7e..07e6160ee03 100644 --- a/apps/webapp/app/services/admin/missingLlmModels.server.ts +++ b/apps/webapp/app/services/admin/missingLlmModels.server.ts @@ -1,4 +1,4 @@ -import { adminClickhouseClient } from "~/services/clickhouseInstance.server"; +import { getAdminClickhouse } from "~/services/clickhouse/clickhouseFactory.server"; import { llmPricingRegistry } from "~/v3/llmPricingRegistry.server"; export type MissingLlmModel = { @@ -13,8 +13,10 @@ export async function getMissingLlmModels(opts: { const lookbackHours = opts.lookbackHours ?? 24; const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000); + const adminClickhouse = getAdminClickhouse(); + // queryBuilderFast returns a factory function — call it to get the builder - const createBuilder = adminClickhouseClient.reader.queryBuilderFast<{ + const createBuilder = adminClickhouse.reader.queryBuilderFast<{ model: string; system: string; cnt: string; @@ -93,7 +95,9 @@ export async function getMissingModelSamples(opts: { const limit = opts.limit ?? 10; const since = new Date(Date.now() - lookbackHours * 60 * 60 * 1000); - const createBuilder = adminClickhouseClient.reader.queryBuilderFast({ + const adminClickhouse = getAdminClickhouse(); + + const createBuilder = adminClickhouse.reader.queryBuilderFast({ name: "missingModelSamples", table: "trigger_dev.task_events_v2", columns: [ diff --git a/apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts b/apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts new file mode 100644 index 00000000000..c2c8c77f7c4 --- /dev/null +++ b/apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts @@ -0,0 +1,109 @@ +import { getSecretStore } from "~/services/secrets/secretStore.server"; +import { prisma } from "~/db.server"; +import { + ClickhouseConnectionSchema, + getClickhouseSecretKey, +} from "./clickhouseSecretSchemas.server"; +import { clearClickhouseCacheForOrganization } from "./clickhouseFactory.server"; + +export async function setOrganizationClickhouseUrl( + organizationId: string, + clientType: "standard" | "events" | "replication", + url: string +): Promise { + // Validate URL format + const connection = ClickhouseConnectionSchema.parse({ url }); + + // Store in SecretStore + const secretStore = getSecretStore("DATABASE"); + const secretKey = getClickhouseSecretKey(organizationId, clientType); + await secretStore.setSecret(secretKey, connection); + + // Update featureFlags to reference the secret + const org = await prisma.organization.findUnique({ + where: { id: organizationId }, + select: { featureFlags: true }, + }); + + const featureFlags = (org?.featureFlags || {}) as any; + const clickhouseConfig = featureFlags.clickhouse || {}; + clickhouseConfig[clientType] = secretKey; + featureFlags.clickhouse = clickhouseConfig; + + await prisma.organization.update({ + where: { id: organizationId }, + data: { featureFlags }, + }); + + // Clear cache + clearClickhouseCacheForOrganization(organizationId); +} + +export async function removeOrganizationClickhouseUrl( + organizationId: string, + clientType: "standard" | "events" | "replication" +): Promise { + // Remove from SecretStore + const secretStore = getSecretStore("DATABASE"); + const secretKey = getClickhouseSecretKey(organizationId, clientType); + await secretStore.deleteSecret(secretKey); + + // Update featureFlags + const org = await prisma.organization.findUnique({ + where: { id: organizationId }, + select: { featureFlags: true }, + }); + + if (org?.featureFlags) { + const featureFlags = org.featureFlags as any; + if (featureFlags.clickhouse && featureFlags.clickhouse[clientType]) { + delete featureFlags.clickhouse[clientType]; + + // If no more clickhouse configs, remove the clickhouse key entirely + if (Object.keys(featureFlags.clickhouse).length === 0) { + delete featureFlags.clickhouse; + } + + await prisma.organization.update({ + where: { id: organizationId }, + data: { featureFlags }, + }); + } + } + + // Clear cache + clearClickhouseCacheForOrganization(organizationId); +} + +export async function getOrganizationClickhouseUrl( + organizationId: string, + clientType: "standard" | "events" | "replication" +): Promise { + const org = await prisma.organization.findUnique({ + where: { id: organizationId }, + select: { featureFlags: true }, + }); + + if (!org?.featureFlags) { + return null; + } + + const clickhouseConfig = (org.featureFlags as any).clickhouse; + if (!clickhouseConfig || typeof clickhouseConfig !== "object") { + return null; + } + + const secretKey = clickhouseConfig[clientType]; + if (!secretKey || typeof secretKey !== "string") { + return null; + } + + const secretStore = getSecretStore("DATABASE"); + const connection = await secretStore.getSecret(ClickhouseConnectionSchema, secretKey); + + if (!connection) { + return null; + } + + return connection.url; +} diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts new file mode 100644 index 00000000000..94498868759 --- /dev/null +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -0,0 +1,422 @@ +/** + * ClickHouse Factory - Organization-Scoped ClickHouse Routing + * + * This module provides organization-scoped ClickHouse instance routing to support: + * - HIPAA compliance (dedicated ClickHouse clusters) + * - High-volume customer isolation + * - Geographic data residency requirements + * - Performance tier differentiation + * + * ## Architecture + * + * ### Credential Storage + * - ClickHouse URLs stored encrypted in SecretStore (AES-256-GCM) + * - Organization references secret via `featureFlags.clickhouse` JSON + * - No plaintext credentials in database + * + * ### Caching Strategy + * - **Org configs**: Unkey cache with LRU memory (5min fresh, 10min stale, SWR) + * - **ClickHouse clients**: Cached by hostname hash (multiple orgs share same instance) + * - **Event repositories**: Cached by hostname hash (stateful, must be reused) + * - **Security**: Memory-only cache for org configs (no credentials in Redis) + * + * ## Usage in Presenters + * + * Presenters should fetch org-specific ClickHouse clients in their `call()` method: + * + * ```typescript + * import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; + * + * export class MyPresenter extends BasePresenter { + * constructor(private options: PresenterOptions = {}) { + * super(); + * } + * + * async call({ organizationId, ... }) { + * const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); + * // Use clickhouse for queries... + * } + * } + * ``` + * + * ## Usage in Services + * + * The replication service and OTLP exporter automatically route data by organization. + * Other services should follow the same pattern when working with ClickHouse. + * + * @module clickhouseFactory + */ + +import { ClickHouse } from "@internal/clickhouse"; +import { createHash } from "crypto"; +import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache"; +import { createLRUMemoryStore } from "@internal/cache"; +import { getSecretStore } from "~/services/secrets/secretStore.server"; +import { prisma } from "~/db.server"; +import { + ClickhouseConnectionSchema, + getClickhouseSecretKey, +} from "./clickhouseSecretSchemas.server"; +import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server"; +import { env } from "~/env.server"; +import { singleton } from "~/utils/singleton"; + +// Module-level caches for ClickHouse clients and event repositories +const clickhouseClientCache = new Map(); +const eventRepositoryCache = new Map(); + +// Default ClickHouse clients (not exported - internal use only) +const defaultClickhouseClient = singleton("clickhouseClient", initializeClickhouseClient); + +function initializeClickhouseClient() { + const url = new URL(env.CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + console.log(`🗃️ Clickhouse service enabled to host ${url.host}`); + + return new ClickHouse({ + url: url.toString(), + name: "clickhouse-instance", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); +} + +const defaultLogsClickhouseClient = singleton( + "logsClickhouseClient", + initializeLogsClickhouseClient +); + +function initializeLogsClickhouseClient() { + if (!env.LOGS_CLICKHOUSE_URL) { + throw new Error("LOGS_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.LOGS_CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "logs-clickhouse", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + clickhouseSettings: { + max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), + max_bytes_before_external_sort: + env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), + max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, + ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { + max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), + }), + ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { + max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, + }), + }, + }); +} + +const defaultAdminClickhouseClient = singleton( + "adminClickhouseClient", + initializeAdminClickhouseClient +); + +function initializeAdminClickhouseClient() { + if (!env.ADMIN_CLICKHOUSE_URL) { + throw new Error("ADMIN_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.ADMIN_CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "admin-clickhouse", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); +} + +const defaultQueryClickhouseClient = singleton( + "queryClickhouseClient", + initializeQueryClickhouseClient +); + +function initializeQueryClickhouseClient() { + if (!env.QUERY_CLICKHOUSE_URL) { + throw new Error("QUERY_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.QUERY_CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "query-clickhouse", + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); +} + +// Org config cache with Unkey (memory-only, no Redis for security) +type OrgClickhouseConfig = { + organizationId: string; + hostnameHash: string; + url: string; + clientType: string; +}; + +const ctx = new DefaultStatefulContext(); +const memory = createLRUMemoryStore(1000); + +const orgConfigCache = createCache({ + orgClickhouse: new Namespace(ctx, { + stores: [memory], // Memory-only, no Redis store for security + fresh: 5 * 60 * 1000, // 5 minutes + stale: 10 * 60 * 1000, // 10 minutes (SWR pattern) + }), +}); + +function hashHostname(url: string): string { + const parsed = new URL(url); + return createHash("sha256").update(parsed.hostname).digest("hex"); +} + +async function getOrgClickhouseConfig( + ctx: DefaultStatefulContext, + orgId: string, + clientType: string +): Promise { + const org = await prisma.organization.findUnique({ + where: { id: orgId }, + select: { featureFlags: true }, + }); + + if (!org?.featureFlags) { + return null; + } + + const clickhouseConfig = (org.featureFlags as any).clickhouse; + if (!clickhouseConfig || typeof clickhouseConfig !== "object") { + return null; + } + + const secretKey = clickhouseConfig[clientType]; + if (!secretKey || typeof secretKey !== "string") { + return null; + } + + const secretStore = getSecretStore("DATABASE"); + const connection = await secretStore.getSecret(ClickhouseConnectionSchema, secretKey); + + if (!connection) { + return null; + } + + const hostnameHash = hashHostname(connection.url); + + return { + organizationId: orgId, + hostnameHash, + url: connection.url, + clientType, + }; +} + +export async function getClickhouseForOrganization( + organizationId: string, + clientType: "standard" | "events" | "replication" | "logs" | "query" | "admin" +): Promise { + // Try to get org-specific config + const configResult = await orgConfigCache.orgClickhouse.swr( + `org:${organizationId}:ch:${clientType}`, + async () => getOrgClickhouseConfig(ctx, organizationId, clientType) + ); + + // Handle Result type - check for error or null value + const config = configResult.err ? null : configResult.val; + + // If no custom config, return appropriate default client + if (!config) { + switch (clientType) { + case "standard": + case "events": + case "replication": + return defaultClickhouseClient; + case "logs": + return defaultLogsClickhouseClient; + case "query": + return defaultQueryClickhouseClient; + case "admin": + return defaultAdminClickhouseClient; + } + } + + // Check if client already exists for this hostname + const cacheKey = `${config.hostnameHash}:${clientType}`; + let client = clickhouseClientCache.get(cacheKey); + + if (!client) { + const url = new URL(config.url); + url.searchParams.delete("secure"); + + client = new ClickHouse({ + url: url.toString(), + name: `org-clickhouse-${clientType}`, + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { + request: true, + }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); + clickhouseClientCache.set(cacheKey, client); + } + + return client; +} + +export async function getEventRepositoryForOrganization( + organizationId: string +): Promise { + // Try to get org-specific config + const configResult = await orgConfigCache.orgClickhouse.swr( + `org:${organizationId}:ch:events`, + async () => getOrgClickhouseConfig(ctx, organizationId, "events") + ); + + // Handle Result type - check for error or null value + const config = configResult.err ? null : configResult.val; + + // If no custom config, return default repository (created on demand) + if (!config) { + const defaultKey = "default:events"; + let defaultRepo = eventRepositoryCache.get(defaultKey); + if (!defaultRepo) { + // Create default event repository using standard clickhouse client + // This matches the existing pattern in clickhouseEventRepositoryInstance.server.ts + const eventsClickhouse = await getEventsClickhouseClient(); + defaultRepo = new ClickhouseEventRepository({ + clickhouse: eventsClickhouse, + batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE, + flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS, + maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT, + maximumTraceDetailedSummaryViewCount: + env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT, + maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING, + insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY, + waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1", + asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, + asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, + startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS, + llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, + llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, + llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, + llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, + version: "v2", + }); + eventRepositoryCache.set(defaultKey, defaultRepo); + } + return defaultRepo; + } + + // Check if repository already exists for this hostname + const cacheKey = `${config.hostnameHash}:events`; + let repository = eventRepositoryCache.get(cacheKey); + + if (!repository) { + const client = await getClickhouseForOrganization(organizationId, "events"); + repository = new ClickhouseEventRepository({ + clickhouse: client, + batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE, + flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS, + maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT, + maximumTraceDetailedSummaryViewCount: + env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT, + maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING, + insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY, + waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1", + asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, + asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, + startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS, + llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, + llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, + llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, + llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, + version: "v2", + }); + eventRepositoryCache.set(cacheKey, repository); + } + + return repository; +} + +// Helper to create the default events ClickHouse client +async function getEventsClickhouseClient(): Promise { + if (!env.EVENTS_CLICKHOUSE_URL) { + throw new Error("EVENTS_CLICKHOUSE_URL is not set"); + } + + const url = new URL(env.EVENTS_CLICKHOUSE_URL); + url.searchParams.delete("secure"); + + return new ClickHouse({ + url: url.toString(), + name: "task-events", + keepAlive: { + enabled: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.EVENTS_CLICKHOUSE_LOG_LEVEL, + compression: { + request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1", + }, + maxOpenConnections: env.EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); +} + +/** + * Get admin ClickHouse client for cross-organization queries + * This should only be used for admin tools and analytics that need to query across all orgs + */ +export function getAdminClickhouse(): ClickHouse { + return defaultAdminClickhouseClient; +} + +// Clear caches when needed (e.g., when org config changes) +export function clearClickhouseCacheForOrganization(organizationId: string): void { + // The Unkey cache will naturally expire based on TTL (5min fresh, 10min stale) + // No explicit removal needed - cache entries will be refreshed on next access + // Note: We don't clear client/repository caches as they're keyed by hostname + // and may be shared by other orgs +} diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.test.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.test.ts new file mode 100644 index 00000000000..f0b24b941b1 --- /dev/null +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.test.ts @@ -0,0 +1,155 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { prisma } from "~/db.server"; +import { + getClickhouseForOrganization, + getEventRepositoryForOrganization, + clearClickhouseCacheForOrganization, +} from "./clickhouseFactory.server"; +import { + setOrganizationClickhouseUrl, + removeOrganizationClickhouseUrl, + getOrganizationClickhouseUrl, +} from "./clickhouseCredentialsService.server"; + +describe("ClickHouse Factory", () => { + const testOrgId = "test-org-" + Date.now(); + const testClickhouseUrl = "https://test-ch.example.com:8443?user=test&password=secret"; + + beforeEach(async () => { + // Clean up any existing test data + await prisma.organization.deleteMany({ + where: { id: testOrgId }, + }); + }); + + it("should return default ClickHouse client when org has no custom config", async () => { + const client = await getClickhouseForOrganization(testOrgId, "standard"); + expect(client).toBeDefined(); + // Default client should be returned (not null) + expect(client).toBeTruthy(); + }); + + it("should set and retrieve organization ClickHouse URL", async () => { + // First create the test organization + await prisma.organization.create({ + data: { + id: testOrgId, + title: "Test Org", + slug: "test-org-" + Date.now(), + }, + }); + + // Set the URL + await setOrganizationClickhouseUrl(testOrgId, "standard", testClickhouseUrl); + + // Retrieve it + const retrievedUrl = await getOrganizationClickhouseUrl(testOrgId, "standard"); + expect(retrievedUrl).toBe(testClickhouseUrl); + + // Verify it's stored in featureFlags + const org = await prisma.organization.findUnique({ + where: { id: testOrgId }, + select: { featureFlags: true }, + }); + + expect(org?.featureFlags).toBeDefined(); + const featureFlags = org?.featureFlags as any; + expect(featureFlags.clickhouse).toBeDefined(); + expect(featureFlags.clickhouse.standard).toBeDefined(); + + // Clean up + await removeOrganizationClickhouseUrl(testOrgId, "standard"); + await prisma.organization.delete({ where: { id: testOrgId } }); + }); + + it("should remove organization ClickHouse URL", async () => { + // First create the test organization + await prisma.organization.create({ + data: { + id: testOrgId, + title: "Test Org", + slug: "test-org-" + Date.now(), + }, + }); + + // Set and then remove + await setOrganizationClickhouseUrl(testOrgId, "standard", testClickhouseUrl); + await removeOrganizationClickhouseUrl(testOrgId, "standard"); + + // Verify it's gone + const retrievedUrl = await getOrganizationClickhouseUrl(testOrgId, "standard"); + expect(retrievedUrl).toBeNull(); + + // Clean up + await prisma.organization.delete({ where: { id: testOrgId } }); + }); + + it("should cache ClickHouse clients by hostname", async () => { + // This test verifies that multiple orgs pointing to the same ClickHouse hostname + // share the same client instance (deduplication) + + const org1Id = testOrgId + "-1"; + const org2Id = testOrgId + "-2"; + + // Create test organizations + await prisma.organization.createMany({ + data: [ + { id: org1Id, title: "Test Org 1", slug: "test-org-1-" + Date.now() }, + { id: org2Id, title: "Test Org 2", slug: "test-org-2-" + Date.now() }, + ], + }); + + // Set both orgs to use the same ClickHouse URL + await setOrganizationClickhouseUrl(org1Id, "standard", testClickhouseUrl); + await setOrganizationClickhouseUrl(org2Id, "standard", testClickhouseUrl); + + // Get clients for both orgs + const client1 = await getClickhouseForOrganization(org1Id, "standard"); + const client2 = await getClickhouseForOrganization(org2Id, "standard"); + + // Both should be defined + expect(client1).toBeDefined(); + expect(client2).toBeDefined(); + + // They should be the same instance (cached by hostname) + expect(client1).toBe(client2); + + // Clean up + await removeOrganizationClickhouseUrl(org1Id, "standard"); + await removeOrganizationClickhouseUrl(org2Id, "standard"); + await prisma.organization.deleteMany({ + where: { id: { in: [org1Id, org2Id] } }, + }); + }); + + it("should clear cache when organization config changes", async () => { + // Create test organization + await prisma.organization.create({ + data: { + id: testOrgId, + title: "Test Org", + slug: "test-org-" + Date.now(), + }, + }); + + // Set URL + await setOrganizationClickhouseUrl(testOrgId, "standard", testClickhouseUrl); + + // Get client to populate cache + const client1 = await getClickhouseForOrganization(testOrgId, "standard"); + + // Clear cache + clearClickhouseCacheForOrganization(testOrgId); + + // Get client again (should hit the database again, not cache) + const client2 = await getClickhouseForOrganization(testOrgId, "standard"); + + // Both should be defined + expect(client1).toBeDefined(); + expect(client2).toBeDefined(); + + // Clean up + await removeOrganizationClickhouseUrl(testOrgId, "standard"); + await prisma.organization.delete({ where: { id: testOrgId } }); + }); +}); diff --git a/apps/webapp/app/services/clickhouse/clickhouseSecretSchemas.server.ts b/apps/webapp/app/services/clickhouse/clickhouseSecretSchemas.server.ts new file mode 100644 index 00000000000..016eb717c18 --- /dev/null +++ b/apps/webapp/app/services/clickhouse/clickhouseSecretSchemas.server.ts @@ -0,0 +1,11 @@ +import { z } from "zod"; + +export const ClickhouseConnectionSchema = z.object({ + url: z.string().url(), +}); + +export type ClickhouseConnection = z.infer; + +export function getClickhouseSecretKey(orgId: string, clientType: string): string { + return `org:${orgId}:clickhouse:${clientType}`; +} diff --git a/apps/webapp/app/services/clickhouseInstance.server.ts b/apps/webapp/app/services/clickhouseInstance.server.ts deleted file mode 100644 index 9c4941671f3..00000000000 --- a/apps/webapp/app/services/clickhouseInstance.server.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { ClickHouse } from "@internal/clickhouse"; -import { env } from "~/env.server"; -import { singleton } from "~/utils/singleton"; - -export const clickhouseClient = singleton("clickhouseClient", initializeClickhouseClient); - -function initializeClickhouseClient() { - const url = new URL(env.CLICKHOUSE_URL); - - // Remove secure param - url.searchParams.delete("secure"); - - console.log(`🗃️ Clickhouse service enabled to host ${url.host}`); - - const clickhouse = new ClickHouse({ - url: url.toString(), - name: "clickhouse-instance", - keepAlive: { - enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", - idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, - }, - logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, - maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - }); - - return clickhouse; -} - -export const logsClickhouseClient = singleton( - "logsClickhouseClient", - initializeLogsClickhouseClient -); - -function initializeLogsClickhouseClient() { - if (!env.LOGS_CLICKHOUSE_URL) { - throw new Error("LOGS_CLICKHOUSE_URL is not set"); - } - - const url = new URL(env.LOGS_CLICKHOUSE_URL); - - // Remove secure param - url.searchParams.delete("secure"); - - return new ClickHouse({ - url: url.toString(), - name: "logs-clickhouse", - keepAlive: { - enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", - idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, - }, - logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, - maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - clickhouseSettings: { - max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), - max_bytes_before_external_sort: - env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), - max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, - ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { - max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), - }), - ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { - max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, - }), - }, - }); -} - -export const adminClickhouseClient = singleton( - "adminClickhouseClient", - initializeAdminClickhouseClient -); - -function initializeAdminClickhouseClient() { - if (!env.ADMIN_CLICKHOUSE_URL) { - throw new Error("ADMIN_CLICKHOUSE_URL is not set"); - } - - const url = new URL(env.ADMIN_CLICKHOUSE_URL); - url.searchParams.delete("secure"); - - return new ClickHouse({ - url: url.toString(), - name: "admin-clickhouse", - keepAlive: { - enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", - idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, - }, - logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, - maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - }); -} - -export const queryClickhouseClient = singleton( - "queryClickhouseClient", - initializeQueryClickhouseClient -); - -function initializeQueryClickhouseClient() { - if (!env.QUERY_CLICKHOUSE_URL) { - throw new Error("QUERY_CLICKHOUSE_URL is not set"); - } - - const url = new URL(env.QUERY_CLICKHOUSE_URL); - - // Remove secure param - url.searchParams.delete("secure"); - - return new ClickHouse({ - url: url.toString(), - name: "query-clickhouse", - keepAlive: { - enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", - idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, - }, - logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, - maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - }); -} diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 1f3bdbba18a..f24df9eb023 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -11,7 +11,7 @@ import type { TableSchema, WhereClauseCondition } from "@internal/tsql"; import { z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { queryClickhouseClient } from "./clickhouseInstance.server"; +import { getClickhouseForOrganization } from "./clickhouse/clickhouseFactory.server"; import { queryConcurrencyLimiter, DEFAULT_ORG_CONCURRENCY_LIMIT, @@ -275,7 +275,8 @@ export async function executeQuery( environment: Object.fromEntries(environments.map((e) => [e.id, e.slug])), }; - const result = await executeTSQL(queryClickhouseClient.reader, { + const queryClickhouse = await getClickhouseForOrganization(organizationId, "query"); + const result = await executeTSQL(queryClickhouse.reader, { ...baseOptions, schema: z.record(z.any()), tableSchema: querySchemas, diff --git a/apps/webapp/app/services/runsReplicationService.server.ts b/apps/webapp/app/services/runsReplicationService.server.ts index 7930c05481f..ca1fba686a2 100644 --- a/apps/webapp/app/services/runsReplicationService.server.ts +++ b/apps/webapp/app/services/runsReplicationService.server.ts @@ -617,18 +617,65 @@ export class RunsReplicationService { payloadInserts: payloadInserts.length, }); + // Group task runs by organization for routing to correct ClickHouse instance + const taskRunsByOrg = new Map(); + for (const taskRun of taskRunInserts) { + const orgId = getTaskRunField(taskRun, "organization_id"); + const orgRuns = taskRunsByOrg.get(orgId) || []; + orgRuns.push(taskRun); + taskRunsByOrg.set(orgId, orgRuns); + } + + // Group payloads by organization (extract from run_id -> task runs mapping) + const payloadsByOrg = new Map(); + for (const payload of payloadInserts) { + const runId = getPayloadField(payload, "run_id"); + // Find the corresponding task run to get its organization + const taskRun = taskRunInserts.find((tr) => getTaskRunField(tr, "run_id") === runId); + if (taskRun) { + const orgId = getTaskRunField(taskRun, "organization_id"); + const orgPayloads = payloadsByOrg.get(orgId) || []; + orgPayloads.push(payload); + payloadsByOrg.set(orgId, orgPayloads); + } + } + // Insert task runs and payloads with retry logic for connection errors - const [taskRunError, taskRunResult] = await this.#insertWithRetry( - (attempt) => this.#insertTaskRunInserts(taskRunInserts, attempt), - "task run inserts", - flushId + // Process each organization's data in parallel + const insertPromises = Array.from(taskRunsByOrg.entries()).map( + async ([orgId, orgTaskRuns]) => { + const orgPayloads = payloadsByOrg.get(orgId) || []; + + const [taskRunError, taskRunResult] = await this.#insertWithRetry( + (attempt) => this.#insertTaskRunInserts(orgId, orgTaskRuns, attempt), + "task run inserts", + flushId + ); + + const [payloadError, payloadResult] = await this.#insertWithRetry( + (attempt) => this.#insertPayloadInserts(orgId, orgPayloads, attempt), + "payload inserts", + flushId + ); + + return { taskRunError, payloadError, orgId }; + } ); - const [payloadError, payloadResult] = await this.#insertWithRetry( - (attempt) => this.#insertPayloadInserts(payloadInserts, attempt), - "payload inserts", - flushId - ); + const results = await Promise.all(insertPromises); + + // Aggregate errors from all organizations + let taskRunError: Error | null = null; + let payloadError: Error | null = null; + + for (const result of results) { + if (result.taskRunError) { + taskRunError = result.taskRunError; + } + if (result.payloadError) { + payloadError = result.payloadError; + } + } // Log any errors that occurred if (taskRunError) { @@ -770,19 +817,32 @@ export class RunsReplicationService { }; } - async #insertTaskRunInserts(taskRunInserts: TaskRunInsertArray[], attempt: number) { + async #insertTaskRunInserts( + organizationId: string, + taskRunInserts: TaskRunInsertArray[], + attempt: number + ) { return await startSpan(this._tracer, "insertTaskRunsInserts", async (span) => { - const [insertError, insertResult] = - await this.options.clickhouse.taskRuns.insertCompactArrays(taskRunInserts, { + // Get the appropriate ClickHouse client for this organization + const { getClickhouseForOrganization } = await import( + "~/services/clickhouse/clickhouseFactory.server" + ); + const clickhouse = await getClickhouseForOrganization(organizationId, "replication"); + + const [insertError, insertResult] = await clickhouse.taskRuns.insertCompactArrays( + taskRunInserts, + { params: { clickhouse_settings: this.#getClickhouseInsertSettings(), }, - }); + } + ); if (insertError) { this.logger.error("Error inserting task run inserts attempt", { error: insertError, attempt, + organizationId, }); recordSpanError(span, insertError); @@ -793,19 +853,32 @@ export class RunsReplicationService { }); } - async #insertPayloadInserts(payloadInserts: PayloadInsertArray[], attempt: number) { + async #insertPayloadInserts( + organizationId: string, + payloadInserts: PayloadInsertArray[], + attempt: number + ) { return await startSpan(this._tracer, "insertPayloadInserts", async (span) => { - const [insertError, insertResult] = - await this.options.clickhouse.taskRuns.insertPayloadsCompactArrays(payloadInserts, { + // Get the appropriate ClickHouse client for this organization + const { getClickhouseForOrganization } = await import( + "~/services/clickhouse/clickhouseFactory.server" + ); + const clickhouse = await getClickhouseForOrganization(organizationId, "replication"); + + const [insertError, insertResult] = await clickhouse.taskRuns.insertPayloadsCompactArrays( + payloadInserts, + { params: { clickhouse_settings: this.#getClickhouseInsertSettings(), }, - }); + } + ); if (insertError) { this.logger.error("Error inserting payload inserts attempt", { error: insertError, attempt, + organizationId, }); recordSpanError(span, insertError); diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 7505693e3ab..f8b22d7c442 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -20,7 +20,6 @@ import { } from "@trigger.dev/otlp-importer"; import type { MetricsV1Input } from "@internal/clickhouse"; import { logger } from "~/services/logger.server"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server"; import { ClickhouseEventRepository } from "./eventRepository/clickhouseEventRepository.server"; import { @@ -118,21 +117,26 @@ class OTLPExporter { async #exportEvents( eventsWithStores: { events: Array; taskEventStore: string }[] ) { - const eventsGroupedByStore = eventsWithStores.reduce((acc, { events, taskEventStore }) => { - acc[taskEventStore] = acc[taskEventStore] || []; - acc[taskEventStore].push(...events); + // Group events by both store and organization for proper routing + const eventsGroupedByStoreAndOrg = eventsWithStores.reduce((acc, { events, taskEventStore }) => { + for (const event of events) { + const orgId = event.organizationId || "default"; + const key = `${taskEventStore}:${orgId}`; + acc[key] = acc[key] || { store: taskEventStore, orgId, events: [] }; + acc[key].events.push(event); + } return acc; - }, {} as Record>); + }, {} as Record }>); let eventCount = 0; - for (const [store, events] of Object.entries(eventsGroupedByStore)) { - const eventRepository = this.#getEventRepositoryForStore(store); + for (const { store, orgId, events } of Object.values(eventsGroupedByStoreAndOrg)) { + const eventRepository = await this.#getEventRepositoryForStoreAndOrg(store, orgId); await waitForLlmPricingReady(); const enrichedEvents = enrichCreatableEvents(events); - this.#logEventsVerbose(enrichedEvents, `exportEvents ${store}`); + this.#logEventsVerbose(enrichedEvents, `exportEvents ${store}:${orgId}`); eventCount += enrichedEvents.length; @@ -142,6 +146,19 @@ class OTLPExporter { return eventCount; } + async #getEventRepositoryForStoreAndOrg(store: string, orgId: string): Promise { + // For ClickHouse stores with a specific org (not "default"), use org-specific repository + if ((store === "clickhouse" || store === "clickhouse_v2") && orgId !== "default") { + const { getEventRepositoryForOrganization } = await import( + "~/services/clickhouse/clickhouseFactory.server" + ); + return await getEventRepositoryForOrganization(orgId); + } + + // Fall back to default repositories for non-ClickHouse stores or default org + return this.#getEventRepositoryForStore(store); + } + #getEventRepositoryForStore(store: string): IEventRepository { if (store === "clickhouse") { return this._clickhouseEventRepository; @@ -1172,12 +1189,22 @@ function hasUnpairedSurrogateAtEnd(str: string): boolean { export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter); -function initializeOTLPExporter() { +async function initializeOTLPExporter() { + // Metrics are written globally (not per-org), use standard clickhouse + // We use a dummy org ID since metrics table is global + const { getClickhouseForOrganization } = await import( + "~/services/clickhouse/clickhouseFactory.server" + ); + + // Use a sentinel org ID for global metrics writes + // In practice, all orgs currently share the same metrics table/instance + const metricsClickhouse = await getClickhouseForOrganization("METRICS_GLOBAL", "standard"); + const metricsFlushScheduler = new DynamicFlushScheduler({ batchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE, flushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS, callback: async (_flushId, batch) => { - await clickhouseClient.metrics.insert(batch); + await metricsClickhouse.metrics.insert(batch); }, minConcurrency: 1, maxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY, diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts index 156b68bff59..07a4286297f 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts @@ -7,7 +7,7 @@ import { } from "@trigger.dev/database"; import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; import { type CreateBulkActionPayload } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction"; -import { clickhouseClient } from "~/services/clickhouseInstance.server"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; import { parseRunListInputOptions, type RunListInputFilters, @@ -38,8 +38,9 @@ export class BulkActionService extends BaseService { const filters = await getFilters(payload, request); // Count the runs that will be affected by the bulk action + const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); const runsRepository = new RunsRepository({ - clickhouse: clickhouseClient, + clickhouse, prisma: this._replica as PrismaClient, }); const count = await runsRepository.countRuns({ @@ -147,8 +148,9 @@ export class BulkActionService extends BaseService { ...rawParams, }); + const clickhouse = await getClickhouseForOrganization(group.project.organizationId, "standard"); const runsRepository = new RunsRepository({ - clickhouse: clickhouseClient, + clickhouse, prisma: this._replica as PrismaClient, }); From 357252aa5973fe1fe32bcce701211a8dd0c77fd7 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Mar 2026 17:32:05 +0000 Subject: [PATCH 02/32] Better replication performance --- .../services/runsReplicationService.server.ts | 102 +++++++++++------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/apps/webapp/app/services/runsReplicationService.server.ts b/apps/webapp/app/services/runsReplicationService.server.ts index ca1fba686a2..78b7e084e31 100644 --- a/apps/webapp/app/services/runsReplicationService.server.ts +++ b/apps/webapp/app/services/runsReplicationService.server.ts @@ -29,6 +29,7 @@ import EventEmitter from "node:events"; import pLimit from "p-limit"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { calculateErrorFingerprint } from "~/utils/errorFingerprinting"; +import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; interface TransactionEvent { tag: "insert" | "update" | "delete"; @@ -617,51 +618,59 @@ export class RunsReplicationService { payloadInserts: payloadInserts.length, }); - // Group task runs by organization for routing to correct ClickHouse instance - const taskRunsByOrg = new Map(); - for (const taskRun of taskRunInserts) { - const orgId = getTaskRunField(taskRun, "organization_id"); - const orgRuns = taskRunsByOrg.get(orgId) || []; - orgRuns.push(taskRun); - taskRunsByOrg.set(orgId, orgRuns); - } + // Task runs are already sorted by org (lines 571-576), so we can stream through + // and flush when org changes - no grouping overhead, no O(n²) lookups - // Group payloads by organization (extract from run_id -> task runs mapping) + // Build run_id -> org_id index for O(1) payload->org lookups + const runIdToOrgId = new Map( + taskRunInserts.map(tr => [getTaskRunField(tr, "run_id"), getTaskRunField(tr, "organization_id")]) + ); + + // Group payloads by org using the index (O(n) instead of O(n²)) const payloadsByOrg = new Map(); for (const payload of payloadInserts) { const runId = getPayloadField(payload, "run_id"); - // Find the corresponding task run to get its organization - const taskRun = taskRunInserts.find((tr) => getTaskRunField(tr, "run_id") === runId); - if (taskRun) { - const orgId = getTaskRunField(taskRun, "organization_id"); - const orgPayloads = payloadsByOrg.get(orgId) || []; - orgPayloads.push(payload); - payloadsByOrg.set(orgId, orgPayloads); + const orgId = runIdToOrgId.get(runId); + if (orgId) { + const orgPayloads = payloadsByOrg.get(orgId); + if (orgPayloads) { + orgPayloads.push(payload); + } else { + payloadsByOrg.set(orgId, [payload]); + } } } - // Insert task runs and payloads with retry logic for connection errors - // Process each organization's data in parallel - const insertPromises = Array.from(taskRunsByOrg.entries()).map( - async ([orgId, orgTaskRuns]) => { - const orgPayloads = payloadsByOrg.get(orgId) || []; + // Stream through task runs, flushing when org changes + const insertPromises: Promise<{ taskRunError: Error | null; payloadError: Error | null; orgId: string }>[] = []; + let currentOrgId: string | null = null; + let currentOrgTaskRuns: TaskRunInsertArray[] = []; - const [taskRunError, taskRunResult] = await this.#insertWithRetry( - (attempt) => this.#insertTaskRunInserts(orgId, orgTaskRuns, attempt), - "task run inserts", - flushId - ); + for (const taskRun of taskRunInserts) { + const orgId = getTaskRunField(taskRun, "organization_id"); - const [payloadError, payloadResult] = await this.#insertWithRetry( - (attempt) => this.#insertPayloadInserts(orgId, orgPayloads, attempt), - "payload inserts", - flushId + // Org changed? Flush previous org's batch + if (currentOrgId !== null && currentOrgId !== orgId) { + const orgPayloads = payloadsByOrg.get(currentOrgId) || []; + insertPromises.push( + this.#insertOrgBatch(currentOrgId, currentOrgTaskRuns, orgPayloads, flushId) ); - - return { taskRunError, payloadError, orgId }; + currentOrgTaskRuns = []; } - ); + currentOrgId = orgId; + currentOrgTaskRuns.push(taskRun); + } + + // Flush final org's batch + if (currentOrgId !== null && currentOrgTaskRuns.length > 0) { + const orgPayloads = payloadsByOrg.get(currentOrgId) || []; + insertPromises.push( + this.#insertOrgBatch(currentOrgId, currentOrgTaskRuns, orgPayloads, flushId) + ); + } + + // Wait for all org batches to complete (parallel execution) const results = await Promise.all(insertPromises); // Aggregate errors from all organizations @@ -817,6 +826,27 @@ export class RunsReplicationService { }; } + async #insertOrgBatch( + organizationId: string, + taskRunInserts: TaskRunInsertArray[], + payloadInserts: PayloadInsertArray[], + flushId: string + ): Promise<{ taskRunError: Error | null; payloadError: Error | null; orgId: string }> { + const [taskRunError] = await this.#insertWithRetry( + (attempt) => this.#insertTaskRunInserts(organizationId, taskRunInserts, attempt), + "task run inserts", + flushId + ); + + const [payloadError] = await this.#insertWithRetry( + (attempt) => this.#insertPayloadInserts(organizationId, payloadInserts, attempt), + "payload inserts", + flushId + ); + + return { taskRunError, payloadError, orgId: organizationId }; + } + async #insertTaskRunInserts( organizationId: string, taskRunInserts: TaskRunInsertArray[], @@ -824,9 +854,6 @@ export class RunsReplicationService { ) { return await startSpan(this._tracer, "insertTaskRunsInserts", async (span) => { // Get the appropriate ClickHouse client for this organization - const { getClickhouseForOrganization } = await import( - "~/services/clickhouse/clickhouseFactory.server" - ); const clickhouse = await getClickhouseForOrganization(organizationId, "replication"); const [insertError, insertResult] = await clickhouse.taskRuns.insertCompactArrays( @@ -860,9 +887,6 @@ export class RunsReplicationService { ) { return await startSpan(this._tracer, "insertPayloadInserts", async (span) => { // Get the appropriate ClickHouse client for this organization - const { getClickhouseForOrganization } = await import( - "~/services/clickhouse/clickhouseFactory.server" - ); const clickhouse = await getClickhouseForOrganization(organizationId, "replication"); const [insertError, insertResult] = await clickhouse.taskRuns.insertPayloadsCompactArrays( From db8b8a76538f3953518af88c413c0b0c0d20bffd Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 26 Mar 2026 17:32:19 +0000 Subject: [PATCH 03/32] Removed dynamic imports --- CLAUDE.md | 11 +++++++++++ apps/webapp/app/v3/otlpExporter.server.ts | 9 +-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 79d931a4548..23003f50983 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,6 +66,17 @@ containerTest("should use both", async ({ prisma, redisOptions }) => { }); ``` +## Code Style + +### Imports + +**Prefer static imports over dynamic imports.** Only use dynamic `import()` when: +- Circular dependencies cannot be resolved otherwise +- Code splitting is genuinely needed for performance +- The module must be loaded conditionally at runtime + +Dynamic imports add unnecessary overhead in hot paths and make code harder to analyze. If you find yourself using `await import()`, ask if a regular `import` statement would work instead. + ## Changesets and Server Changes When modifying any public package (`packages/*` or `integrations/*`), add a changeset: diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index f8b22d7c442..fd16717a584 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -40,6 +40,7 @@ import { waitForLlmPricingReady } from "./llmPricingRegistry.server"; import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; +import { getClickhouseForOrganization, getEventRepositoryForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; class OTLPExporter { private _tracer: Tracer; @@ -149,9 +150,6 @@ class OTLPExporter { async #getEventRepositoryForStoreAndOrg(store: string, orgId: string): Promise { // For ClickHouse stores with a specific org (not "default"), use org-specific repository if ((store === "clickhouse" || store === "clickhouse_v2") && orgId !== "default") { - const { getEventRepositoryForOrganization } = await import( - "~/services/clickhouse/clickhouseFactory.server" - ); return await getEventRepositoryForOrganization(orgId); } @@ -1191,11 +1189,6 @@ export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter); async function initializeOTLPExporter() { // Metrics are written globally (not per-org), use standard clickhouse - // We use a dummy org ID since metrics table is global - const { getClickhouseForOrganization } = await import( - "~/services/clickhouse/clickhouseFactory.server" - ); - // Use a sentinel org ID for global metrics writes // In practice, all orgs currently share the same metrics table/instance const metricsClickhouse = await getClickhouseForOrganization("METRICS_GLOBAL", "standard"); From 8c03961a01208e8311c68c874ce94b8c249bde4a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 27 Mar 2026 16:48:56 +0000 Subject: [PATCH 04/32] otlpExporter.server reverted to main --- apps/webapp/app/v3/otlpExporter.server.ts | 40 ++++++----------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index fd16717a584..7505693e3ab 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -20,6 +20,7 @@ import { } from "@trigger.dev/otlp-importer"; import type { MetricsV1Input } from "@internal/clickhouse"; import { logger } from "~/services/logger.server"; +import { clickhouseClient } from "~/services/clickhouseInstance.server"; import { DynamicFlushScheduler } from "./dynamicFlushScheduler.server"; import { ClickhouseEventRepository } from "./eventRepository/clickhouseEventRepository.server"; import { @@ -40,7 +41,6 @@ import { waitForLlmPricingReady } from "./llmPricingRegistry.server"; import { env } from "~/env.server"; import { detectBadJsonStrings } from "~/utils/detectBadJsonStrings"; import { singleton } from "~/utils/singleton"; -import { getClickhouseForOrganization, getEventRepositoryForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; class OTLPExporter { private _tracer: Tracer; @@ -118,26 +118,21 @@ class OTLPExporter { async #exportEvents( eventsWithStores: { events: Array; taskEventStore: string }[] ) { - // Group events by both store and organization for proper routing - const eventsGroupedByStoreAndOrg = eventsWithStores.reduce((acc, { events, taskEventStore }) => { - for (const event of events) { - const orgId = event.organizationId || "default"; - const key = `${taskEventStore}:${orgId}`; - acc[key] = acc[key] || { store: taskEventStore, orgId, events: [] }; - acc[key].events.push(event); - } + const eventsGroupedByStore = eventsWithStores.reduce((acc, { events, taskEventStore }) => { + acc[taskEventStore] = acc[taskEventStore] || []; + acc[taskEventStore].push(...events); return acc; - }, {} as Record }>); + }, {} as Record>); let eventCount = 0; - for (const { store, orgId, events } of Object.values(eventsGroupedByStoreAndOrg)) { - const eventRepository = await this.#getEventRepositoryForStoreAndOrg(store, orgId); + for (const [store, events] of Object.entries(eventsGroupedByStore)) { + const eventRepository = this.#getEventRepositoryForStore(store); await waitForLlmPricingReady(); const enrichedEvents = enrichCreatableEvents(events); - this.#logEventsVerbose(enrichedEvents, `exportEvents ${store}:${orgId}`); + this.#logEventsVerbose(enrichedEvents, `exportEvents ${store}`); eventCount += enrichedEvents.length; @@ -147,16 +142,6 @@ class OTLPExporter { return eventCount; } - async #getEventRepositoryForStoreAndOrg(store: string, orgId: string): Promise { - // For ClickHouse stores with a specific org (not "default"), use org-specific repository - if ((store === "clickhouse" || store === "clickhouse_v2") && orgId !== "default") { - return await getEventRepositoryForOrganization(orgId); - } - - // Fall back to default repositories for non-ClickHouse stores or default org - return this.#getEventRepositoryForStore(store); - } - #getEventRepositoryForStore(store: string): IEventRepository { if (store === "clickhouse") { return this._clickhouseEventRepository; @@ -1187,17 +1172,12 @@ function hasUnpairedSurrogateAtEnd(str: string): boolean { export const otlpExporter = singleton("otlpExporter", initializeOTLPExporter); -async function initializeOTLPExporter() { - // Metrics are written globally (not per-org), use standard clickhouse - // Use a sentinel org ID for global metrics writes - // In practice, all orgs currently share the same metrics table/instance - const metricsClickhouse = await getClickhouseForOrganization("METRICS_GLOBAL", "standard"); - +function initializeOTLPExporter() { const metricsFlushScheduler = new DynamicFlushScheduler({ batchSize: env.METRICS_CLICKHOUSE_BATCH_SIZE, flushInterval: env.METRICS_CLICKHOUSE_FLUSH_INTERVAL_MS, callback: async (_flushId, batch) => { - await metricsClickhouse.metrics.insert(batch); + await clickhouseClient.metrics.insert(batch); }, minConcurrency: 1, maxConcurrency: env.METRICS_CLICKHOUSE_MAX_CONCURRENCY, From 77949e8cb9b9d7b363db28a03d6282af32ffd4eb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 31 Mar 2026 22:44:36 +0100 Subject: [PATCH 05/32] Switch to a DataStore registry --- apps/webapp/app/env.server.ts | 3 + .../clickhouseCredentialsService.server.ts | 109 ------ .../clickhouse/clickhouseFactory.server.ts | 340 ++++++++---------- ...ganizationDataStoreConfigSchemas.server.ts | 35 ++ .../organizationDataStoresRegistry.server.ts | 82 +++++ ...zationDataStoresRegistryInstance.server.ts | 46 +++ .../migration.sql | 21 ++ .../database/prisma/schema.prisma | 23 +- 8 files changed, 362 insertions(+), 297 deletions(-) delete mode 100644 apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts create mode 100644 apps/webapp/app/services/dataStores/organizationDataStoreConfigSchemas.server.ts create mode 100644 apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts create mode 100644 apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts create mode 100644 internal-packages/database/prisma/migrations/20260331212308_add_organization_data_stores/migration.sql diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 8d72b2e51b2..306acb5077b 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -1298,6 +1298,9 @@ const EnvironmentSchema = z EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(5_000), EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING: z.coerce.number().int().default(2000), + // Organization data stores registry + ORGANIZATION_DATA_STORES_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + // LLM cost tracking LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes diff --git a/apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts b/apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts deleted file mode 100644 index c2c8c77f7c4..00000000000 --- a/apps/webapp/app/services/clickhouse/clickhouseCredentialsService.server.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { getSecretStore } from "~/services/secrets/secretStore.server"; -import { prisma } from "~/db.server"; -import { - ClickhouseConnectionSchema, - getClickhouseSecretKey, -} from "./clickhouseSecretSchemas.server"; -import { clearClickhouseCacheForOrganization } from "./clickhouseFactory.server"; - -export async function setOrganizationClickhouseUrl( - organizationId: string, - clientType: "standard" | "events" | "replication", - url: string -): Promise { - // Validate URL format - const connection = ClickhouseConnectionSchema.parse({ url }); - - // Store in SecretStore - const secretStore = getSecretStore("DATABASE"); - const secretKey = getClickhouseSecretKey(organizationId, clientType); - await secretStore.setSecret(secretKey, connection); - - // Update featureFlags to reference the secret - const org = await prisma.organization.findUnique({ - where: { id: organizationId }, - select: { featureFlags: true }, - }); - - const featureFlags = (org?.featureFlags || {}) as any; - const clickhouseConfig = featureFlags.clickhouse || {}; - clickhouseConfig[clientType] = secretKey; - featureFlags.clickhouse = clickhouseConfig; - - await prisma.organization.update({ - where: { id: organizationId }, - data: { featureFlags }, - }); - - // Clear cache - clearClickhouseCacheForOrganization(organizationId); -} - -export async function removeOrganizationClickhouseUrl( - organizationId: string, - clientType: "standard" | "events" | "replication" -): Promise { - // Remove from SecretStore - const secretStore = getSecretStore("DATABASE"); - const secretKey = getClickhouseSecretKey(organizationId, clientType); - await secretStore.deleteSecret(secretKey); - - // Update featureFlags - const org = await prisma.organization.findUnique({ - where: { id: organizationId }, - select: { featureFlags: true }, - }); - - if (org?.featureFlags) { - const featureFlags = org.featureFlags as any; - if (featureFlags.clickhouse && featureFlags.clickhouse[clientType]) { - delete featureFlags.clickhouse[clientType]; - - // If no more clickhouse configs, remove the clickhouse key entirely - if (Object.keys(featureFlags.clickhouse).length === 0) { - delete featureFlags.clickhouse; - } - - await prisma.organization.update({ - where: { id: organizationId }, - data: { featureFlags }, - }); - } - } - - // Clear cache - clearClickhouseCacheForOrganization(organizationId); -} - -export async function getOrganizationClickhouseUrl( - organizationId: string, - clientType: "standard" | "events" | "replication" -): Promise { - const org = await prisma.organization.findUnique({ - where: { id: organizationId }, - select: { featureFlags: true }, - }); - - if (!org?.featureFlags) { - return null; - } - - const clickhouseConfig = (org.featureFlags as any).clickhouse; - if (!clickhouseConfig || typeof clickhouseConfig !== "object") { - return null; - } - - const secretKey = clickhouseConfig[clientType]; - if (!secretKey || typeof secretKey !== "string") { - return null; - } - - const secretStore = getSecretStore("DATABASE"); - const connection = await secretStore.getSecret(ClickhouseConnectionSchema, secretKey); - - if (!connection) { - return null; - } - - return connection.url; -} diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index 94498868759..69a7555384a 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -11,61 +11,45 @@ * * ### Credential Storage * - ClickHouse URLs stored encrypted in SecretStore (AES-256-GCM) - * - Organization references secret via `featureFlags.clickhouse` JSON + * - Organization data store overrides live in the `OrganizationDataStore` table + * - The config JSON stores a `secretKey` that references the SecretStore entry * - No plaintext credentials in database * * ### Caching Strategy - * - **Org configs**: Unkey cache with LRU memory (5min fresh, 10min stale, SWR) - * - **ClickHouse clients**: Cached by hostname hash (multiple orgs share same instance) - * - **Event repositories**: Cached by hostname hash (stateful, must be reused) - * - **Security**: Memory-only cache for org configs (no credentials in Redis) + * - **Org → data store mapping**: `OrganizationDataStoresRegistry` (in-memory Map, reloaded + * periodically via setInterval) + * - **SecretKey → resolved URL**: module-level Map (persists for process lifetime) + * - **ClickHouse clients**: cached by hostname hash (multiple orgs share same instance) + * - **Event repositories**: cached by hostname hash (stateful, must be reused) * * ## Usage in Presenters * - * Presenters should fetch org-specific ClickHouse clients in their `call()` method: - * * ```typescript * import { getClickhouseForOrganization } from "~/services/clickhouse/clickhouseFactory.server"; * * export class MyPresenter extends BasePresenter { - * constructor(private options: PresenterOptions = {}) { - * super(); - * } - * * async call({ organizationId, ... }) { * const clickhouse = await getClickhouseForOrganization(organizationId, "standard"); - * // Use clickhouse for queries... * } * } * ``` * - * ## Usage in Services - * - * The replication service and OTLP exporter automatically route data by organization. - * Other services should follow the same pattern when working with ClickHouse. - * * @module clickhouseFactory */ import { ClickHouse } from "@internal/clickhouse"; import { createHash } from "crypto"; -import { createCache, DefaultStatefulContext, Namespace } from "@unkey/cache"; -import { createLRUMemoryStore } from "@internal/cache"; import { getSecretStore } from "~/services/secrets/secretStore.server"; -import { prisma } from "~/db.server"; -import { - ClickhouseConnectionSchema, - getClickhouseSecretKey, -} from "./clickhouseSecretSchemas.server"; +import { ClickhouseConnectionSchema } from "./clickhouseSecretSchemas.server"; import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; +import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; -// Module-level caches for ClickHouse clients and event repositories -const clickhouseClientCache = new Map(); -const eventRepositoryCache = new Map(); +// --------------------------------------------------------------------------- +// Default clients (singleton per process) +// --------------------------------------------------------------------------- -// Default ClickHouse clients (not exported - internal use only) const defaultClickhouseClient = singleton("clickhouseClient", initializeClickhouseClient); function initializeClickhouseClient() { @@ -82,9 +66,7 @@ function initializeClickhouseClient() { idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, }, logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, + compression: { request: true }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); } @@ -110,9 +92,7 @@ function initializeLogsClickhouseClient() { idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, }, logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, + compression: { request: true }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, clickhouseSettings: { max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), @@ -150,9 +130,7 @@ function initializeAdminClickhouseClient() { idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, }, logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, + compression: { request: true }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); } @@ -178,93 +156,108 @@ function initializeQueryClickhouseClient() { idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, }, logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, + compression: { request: true }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); } -// Org config cache with Unkey (memory-only, no Redis for security) -type OrgClickhouseConfig = { - organizationId: string; - hostnameHash: string; - url: string; - clientType: string; -}; - -const ctx = new DefaultStatefulContext(); -const memory = createLRUMemoryStore(1000); - -const orgConfigCache = createCache({ - orgClickhouse: new Namespace(ctx, { - stores: [memory], // Memory-only, no Redis store for security - fresh: 5 * 60 * 1000, // 5 minutes - stale: 10 * 60 * 1000, // 10 minutes (SWR pattern) - }), -}); +// --------------------------------------------------------------------------- +// Org-scoped client caches +// --------------------------------------------------------------------------- + +/** ClickHouse clients keyed by hostname hash (shared across orgs pointing at the same host). */ +const clickhouseClientCache = new Map(); + +/** Event repositories keyed by hostname hash (stateful, must be reused). */ +const eventRepositoryCache = new Map(); + +/** Resolved connection URLs keyed by secret-store key (avoids repeated secret fetches). */ +const resolvedConnectionCache = new Map(); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- function hashHostname(url: string): string { const parsed = new URL(url); return createHash("sha256").update(parsed.hostname).digest("hex"); } -async function getOrgClickhouseConfig( - ctx: DefaultStatefulContext, - orgId: string, - clientType: string -): Promise { - const org = await prisma.organization.findUnique({ - where: { id: orgId }, - select: { featureFlags: true }, - }); +type ClientType = "standard" | "events" | "replication" | "logs" | "query" | "admin"; - if (!org?.featureFlags) { - return null; - } - - const clickhouseConfig = (org.featureFlags as any).clickhouse; - if (!clickhouseConfig || typeof clickhouseConfig !== "object") { - return null; - } - - const secretKey = clickhouseConfig[clientType]; - if (!secretKey || typeof secretKey !== "string") { - return null; - } +/** + * Resolve a secret-store key to a connection URL + hostname hash. + * Results are cached for the process lifetime (the registry reloads keep org→key mapping fresh). + */ +async function resolveSecretKey( + secretKey: string +): Promise<{ url: string; hostnameHash: string } | null> { + const cached = resolvedConnectionCache.get(secretKey); + if (cached) return cached; const secretStore = getSecretStore("DATABASE"); const connection = await secretStore.getSecret(ClickhouseConnectionSchema, secretKey); + if (!connection) return null; - if (!connection) { - return null; - } + const resolved = { url: connection.url, hostnameHash: hashHostname(connection.url) }; + resolvedConnectionCache.set(secretKey, resolved); + return resolved; +} - const hostnameHash = hashHostname(connection.url); +function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHouse { + const parsed = new URL(url); + parsed.searchParams.delete("secure"); - return { - organizationId: orgId, - hostnameHash, - url: connection.url, - clientType, - }; + return new ClickHouse({ + url: parsed.toString(), + name: `org-clickhouse-${clientType}`, + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { request: true }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); } +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + export async function getClickhouseForOrganization( organizationId: string, - clientType: "standard" | "events" | "replication" | "logs" | "query" | "admin" + clientType: ClientType ): Promise { - // Try to get org-specific config - const configResult = await orgConfigCache.orgClickhouse.swr( - `org:${organizationId}:ch:${clientType}`, - async () => getOrgClickhouseConfig(ctx, organizationId, clientType) - ); + if (!organizationDataStoresRegistry.isLoaded) { + await organizationDataStoresRegistry.isReady; + } + + const dataStore = organizationDataStoresRegistry.get(organizationId, "CLICKHOUSE"); + + if (!dataStore) { + // No override — use the appropriate default client. + switch (clientType) { + case "standard": + case "events": + case "replication": + return defaultClickhouseClient; + case "logs": + return defaultLogsClickhouseClient; + case "query": + return defaultQueryClickhouseClient; + case "admin": + return defaultAdminClickhouseClient; + } + } - // Handle Result type - check for error or null value - const config = configResult.err ? null : configResult.val; + const { secretKey } = dataStore.config.data; + const connection = await resolveSecretKey(secretKey); - // If no custom config, return appropriate default client - if (!config) { + if (!connection) { + console.warn( + `[clickhouseFactory] Secret key "${secretKey}" not found for org ${organizationId}; falling back to default` + ); switch (clientType) { case "standard": case "events": @@ -279,27 +272,11 @@ export async function getClickhouseForOrganization( } } - // Check if client already exists for this hostname - const cacheKey = `${config.hostnameHash}:${clientType}`; + const cacheKey = `${connection.hostnameHash}:${clientType}`; let client = clickhouseClientCache.get(cacheKey); if (!client) { - const url = new URL(config.url); - url.searchParams.delete("secure"); - - client = new ClickHouse({ - url: url.toString(), - name: `org-clickhouse-${clientType}`, - keepAlive: { - enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", - idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, - }, - logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { - request: true, - }, - maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - }); + client = buildOrgClickhouseClient(connection.url, clientType); clickhouseClientCache.set(cacheKey, client); } @@ -309,79 +286,64 @@ export async function getClickhouseForOrganization( export async function getEventRepositoryForOrganization( organizationId: string ): Promise { - // Try to get org-specific config - const configResult = await orgConfigCache.orgClickhouse.swr( - `org:${organizationId}:ch:events`, - async () => getOrgClickhouseConfig(ctx, organizationId, "events") - ); + if (!organizationDataStoresRegistry.isLoaded) { + await organizationDataStoresRegistry.isReady; + } - // Handle Result type - check for error or null value - const config = configResult.err ? null : configResult.val; + const dataStore = organizationDataStoresRegistry.get(organizationId, "CLICKHOUSE"); - // If no custom config, return default repository (created on demand) - if (!config) { + if (!dataStore) { const defaultKey = "default:events"; let defaultRepo = eventRepositoryCache.get(defaultKey); if (!defaultRepo) { - // Create default event repository using standard clickhouse client - // This matches the existing pattern in clickhouseEventRepositoryInstance.server.ts const eventsClickhouse = await getEventsClickhouseClient(); - defaultRepo = new ClickhouseEventRepository({ - clickhouse: eventsClickhouse, - batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE, - flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS, - maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT, - maximumTraceDetailedSummaryViewCount: - env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT, - maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING, - insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY, - waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1", - asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, - asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, - startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS, - llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, - llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, - llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, - llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, - version: "v2", - }); + defaultRepo = buildEventRepository(eventsClickhouse); eventRepositoryCache.set(defaultKey, defaultRepo); } return defaultRepo; } - // Check if repository already exists for this hostname - const cacheKey = `${config.hostnameHash}:events`; + const { secretKey } = dataStore.config.data; + const connection = await resolveSecretKey(secretKey); + + if (!connection) { + console.warn( + `[clickhouseFactory] Secret key "${secretKey}" not found for org ${organizationId}; falling back to default event repository` + ); + const defaultKey = "default:events"; + let defaultRepo = eventRepositoryCache.get(defaultKey); + if (!defaultRepo) { + const eventsClickhouse = await getEventsClickhouseClient(); + defaultRepo = buildEventRepository(eventsClickhouse); + eventRepositoryCache.set(defaultKey, defaultRepo); + } + return defaultRepo; + } + + const cacheKey = `${connection.hostnameHash}:events`; let repository = eventRepositoryCache.get(cacheKey); if (!repository) { const client = await getClickhouseForOrganization(organizationId, "events"); - repository = new ClickhouseEventRepository({ - clickhouse: client, - batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE, - flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS, - maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT, - maximumTraceDetailedSummaryViewCount: - env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT, - maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING, - insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY, - waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1", - asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, - asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, - startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS, - llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, - llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, - llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, - llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, - version: "v2", - }); + repository = buildEventRepository(client); eventRepositoryCache.set(cacheKey, repository); } return repository; } -// Helper to create the default events ClickHouse client +/** + * Get admin ClickHouse client for cross-organization queries. + * Only use for admin tools and analytics that need to query across all orgs. + */ +export function getAdminClickhouse(): ClickHouse { + return defaultAdminClickhouseClient; +} + +// --------------------------------------------------------------------------- +// Private helpers +// --------------------------------------------------------------------------- + async function getEventsClickhouseClient(): Promise { if (!env.EVENTS_CLICKHOUSE_URL) { throw new Error("EVENTS_CLICKHOUSE_URL is not set"); @@ -398,25 +360,29 @@ async function getEventsClickhouseClient(): Promise { idleSocketTtl: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, }, logLevel: env.EVENTS_CLICKHOUSE_LOG_LEVEL, - compression: { - request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1", - }, + compression: { request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1" }, maxOpenConnections: env.EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS, }); } -/** - * Get admin ClickHouse client for cross-organization queries - * This should only be used for admin tools and analytics that need to query across all orgs - */ -export function getAdminClickhouse(): ClickHouse { - return defaultAdminClickhouseClient; -} - -// Clear caches when needed (e.g., when org config changes) -export function clearClickhouseCacheForOrganization(organizationId: string): void { - // The Unkey cache will naturally expire based on TTL (5min fresh, 10min stale) - // No explicit removal needed - cache entries will be refreshed on next access - // Note: We don't clear client/repository caches as they're keyed by hostname - // and may be shared by other orgs +function buildEventRepository(clickhouse: ClickHouse): ClickhouseEventRepository { + return new ClickhouseEventRepository({ + clickhouse, + batchSize: env.EVENTS_CLICKHOUSE_BATCH_SIZE, + flushInterval: env.EVENTS_CLICKHOUSE_FLUSH_INTERVAL_MS, + maximumTraceSummaryViewCount: env.EVENTS_CLICKHOUSE_MAX_TRACE_SUMMARY_VIEW_COUNT, + maximumTraceDetailedSummaryViewCount: + env.EVENTS_CLICKHOUSE_MAX_TRACE_DETAILED_SUMMARY_VIEW_COUNT, + maximumLiveReloadingSetting: env.EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING, + insertStrategy: env.EVENTS_CLICKHOUSE_INSERT_STRATEGY, + waitForAsyncInsert: env.EVENTS_CLICKHOUSE_WAIT_FOR_ASYNC_INSERT === "1", + asyncInsertMaxDataSize: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_MAX_DATA_SIZE, + asyncInsertBusyTimeoutMs: env.EVENTS_CLICKHOUSE_ASYNC_INSERT_BUSY_TIMEOUT_MS, + startTimeMaxAgeMs: env.EVENTS_CLICKHOUSE_START_TIME_MAX_AGE_MS, + llmMetricsBatchSize: env.LLM_METRICS_BATCH_SIZE, + llmMetricsFlushInterval: env.LLM_METRICS_FLUSH_INTERVAL_MS, + llmMetricsMaxBatchSize: env.LLM_METRICS_MAX_BATCH_SIZE, + llmMetricsMaxConcurrency: env.LLM_METRICS_MAX_CONCURRENCY, + version: "v2", + }); } diff --git a/apps/webapp/app/services/dataStores/organizationDataStoreConfigSchemas.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoreConfigSchemas.server.ts new file mode 100644 index 00000000000..89465ff5b4a --- /dev/null +++ b/apps/webapp/app/services/dataStores/organizationDataStoreConfigSchemas.server.ts @@ -0,0 +1,35 @@ +import { z } from "zod"; + +// --------------------------------------------------------------------------- +// ClickHouse config (kind = CLICKHOUSE) +// --------------------------------------------------------------------------- + +/** V1: single secret-store key that supplies the ClickHouse connection URL. */ +export const ClickhouseDataStoreConfigV1 = z.object({ + version: z.literal(1), + data: z.object({ + /** Key into the SecretStore that resolves to a ClickhouseConnection ({url}). */ + secretKey: z.string(), + }), +}); + +export type ClickhouseDataStoreConfigV1 = z.infer; + +/** Discriminated union over version — extend by adding new literals here. */ +export const ClickhouseDataStoreConfig = z.discriminatedUnion("version", [ + ClickhouseDataStoreConfigV1, +]); + +export type ClickhouseDataStoreConfig = z.infer; + +// --------------------------------------------------------------------------- +// Top-level per-kind union +// --------------------------------------------------------------------------- + +export type ParsedClickhouseDataStore = { + kind: "CLICKHOUSE"; + config: ClickhouseDataStoreConfig; +}; + +/** Union of all parsed data store types. Extend as new DataStoreKind values are added. */ +export type ParsedDataStore = ParsedClickhouseDataStore; diff --git a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts new file mode 100644 index 00000000000..8f398f680ea --- /dev/null +++ b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts @@ -0,0 +1,82 @@ +import type { PrismaClient, PrismaReplicaClient } from "@trigger.dev/database"; +import { + ClickhouseDataStoreConfig, + type ParsedDataStore, +} from "./organizationDataStoreConfigSchemas.server"; + +export class OrganizationDataStoresRegistry { + private _prisma: PrismaClient | PrismaReplicaClient; + /** Keyed by `${organizationId}:${kind}` */ + private _lookup: Map = new Map(); + private _loaded = false; + private _readyResolve!: () => void; + + /** Resolves once the initial `loadFromDatabase()` completes successfully. */ + readonly isReady: Promise; + + constructor(prisma: PrismaClient | PrismaReplicaClient) { + this._prisma = prisma; + this.isReady = new Promise((resolve) => { + this._readyResolve = resolve; + }); + } + + get isLoaded(): boolean { + return this._loaded; + } + + async loadFromDatabase(): Promise { + const rows = await this._prisma.organizationDataStore.findMany(); + + const lookup = new Map(); + + for (const row of rows) { + let parsed: ParsedDataStore | null = null; + + switch (row.kind) { + case "CLICKHOUSE": { + const result = ClickhouseDataStoreConfig.safeParse(row.config); + if (!result.success) { + console.warn( + `[OrganizationDataStoresRegistry] Invalid config for OrganizationDataStore "${row.key}" (kind=CLICKHOUSE): ${result.error.message}` + ); + continue; + } + parsed = { kind: "CLICKHOUSE", config: result.data }; + break; + } + default: { + console.warn( + `[OrganizationDataStoresRegistry] Unknown kind "${row.kind}" for OrganizationDataStore "${row.key}" — skipping` + ); + continue; + } + } + + for (const orgId of row.organizationIds) { + const key = `${orgId}:${row.kind}`; + lookup.set(key, parsed); + } + } + + this._lookup = lookup; + + if (!this._loaded) { + this._loaded = true; + this._readyResolve(); + } + } + + async reload(): Promise { + await this.loadFromDatabase(); + } + + /** + * Returns the parsed data store config for the given organization and kind, + * or `null` if no override is configured (caller should use the default). + */ + get(organizationId: string, kind: "CLICKHOUSE"): ParsedDataStore | null { + if (!this._loaded) return null; + return this._lookup.get(`${organizationId}:${kind}`) ?? null; + } +} diff --git a/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts new file mode 100644 index 00000000000..ac5c25d9aaa --- /dev/null +++ b/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts @@ -0,0 +1,46 @@ +import { $replica } from "~/db.server"; +import { env } from "~/env.server"; +import { signalsEmitter } from "~/services/signals.server"; +import { singleton } from "~/utils/singleton"; +import { OrganizationDataStoresRegistry } from "./organizationDataStoresRegistry.server"; + +export const organizationDataStoresRegistry = singleton( + "organizationDataStoresRegistry", + () => { + const registry = new OrganizationDataStoresRegistry($replica); + + registry.loadFromDatabase().catch((err) => { + console.error("[OrganizationDataStoresRegistry] Failed to initialize", err); + }); + + const interval = setInterval( + () => { + registry.reload().catch((err) => { + console.error("[OrganizationDataStoresRegistry] Failed to reload", err); + }); + }, + env.ORGANIZATION_DATA_STORES_RELOAD_INTERVAL_MS + ); + + signalsEmitter.on("SIGTERM", () => clearInterval(interval)); + signalsEmitter.on("SIGINT", () => clearInterval(interval)); + + return registry; + } +); + +/** + * Wait for the registry to finish its initial load, with a timeout. + * After the first call resolves (or times out), subsequent calls are no-ops. + */ +export async function waitForOrganizationDataStoresReady( + timeoutMs = 5000 +): Promise { + if (organizationDataStoresRegistry.isLoaded) return; + if (timeoutMs <= 0) return; + + await Promise.race([ + organizationDataStoresRegistry.isReady, + new Promise((resolve) => setTimeout(resolve, timeoutMs)), + ]); +} diff --git a/internal-packages/database/prisma/migrations/20260331212308_add_organization_data_stores/migration.sql b/internal-packages/database/prisma/migrations/20260331212308_add_organization_data_stores/migration.sql new file mode 100644 index 00000000000..52b8385539a --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260331212308_add_organization_data_stores/migration.sql @@ -0,0 +1,21 @@ +-- CreateEnum +CREATE TYPE "public"."DataStoreKind" AS ENUM ('CLICKHOUSE'); + +-- CreateTable +CREATE TABLE "public"."OrganizationDataStore" ( + "id" TEXT NOT NULL, + "key" TEXT NOT NULL, + "organizationIds" TEXT[], + "kind" "public"."DataStoreKind" NOT NULL, + "config" JSONB NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "OrganizationDataStore_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "OrganizationDataStore_key_key" ON "public"."OrganizationDataStore"("key"); + +-- CreateIndex +CREATE INDEX "OrganizationDataStore_kind_idx" ON "public"."OrganizationDataStore"("kind"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index de545599737..d8e9fcb0b22 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2947,4 +2947,25 @@ model TaskIdentifier { @@unique([runtimeEnvironmentId, slug]) @@index([runtimeEnvironmentId, isInLatestDeployment]) -} \ No newline at end of file +} + +enum DataStoreKind { + CLICKHOUSE +} + +/// Defines org-scoped data store overrides (e.g. dedicated ClickHouse for HIPAA orgs). +/// Multiple organizations can share a single data store row via organizationIds. +model OrganizationDataStore { + id String @id @default(cuid()) + /// Human-readable unique key (e.g. "hipaa-clickhouse-us-east") + key String @unique + /// Organization IDs that use this data store + organizationIds String[] + kind DataStoreKind + /// Versioned config JSON. Structure is discriminated by the top-level `version` field. + config Json + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@index([kind]) +} From c3ab67f72ad4ee9cb62835eb5500762a483fec9e Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Wed, 1 Apr 2026 12:10:15 +0100 Subject: [PATCH 06/32] Admin page for adding data stores --- apps/webapp/app/routes/admin.data-stores.tsx | 368 +++++++++++++++++++ apps/webapp/app/routes/admin.tsx | 4 + 2 files changed, 372 insertions(+) create mode 100644 apps/webapp/app/routes/admin.data-stores.tsx diff --git a/apps/webapp/app/routes/admin.data-stores.tsx b/apps/webapp/app/routes/admin.data-stores.tsx new file mode 100644 index 00000000000..4397c6d33d1 --- /dev/null +++ b/apps/webapp/app/routes/admin.data-stores.tsx @@ -0,0 +1,368 @@ +import { useState } from "react"; +import { useFetcher } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { redirect } from "@remix-run/server-runtime"; +import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { z } from "zod"; +import { Button } from "~/components/primitives/Buttons"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "~/components/primitives/Dialog"; +import { Input } from "~/components/primitives/Input"; +import { Paragraph } from "~/components/primitives/Paragraph"; +import { Popover, PopoverContent, PopoverTrigger } from "~/components/primitives/Popover"; +import { + Table, + TableBlankRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, +} from "~/components/primitives/Table"; +import { prisma } from "~/db.server"; +import { requireUser } from "~/services/session.server"; +import { getSecretStore } from "~/services/secrets/secretStore.server"; +import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server"; + +// --------------------------------------------------------------------------- +// Loader +// --------------------------------------------------------------------------- + +export const loader = async ({ request }: LoaderFunctionArgs) => { + const user = await requireUser(request); + if (!user.admin) throw redirect("/"); + + const dataStores = await prisma.organizationDataStore.findMany({ + orderBy: { createdAt: "desc" }, + }); + + return typedjson({ dataStores }); +}; + +// --------------------------------------------------------------------------- +// Action +// --------------------------------------------------------------------------- + +const AddSchema = z.object({ + _action: z.literal("add"), + key: z.string().min(1), + organizationIds: z.string().min(1), + connectionUrl: z.string().url(), +}); + +const DeleteSchema = z.object({ + _action: z.literal("delete"), + id: z.string().min(1), +}); + +export async function action({ request }: ActionFunctionArgs) { + const user = await requireUser(request); + if (!user.admin) throw redirect("/"); + + const formData = await request.formData(); + const _action = formData.get("_action"); + + if (_action === "add") { + const result = AddSchema.safeParse(Object.fromEntries(formData)); + if (!result.success) { + return typedjson( + { error: result.error.issues.map((i) => i.message).join(", ") }, + { status: 400 } + ); + } + + const { key, organizationIds: rawOrgIds, connectionUrl } = result.data; + const organizationIds = rawOrgIds + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + + const secretKey = `data-store:${key}:clickhouse`; + + const secretStore = getSecretStore("DATABASE"); + await secretStore.setSecret(secretKey, ClickhouseConnectionSchema.parse({ url: connectionUrl })); + + await prisma.organizationDataStore.create({ + data: { + key, + organizationIds, + kind: "CLICKHOUSE", + config: { version: 1, data: { secretKey } }, + }, + }); + + + return typedjson({ success: true }); + } + + if (_action === "delete") { + const result = DeleteSchema.safeParse(Object.fromEntries(formData)); + if (!result.success) { + return typedjson({ error: "Invalid request" }, { status: 400 }); + } + + const { id } = result.data; + + const dataStore = await prisma.organizationDataStore.findFirst({ where: { id } }); + if (!dataStore) { + return typedjson({ error: "Data store not found" }, { status: 404 }); + } + + // Delete secret if config references one + const config = dataStore.config as any; + if (config?.data?.secretKey) { + const secretStore = getSecretStore("DATABASE"); + await secretStore.deleteSecret(config.data.secretKey).catch(() => { + // Secret may not exist — proceed with deletion + }); + } + + await prisma.organizationDataStore.delete({ where: { id } }); + + return typedjson({ success: true }); + } + + return typedjson({ error: "Unknown action" }, { status: 400 }); +} + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +export default function AdminDataStoresRoute() { + const { dataStores } = useTypedLoaderData(); + const [addOpen, setAddOpen] = useState(false); + + return ( +
+
+
+ + {dataStores.length} data store{dataStores.length !== 1 ? "s" : ""} + + +
+ + + + + Key + Kind + Organizations + Created + Updated + + Actions + + + + + {dataStores.length === 0 ? ( + + No data stores configured + + ) : ( + dataStores.map((ds) => ( + + + {ds.key} + + + + {ds.kind} + + + + + {ds.organizationIds.length} org{ds.organizationIds.length !== 1 ? "s" : ""} + + {ds.organizationIds.length > 0 && ( + + ({ds.organizationIds.slice(0, 2).join(", ")} + {ds.organizationIds.length > 2 + ? ` +${ds.organizationIds.length - 2} more` + : ""} + ) + + )} + + + + {new Date(ds.createdAt).toLocaleString()} + + + + + {new Date(ds.updatedAt).toLocaleString()} + + + + + + + )) + )} + +
+
+ + +
+ ); +} + +// --------------------------------------------------------------------------- +// Delete button with popover confirmation +// --------------------------------------------------------------------------- + +function DeleteButton({ id, name }: { id: string; name: string }) { + const [open, setOpen] = useState(false); + const fetcher = useFetcher<{ success?: boolean; error?: string }>(); + const isDeleting = fetcher.state !== "idle"; + + return ( + + + + + + + Delete {name}? + + + This will remove the data store and its secret. Organizations using it will fall back to + the default ClickHouse instance. + +
+ + setOpen(false)}> + + + + +
+ + + ); +} + +// --------------------------------------------------------------------------- +// Add data store dialog +// --------------------------------------------------------------------------- + +function AddDataStoreDialog({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const fetcher = useFetcher<{ success?: boolean; error?: string }>(); + const isSubmitting = fetcher.state !== "idle"; + + // Close dialog on success + if (fetcher.data?.success && open) { + onOpenChange(false); + } + + return ( + + + + Add data store + + + + + +
+ + +

+ Unique identifier for this data store. Used as the secret key prefix. +

+
+ +
+ + +
+ +
+ + +

Comma-separated organization IDs.

+
+ +
+ + +

+ Stored encrypted in SecretStore. Never logged or displayed again. +

+
+ + {fetcher.data?.error && ( +

{fetcher.data.error}

+ )} + + + + + + + + + ); +} diff --git a/apps/webapp/app/routes/admin.tsx b/apps/webapp/app/routes/admin.tsx index 4cd2deca533..3b0858f899a 100644 --- a/apps/webapp/app/routes/admin.tsx +++ b/apps/webapp/app/routes/admin.tsx @@ -44,6 +44,10 @@ export default function Page() { label: "Notifications", to: "/admin/notifications", }, + { + label: "Data Stores", + to: "/admin/data-stores", + }, ]} layoutId={"admin"} /> From c7d425ecf05e421c5e178b21f7672acf6c75f859 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 2 Apr 2026 11:42:48 +0100 Subject: [PATCH 07/32] New admin page, lots of improvements to make it more robust and testable --- apps/webapp/app/routes/admin.data-stores.tsx | 150 +++++++----- .../clickhouse/clickhouseFactory.server.ts | 214 ++++++++---------- .../clickhouse/clickhouseFactory.test.ts | 155 ------------- ...ganizationDataStoreConfigSchemas.server.ts | 6 +- .../organizationDataStoresRegistry.server.ts | 91 +++++++- ...zationDataStoresRegistryInstance.server.ts | 50 ++-- apps/webapp/test/clickhouseFactory.test.ts | 130 +++++++++++ .../organizationDataStoresRegistry.test.ts | 197 ++++++++++++++++ 8 files changed, 614 insertions(+), 379 deletions(-) delete mode 100644 apps/webapp/app/services/clickhouse/clickhouseFactory.test.ts create mode 100644 apps/webapp/test/clickhouseFactory.test.ts create mode 100644 apps/webapp/test/organizationDataStoresRegistry.test.ts diff --git a/apps/webapp/app/routes/admin.data-stores.tsx b/apps/webapp/app/routes/admin.data-stores.tsx index 4397c6d33d1..033b143151f 100644 --- a/apps/webapp/app/routes/admin.data-stores.tsx +++ b/apps/webapp/app/routes/admin.data-stores.tsx @@ -26,8 +26,9 @@ import { } from "~/components/primitives/Table"; import { prisma } from "~/db.server"; import { requireUser } from "~/services/session.server"; -import { getSecretStore } from "~/services/secrets/secretStore.server"; import { ClickhouseConnectionSchema } from "~/services/clickhouse/clickhouseSecretSchemas.server"; +import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; +import { tryCatch } from "@trigger.dev/core"; // --------------------------------------------------------------------------- // Loader @@ -55,79 +56,106 @@ const AddSchema = z.object({ connectionUrl: z.string().url(), }); +const UpdateSchema = z.object({ + _action: z.literal("update"), + key: z.string().min(1), + organizationIds: z.string().min(1), + connectionUrl: z.string().url().optional(), +}); + const DeleteSchema = z.object({ _action: z.literal("delete"), - id: z.string().min(1), + key: z.string().min(1), }); +const FormSchema = z.discriminatedUnion("_action", [AddSchema, UpdateSchema, DeleteSchema]); + export async function action({ request }: ActionFunctionArgs) { const user = await requireUser(request); if (!user.admin) throw redirect("/"); const formData = await request.formData(); - const _action = formData.get("_action"); - - if (_action === "add") { - const result = AddSchema.safeParse(Object.fromEntries(formData)); - if (!result.success) { - return typedjson( - { error: result.error.issues.map((i) => i.message).join(", ") }, - { status: 400 } - ); - } - const { key, organizationIds: rawOrgIds, connectionUrl } = result.data; - const organizationIds = rawOrgIds - .split(",") - .map((s) => s.trim()) - .filter(Boolean); + const result = FormSchema.safeParse(Object.fromEntries(formData)); - const secretKey = `data-store:${key}:clickhouse`; + if (!result.success) { + return typedjson( + { error: result.error.issues.map((i) => i.message).join(", ") }, + { status: 400 } + ); + } - const secretStore = getSecretStore("DATABASE"); - await secretStore.setSecret(secretKey, ClickhouseConnectionSchema.parse({ url: connectionUrl })); + switch (result.data._action) { + case "add": { + const { key, organizationIds: rawOrgIds, connectionUrl } = result.data; + const organizationIds = rawOrgIds + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + + const config = ClickhouseConnectionSchema.parse({ url: connectionUrl }); + + const [error, _] = await tryCatch( + organizationDataStoresRegistry.addDataStore({ + key, + kind: "CLICKHOUSE", + organizationIds, + config, + }) + ); - await prisma.organizationDataStore.create({ - data: { - key, - organizationIds, - kind: "CLICKHOUSE", - config: { version: 1, data: { secretKey } }, - }, - }); + if (error) { + return typedjson({ error: error.message }, { status: 400 }); + } + return typedjson({ success: true }); + } + case "update": { + const { key, organizationIds: rawOrgIds, connectionUrl } = result.data; + const organizationIds = rawOrgIds + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + + const config = connectionUrl + ? ClickhouseConnectionSchema.parse({ url: connectionUrl }) + : undefined; + + const [error, _] = await tryCatch( + organizationDataStoresRegistry.updateDataStore({ + key, + kind: "CLICKHOUSE", + organizationIds, + config, + }) + ); - return typedjson({ success: true }); - } + if (error) { + return typedjson({ error: error.message }, { status: 400 }); + } - if (_action === "delete") { - const result = DeleteSchema.safeParse(Object.fromEntries(formData)); - if (!result.success) { - return typedjson({ error: "Invalid request" }, { status: 400 }); + return typedjson({ success: true }); } + case "delete": { + const { key } = result.data; + + const [error, _] = await tryCatch( + organizationDataStoresRegistry.deleteDataStore({ + key, + kind: "CLICKHOUSE", + }) + ); - const { id } = result.data; + if (error) { + return typedjson({ error: error.message }, { status: 400 }); + } - const dataStore = await prisma.organizationDataStore.findFirst({ where: { id } }); - if (!dataStore) { - return typedjson({ error: "Data store not found" }, { status: 404 }); + return typedjson({ success: true }); } - - // Delete secret if config references one - const config = dataStore.config as any; - if (config?.data?.secretKey) { - const secretStore = getSecretStore("DATABASE"); - await secretStore.deleteSecret(config.data.secretKey).catch(() => { - // Secret may not exist — proceed with deletion - }); + default: { + return typedjson({ error: "Unknown action" }, { status: 400 }); } - - await prisma.organizationDataStore.delete({ where: { id } }); - - return typedjson({ success: true }); } - - return typedjson({ error: "Unknown action" }, { status: 400 }); } // --------------------------------------------------------------------------- @@ -207,7 +235,7 @@ export default function AdminDataStoresRoute() { - + )) @@ -225,7 +253,7 @@ export default function AdminDataStoresRoute() { // Delete button with popover confirmation // --------------------------------------------------------------------------- -function DeleteButton({ id, name }: { id: string; name: string }) { +function DeleteButton({ name }: { name: string }) { const [open, setOpen] = useState(false); const fetcher = useFetcher<{ success?: boolean; error?: string }>(); const isDeleting = fetcher.state !== "idle"; @@ -251,7 +279,7 @@ function DeleteButton({ id, name }: { id: string; name: string }) { setOpen(false)}> - + @@ -311,7 +339,13 @@ function AddDataStoreDialog({ - +
@@ -344,9 +378,7 @@ function AddDataStoreDialog({

- {fetcher.data?.error && ( -

{fetcher.data.error}

- )} + {fetcher.data?.error &&

{fetcher.data.error}

} + + + + Edit data store + + + + + + +
+ + +
+ +
+ + +

Comma-separated organization IDs.

+
+ + {fetcher.data?.error &&

{fetcher.data.error}

} + + + + + + + + + + ); +} + // --------------------------------------------------------------------------- // Add data store dialog // --------------------------------------------------------------------------- From 247714cd910b5e260fff68cfa6b33bf5d52d668b Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 14 Apr 2026 13:52:27 +0100 Subject: [PATCH 19/32] Reload the data store every minute --- apps/webapp/app/env.server.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 306acb5077b..988f5258e9e 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -359,7 +359,10 @@ const EnvironmentSchema = z // If specified, you must configure the corresponding provider using OBJECT_STORE_{PROTOCOL}_* env vars. // Example: OBJECT_STORE_DEFAULT_PROTOCOL=s3 requires OBJECT_STORE_S3_BASE_URL, OBJECT_STORE_S3_ACCESS_KEY_ID, etc. // Enables zero-downtime migration between providers (old data keeps working, new data uses new provider). - OBJECT_STORE_DEFAULT_PROTOCOL: z.string().regex(/^[a-z0-9]+$/).optional(), + OBJECT_STORE_DEFAULT_PROTOCOL: z + .string() + .regex(/^[a-z0-9]+$/) + .optional(), ARTIFACTS_OBJECT_STORE_BUCKET: z.string().optional(), ARTIFACTS_OBJECT_STORE_BASE_URL: z.string().optional(), @@ -1299,11 +1302,17 @@ const EnvironmentSchema = z EVENTS_CLICKHOUSE_MAX_LIVE_RELOADING_SETTING: z.coerce.number().int().default(2000), // Organization data stores registry - ORGANIZATION_DATA_STORES_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + ORGANIZATION_DATA_STORES_RELOAD_INTERVAL_MS: z.coerce + .number() + .int() + .default(60 * 1000), // 1 minute // LLM cost tracking LLM_COST_TRACKING_ENABLED: BoolEnv.default(true), - LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce.number().int().default(5 * 60 * 1000), // 5 minutes + LLM_PRICING_RELOAD_INTERVAL_MS: z.coerce + .number() + .int() + .default(5 * 60 * 1000), // 5 minutes LLM_PRICING_SEED_ON_STARTUP: BoolEnv.default(false), LLM_PRICING_READY_TIMEOUT_MS: z.coerce.number().int().default(500), LLM_METRICS_BATCH_SIZE: z.coerce.number().int().default(5000), From 1217be2bcc4177347f703617d0227c45dc8eceb2 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 16 Apr 2026 13:29:34 +0100 Subject: [PATCH 20/32] Work with Postgres EventRepository for self-hosters --- .../app/v3/eventRepository/index.server.ts | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index 98d6858ed9f..04d6678f92b 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -26,7 +26,8 @@ export function resolveEventRepositoryForStore( organizationId: string ): IEventRepository { if (store === EVENT_STORE_TYPES.CLICKHOUSE || store === EVENT_STORE_TYPES.CLICKHOUSE_V2) { - return clickhouseFactory.getEventRepositoryForOrganizationSync(store, organizationId).repository; + return clickhouseFactory.getEventRepositoryForOrganizationSync(store, organizationId) + .repository; } return eventRepository; } @@ -56,10 +57,8 @@ export async function getConfiguredEventRepository( (organization.featureFlags as Record | null) ?? undefined ); - const { repository: resolvedRepository } = await clickhouseFactory.getEventRepositoryForOrganization( - taskEventStore, - organizationId - ); + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId); if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE_V2) { return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 }; @@ -78,20 +77,26 @@ export async function getEventRepository( parentStore: string | undefined ): Promise<{ repository: IEventRepository; store: string }> { const taskEventStore = parentStore ?? (await resolveTaskEventRepositoryFlag(featureFlags)); - const { repository: resolvedRepository } = await clickhouseFactory.getEventRepositoryForOrganization( - taskEventStore, - organizationId - ); - if (taskEventStore === "clickhouse_v2") { - return { repository: resolvedRepository, store: "clickhouse_v2" }; + // Support legacy Postgres store for self-hosters + if (taskEventStore === EVENT_STORE_TYPES.POSTGRES) { + return { repository: eventRepository, store: getTaskEventStore() }; } - if (taskEventStore === "clickhouse") { - return { repository: resolvedRepository, store: "clickhouse" }; - } + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId); - return { repository: eventRepository, store: getTaskEventStore() }; + switch (taskEventStore) { + case EVENT_STORE_TYPES.CLICKHOUSE_V2: { + return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 }; + } + case EVENT_STORE_TYPES.CLICKHOUSE: { + return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE }; + } + default: { + return { repository: eventRepository, store: getTaskEventStore() }; + } + } } export async function getV3EventRepository( @@ -99,24 +104,18 @@ export async function getV3EventRepository( parentStore: string | undefined ): Promise<{ repository: IEventRepository; store: string }> { if (typeof parentStore === "string") { - const { repository: resolvedRepository } = await clickhouseFactory.getEventRepositoryForOrganization( - parentStore, - organizationId - ); + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization(parentStore, organizationId); return { repository: resolvedRepository, store: parentStore }; } if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse_v2") { - const { repository: resolvedRepository } = await clickhouseFactory.getEventRepositoryForOrganization( - "clickhouse_v2", - organizationId - ); + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization("clickhouse_v2", organizationId); return { repository: resolvedRepository, store: "clickhouse_v2" }; } else if (env.EVENT_REPOSITORY_DEFAULT_STORE === "clickhouse") { - const { repository: resolvedRepository } = await clickhouseFactory.getEventRepositoryForOrganization( - "clickhouse", - organizationId - ); + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization("clickhouse", organizationId); return { repository: resolvedRepository, store: "clickhouse" }; } else { return { repository: eventRepository, store: getTaskEventStore() }; @@ -206,10 +205,11 @@ async function recordRunEvent( }; } - const { repository: $eventRepository } = await clickhouseFactory.getEventRepositoryForOrganization( - foundRun.taskEventStore, - foundRun.runtimeEnvironment.organizationId - ); + const { repository: $eventRepository } = + await clickhouseFactory.getEventRepositoryForOrganization( + foundRun.taskEventStore, + foundRun.runtimeEnvironment.organizationId + ); const { attributes, startTime, ...optionsRest } = options; From ce15a8abb6bc974dfa820afcbe492e8e76ab30a0 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 16 Apr 2026 13:31:29 +0100 Subject: [PATCH 21/32] Error fingerprint should use the logs client --- .../route.tsx | 92 +++++++++---------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index 491de3766cd..b65eb3e819d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -1,55 +1,48 @@ -import { type LoaderFunctionArgs, type ActionFunctionArgs, json } from "@remix-run/server-runtime"; -import { type MetaFunction, useFetcher, useRevalidator } from "@remix-run/react"; +import { parse } from "@conform-to/zod"; import { BellAlertIcon } from "@heroicons/react/20/solid"; +import { type MetaFunction, useFetcher, useRevalidator } from "@remix-run/react"; +import { type ActionFunctionArgs, json, type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { IconAlarmSnooze as IconAlarmSnoozeBase, IconCircleDotted } from "@tabler/icons-react"; -import { parse } from "@conform-to/zod"; +import { AnimatePresence, motion } from "framer-motion"; +import { Suspense, useEffect, useMemo, useRef, useState } from "react"; +import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { ErrorStatusBadge } from "~/components/errors/ErrorStatusBadge"; -import { ServiceValidationError } from "~/v3/services/baseService.server"; -import { TypedAwait, typeddefer, useTypedLoaderData } from "remix-typedjson"; -import { requireUser, requireUserId } from "~/services/session.server"; +import { PageBody } from "~/components/layout/AppLayout"; +import { Callout } from "~/components/primitives/Callout"; +import { Header2, Header3 } from "~/components/primitives/Headers"; +import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; +import { Paragraph } from "~/components/primitives/Paragraph"; import { - EnvironmentParamSchema, - v3CreateBulkActionPath, - v3ErrorsPath, - v3RunsPath, -} from "~/utils/pathBuilder"; + ResizableHandle, + ResizablePanel, + ResizablePanelGroup, +} from "~/components/primitives/Resizable"; +import { Spinner } from "~/components/primitives/Spinner"; +import { $replica } from "~/db.server"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { - ErrorGroupPresenter, type ErrorGroupActivity, type ErrorGroupActivityVersions, type ErrorGroupOccurrences, - type ErrorGroupSummary, + ErrorGroupPresenter, type ErrorGroupState, + type ErrorGroupSummary, } from "~/presenters/v3/ErrorGroupPresenter.server"; import { type NextRunList } from "~/presenters/v3/NextRunListPresenter.server"; -import { $replica } from "~/db.server"; -import { - clickhouseFactory, - getDefaultClickhouseClient, - getDefaultLogsClickhouseClient, -} from "~/services/clickhouse/clickhouseFactory.server"; -import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; -import { PageBody } from "~/components/layout/AppLayout"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { requireUser, requireUserId } from "~/services/session.server"; import { - ResizableHandle, - ResizablePanel, - ResizablePanelGroup, -} from "~/components/primitives/Resizable"; -import { AnimatePresence, motion } from "framer-motion"; -import { Suspense, useEffect, useMemo, useRef, useState } from "react"; -import { Spinner } from "~/components/primitives/Spinner"; -import { Paragraph } from "~/components/primitives/Paragraph"; -import { Callout } from "~/components/primitives/Callout"; -import { Header2, Header3 } from "~/components/primitives/Headers"; + EnvironmentParamSchema, + v3CreateBulkActionPath, + v3ErrorsPath, + v3RunsPath, +} from "~/utils/pathBuilder"; +import { ServiceValidationError } from "~/v3/services/baseService.server"; -import { formatDistanceToNow, isPast } from "date-fns"; +import { isPast } from "date-fns"; -import * as Property from "~/components/primitives/PropertyTable"; -import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; -import { DateTime, RelativeDateTime } from "~/components/primitives/DateTime"; import { ErrorId } from "@trigger.dev/core/v3/isomorphic"; import { Bar, @@ -61,31 +54,34 @@ import { XAxis, YAxis, } from "recharts"; +import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; +import { RunsIcon } from "~/assets/icons/RunsIcon"; +import { CodeBlock } from "~/components/code/CodeBlock"; +import { DirectionSchema, ListPagination } from "~/components/ListPagination"; +import { LogsVersionFilter } from "~/components/logs/LogsVersionFilter"; +import { LinkButton } from "~/components/primitives/Buttons"; +import { CopyableText } from "~/components/primitives/CopyableText"; +import { DateTime, RelativeDateTime } from "~/components/primitives/DateTime"; +import * as Property from "~/components/primitives/PropertyTable"; import TooltipPortal from "~/components/primitives/TooltipPortal"; +import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; import { TimeFilter, timeFilterFromTo } from "~/components/runs/v3/SharedFilters"; +import { TaskRunsTable } from "~/components/runs/v3/TaskRunsTable"; +import { useEnvironment } from "~/hooks/useEnvironment"; import { useOptimisticLocation } from "~/hooks/useOptimisticLocation"; -import { DirectionSchema, ListPagination } from "~/components/ListPagination"; -import { Button, LinkButton } from "~/components/primitives/Buttons"; -import { ListCheckedIcon } from "~/assets/icons/ListCheckedIcon"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; -import { useEnvironment } from "~/hooks/useEnvironment"; -import { RunsIcon } from "~/assets/icons/RunsIcon"; -import type { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters"; import { useSearchParams } from "~/hooks/useSearchParam"; -import { CopyableText } from "~/components/primitives/CopyableText"; import { cn } from "~/utils/cn"; -import { LogsVersionFilter } from "~/components/logs/LogsVersionFilter"; -import { CodeBlock } from "~/components/code/CodeBlock"; -import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover"; -import { ErrorGroupActions } from "~/v3/services/errorGroupActions.server"; import { - ErrorStatusMenuItems, CustomIgnoreDialog, + ErrorStatusMenuItems, statusActionToastMessage, } from "~/components/errors/ErrorStatusMenu"; +import { Popover, PopoverArrowTrigger, PopoverContent } from "~/components/primitives/Popover"; import { useToast } from "~/components/primitives/Toast"; +import { ErrorGroupActions } from "~/v3/services/errorGroupActions.server"; export const meta: MetaFunction = ({ data }) => { return [ @@ -247,7 +243,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const clickhouseClient = await clickhouseFactory.getClickhouseForOrganization( environment.organizationId, - "query" + "logs" ); const presenter = new ErrorGroupPresenter($replica, clickhouseClient, clickhouseClient); From 023154b62743f577e04964c1e287b12e7f8fa170 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Thu, 16 Apr 2026 14:02:43 +0100 Subject: [PATCH 22/32] Retry the initial boot using p-retry --- .../organizationDataStoresRegistry.server.ts | 7 +++++- ...zationDataStoresRegistryInstance.server.ts | 22 +++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts index 379084a0d74..f2ef02cc3bd 100644 --- a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts +++ b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts @@ -13,7 +13,12 @@ export class OrganizationDataStoresRegistry { private _loaded = false; private _readyResolve!: () => void; - /** Resolves once the initial `loadFromDatabase()` completes successfully. */ + /** + * Resolves once the initial `loadFromDatabase()` completes successfully. + * At process startup the singleton loads the registry with unbounded retries + * (exponential backoff, capped delay) until Postgres is reachable; until then + * this promise stays pending and callers that await readiness will block. + */ readonly isReady: Promise; constructor(prisma: PrismaClient | PrismaReplicaClient) { diff --git a/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts index f7d82da5682..24ec572c5d5 100644 --- a/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts +++ b/apps/webapp/app/services/dataStores/organizationDataStoresRegistryInstance.server.ts @@ -1,5 +1,7 @@ +import pRetry from "p-retry"; import { $replica } from "~/db.server"; import { env } from "~/env.server"; +import { logger } from "~/services/logger.server"; import { signalsEmitter } from "~/services/signals.server"; import { singleton } from "~/utils/singleton"; import { OrganizationDataStoresRegistry } from "./organizationDataStoresRegistry.server"; @@ -7,8 +9,24 @@ import { OrganizationDataStoresRegistry } from "./organizationDataStoresRegistry export const organizationDataStoresRegistry = singleton("organizationDataStoresRegistry", () => { const registry = new OrganizationDataStoresRegistry($replica); - registry.loadFromDatabase().catch((err) => { - console.error("[OrganizationDataStoresRegistry] Failed to initialize", err); + // Runs as soon as this singleton is created (first import of this module). The + // registry’s `isReady` promise resolves when this eventually succeeds. + const startupLoadPromise = pRetry(() => registry.loadFromDatabase(), { + forever: true, + retries: 10, + minTimeout: 1_000, + maxTimeout: 60_000, + factor: 2, + onFailedAttempt: (error) => { + logger.warn("[OrganizationDataStoresRegistry] Startup load failed, retrying", { + attemptNumber: error.attemptNumber, + retriesLeft: error.retriesLeft, + error: error.message, + }); + }, + }); + startupLoadPromise.catch((err) => { + console.error("[OrganizationDataStoresRegistry] Unexpected startup load failure", err); }); const interval = setInterval(() => { From 5cd8588b25a8ac3d0e5130eb81615bdc307db958 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 16:43:07 +0000 Subject: [PATCH 23/32] Fix Devin Review bugs: postgres event repository fallback and error presenter clients Co-Authored-By: Matt Aitken --- .../route.tsx | 10 +++++----- .../route.tsx | 6 +++--- .../app/v3/eventRepository/index.server.ts | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index b65eb3e819d..a63c3fe814c 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -241,12 +241,12 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const directionRaw = url.searchParams.get("direction") ?? undefined; const direction = directionRaw ? DirectionSchema.parse(directionRaw) : undefined; - const clickhouseClient = await clickhouseFactory.getClickhouseForOrganization( - environment.organizationId, - "logs" - ); + const [logsClickhouseClient, clickhouseClient] = await Promise.all([ + clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "logs"), + clickhouseFactory.getClickhouseForOrganization(environment.organizationId, "standard"), + ]); - const presenter = new ErrorGroupPresenter($replica, clickhouseClient, clickhouseClient); + const presenter = new ErrorGroupPresenter($replica, logsClickhouseClient, clickhouseClient); const detailPromise = presenter .call(project.organizationId, environment.id, { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx index d7ed8c35798..1c9929b2958 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx @@ -123,11 +123,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const plan = await getCurrentPlan(project.organizationId); const retentionLimitDays = plan?.v3Subscription?.plan?.limits.logRetentionDays.number ?? 30; - const queryClickhouse = await clickhouseFactory.getClickhouseForOrganization( + const logsClickhouseClient = await clickhouseFactory.getClickhouseForOrganization( project.organizationId, - "query" + "logs" ); - const presenter = new ErrorsListPresenter($replica, queryClickhouse); + const presenter = new ErrorsListPresenter($replica, logsClickhouseClient); const listPromise = presenter .call(project.organizationId, environment.id, { diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index 04d6678f92b..69c6e80ef25 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -57,14 +57,15 @@ export async function getConfiguredEventRepository( (organization.featureFlags as Record | null) ?? undefined ); - const { repository: resolvedRepository } = - await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId); - if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE_V2) { + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId); return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE_V2 }; } if (taskEventStore === EVENT_STORE_TYPES.CLICKHOUSE) { + const { repository: resolvedRepository } = + await clickhouseFactory.getEventRepositoryForOrganization(taskEventStore, organizationId); return { repository: resolvedRepository, store: EVENT_STORE_TYPES.CLICKHOUSE }; } @@ -104,6 +105,15 @@ export async function getV3EventRepository( parentStore: string | undefined ): Promise<{ repository: IEventRepository; store: string }> { if (typeof parentStore === "string") { + // Support legacy Postgres store for self-hosters and runs persisted with a + // non-ClickHouse store — fall back to the Prisma-based event repository. + if ( + parentStore !== EVENT_STORE_TYPES.CLICKHOUSE && + parentStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2 + ) { + return { repository: eventRepository, store: parentStore }; + } + const { repository: resolvedRepository } = await clickhouseFactory.getEventRepositoryForOrganization(parentStore, organizationId); return { repository: resolvedRepository, store: parentStore }; From b1694a37a4aad53b20729ff40385d741eb990f75 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 10:46:37 +0000 Subject: [PATCH 24/32] Fall back to Postgres eventRepository for non-ClickHouse stores Co-Authored-By: Matt Aitken --- .../app/services/clickhouse/clickhouseFactory.server.ts | 9 +++++++++ apps/webapp/app/v3/eventRepository/index.server.ts | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index 92df7de2b54..55c0feef7f5 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -6,6 +6,7 @@ import { singleton } from "~/utils/singleton"; import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server"; import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types"; +import { eventRepository as postgresEventRepository } from "~/v3/eventRepository/eventRepository.server"; // --------------------------------------------------------------------------- // Default clients (singleton per process) @@ -254,6 +255,14 @@ export class ClickhouseFactory { store: string, organizationId: string ): { key: string; repository: IEventRepository } { + // Non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed + // runs, or "postgres") fall back to the Prisma event repository. This lets + // callers pass `run.taskEventStore` directly without needing to guard + // against legacy/Postgres values. + if (store !== "clickhouse" && store !== "clickhouse_v2") { + return { key: `postgres:${store}`, repository: postgresEventRepository }; + } + const dataStore = this._registry.get(organizationId, "CLICKHOUSE"); if (!dataStore) { diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index 69c6e80ef25..f3449564fce 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -79,8 +79,12 @@ export async function getEventRepository( ): Promise<{ repository: IEventRepository; store: string }> { const taskEventStore = parentStore ?? (await resolveTaskEventRepositoryFlag(featureFlags)); - // Support legacy Postgres store for self-hosters - if (taskEventStore === EVENT_STORE_TYPES.POSTGRES) { + // Non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed + // runs, or the legacy "postgres" value) resolve to the Prisma event repo. + if ( + taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE && + taskEventStore !== EVENT_STORE_TYPES.CLICKHOUSE_V2 + ) { return { repository: eventRepository, store: getTaskEventStore() }; } From e9db1e901e5b3006c24dd1ff9c886eadaa75ce4a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 10:58:46 +0000 Subject: [PATCH 25/32] Move Postgres event repository fallback out of ClickHouse factory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The factory should not import eventRepository.server.ts — doing so pulls the tracePubSub singleton into any module graph that imports the factory, which eagerly connects to Redis at module load time (see singleton.ts). The fallback now lives in index.server.ts via getEventRepositoryForStore, which is called from RunPresenter, SpanPresenter, and recordRunEvent. This restores runsReplicationBenchmark.test.ts's module isolation while still handling non-ClickHouse ("taskEvent"/"postgres") stores. Co-Authored-By: Matt Aitken --- .../app/presenters/v3/RunPresenter.server.ts | 4 +-- .../app/presenters/v3/SpanPresenter.server.ts | 4 +-- .../clickhouse/clickhouseFactory.server.ts | 9 ------ .../app/v3/eventRepository/index.server.ts | 30 +++++++++++++++---- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/apps/webapp/app/presenters/v3/RunPresenter.server.ts b/apps/webapp/app/presenters/v3/RunPresenter.server.ts index cd437b07b91..b4733144907 100644 --- a/apps/webapp/app/presenters/v3/RunPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunPresenter.server.ts @@ -7,7 +7,7 @@ import { SpanSummary } from "~/v3/eventRepository/eventRepository.types"; import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server"; import { isFinalRunStatus } from "~/v3/taskStatus"; import { env } from "~/env.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server"; type Result = Awaited>; export type Run = Result["run"]; @@ -145,7 +145,7 @@ export class RunPresenter { }; } - const { repository } = await clickhouseFactory.getEventRepositoryForOrganization( + const repository = await getEventRepositoryForStore( run.taskEventStore, run.runtimeEnvironment.organizationId ); diff --git a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts index 353c9b25ebd..f6f8f708539 100644 --- a/apps/webapp/app/presenters/v3/SpanPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/SpanPresenter.server.ts @@ -29,7 +29,7 @@ import { extractAIToolCallData, extractAIEmbedData, } from "~/components/runs/v3/ai"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server"; export type PromptSpanData = { slug: string; @@ -130,7 +130,7 @@ export class SpanPresenter extends BasePresenter { const { traceId } = parentRun; - const { repository } = await clickhouseFactory.getEventRepositoryForOrganization( + const repository = await getEventRepositoryForStore( parentRun.taskEventStore, project.organizationId ); diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index 55c0feef7f5..92df7de2b54 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -6,7 +6,6 @@ import { singleton } from "~/utils/singleton"; import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server"; import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types"; -import { eventRepository as postgresEventRepository } from "~/v3/eventRepository/eventRepository.server"; // --------------------------------------------------------------------------- // Default clients (singleton per process) @@ -255,14 +254,6 @@ export class ClickhouseFactory { store: string, organizationId: string ): { key: string; repository: IEventRepository } { - // Non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed - // runs, or "postgres") fall back to the Prisma event repository. This lets - // callers pass `run.taskEventStore` directly without needing to guard - // against legacy/Postgres values. - if (store !== "clickhouse" && store !== "clickhouse_v2") { - return { key: `postgres:${store}`, repository: postgresEventRepository }; - } - const dataStore = this._registry.get(organizationId, "CLICKHOUSE"); if (!dataStore) { diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index f3449564fce..c2e772cc127 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -32,6 +32,27 @@ export function resolveEventRepositoryForStore( return eventRepository; } +/** + * Async variant of {@link resolveEventRepositoryForStore}. Awaits the factory's + * registry readiness before returning the ClickHouse event repository; for + * non-ClickHouse stores (e.g. the "taskEvent" DB default for Postgres-backed + * runs) it returns the Prisma event repository without ever touching the + * factory — so the factory never needs to know about Postgres. + */ +export async function getEventRepositoryForStore( + store: string, + organizationId: string +): Promise { + if (store !== EVENT_STORE_TYPES.CLICKHOUSE && store !== EVENT_STORE_TYPES.CLICKHOUSE_V2) { + return eventRepository; + } + const { repository } = await clickhouseFactory.getEventRepositoryForOrganization( + store, + organizationId + ); + return repository; +} + export async function getConfiguredEventRepository( organizationId: string ): Promise<{ repository: IEventRepository; store: EventStoreType }> { @@ -219,11 +240,10 @@ async function recordRunEvent( }; } - const { repository: $eventRepository } = - await clickhouseFactory.getEventRepositoryForOrganization( - foundRun.taskEventStore, - foundRun.runtimeEnvironment.organizationId - ); + const $eventRepository = await getEventRepositoryForStore( + foundRun.taskEventStore, + foundRun.runtimeEnvironment.organizationId + ); const { attributes, startTime, ...optionsRest } = options; From 7c67046d594b6eb7ce92f33cb1542bb54b9f275f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:27:52 +0000 Subject: [PATCH 26/32] Make OrganizationDataStoresRegistry deterministic on overlap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sort findMany() by `key` for a stable winner when multiple rows assign the same `${orgId}:${kind}`, and log an error identifying the winning and ignored rows instead of overwriting silently. Does not fail the load — failing the registry would break every customer, not just the misconfigured orgs. Co-Authored-By: Matt Aitken --- .../organizationDataStoresRegistry.server.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts index f2ef02cc3bd..838dd5ae5b8 100644 --- a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts +++ b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts @@ -33,10 +33,18 @@ export class OrganizationDataStoresRegistry { } async loadFromDatabase(): Promise { - const rows = await this._prisma.organizationDataStore.findMany(); + // Sort by `key` (unique, immutable) to ensure a deterministic winner when the + // same `${orgId}:${kind}` appears in multiple rows. The registry must never + // throw on overlap — failing the load would break every customer, not just the + // misconfigured orgs — so we keep the first entry and log an error instead. + const rows = await this._prisma.organizationDataStore.findMany({ + orderBy: { key: "asc" }, + }); const secretStore = getSecretStore("DATABASE", { prismaClient: this._prisma }); const lookup = new Map(); + /** Tracks which row's `key` already owns each `${orgId}:${kind}` so we can log conflicts. */ + const winnerByLookupKey = new Map(); for (const row of rows) { let parsed: ParsedDataStore | null = null; @@ -75,8 +83,16 @@ export class OrganizationDataStoresRegistry { } for (const orgId of row.organizationIds) { - const key = `${orgId}:${row.kind}`; - lookup.set(key, parsed); + const lookupKey = `${orgId}:${row.kind}`; + const existingWinner = winnerByLookupKey.get(lookupKey); + if (existingWinner) { + console.error( + `[OrganizationDataStoresRegistry] Overlapping OrganizationDataStore assignment for orgId="${orgId}" kind=${row.kind}: already routed to "${existingWinner}", ignoring "${row.key}". Pick one store per (org, kind) to resolve.` + ); + continue; + } + winnerByLookupKey.set(lookupKey, row.key); + lookup.set(lookupKey, parsed); } } From fd4e207907676bbb3c7ec057fefb21ca585477e4 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 17 Apr 2026 12:17:56 +0100 Subject: [PATCH 27/32] try/catch getting the clickhouse client --- .../route.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx index 69ce432f4e0..d6e866184a5 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx @@ -102,9 +102,13 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }); } - const clickhouse = await clickhouseFactory.getClickhouseForOrganization(project.organizationId, "standard"); - const presenter = new TestTaskPresenter($replica, clickhouse); try { + const clickhouse = await clickhouseFactory.getClickhouseForOrganization( + project.organizationId, + "standard" + ); + const presenter = new TestTaskPresenter($replica, clickhouse); + const [result, regionsResult] = await Promise.all([ presenter.call({ userId: user.id, From 95dc13085832046dbde15d943d0f7101d3a0997a Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 17 Apr 2026 12:40:48 +0100 Subject: [PATCH 28/32] Better handling for no org id on task run --- .../resources.runs.$runParam.logs.download.ts | 7 +- .../webapp/app/v3/runEngineHandlers.server.ts | 485 +++++++++--------- 2 files changed, 253 insertions(+), 239 deletions(-) diff --git a/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts b/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts index 5da10a8a8af..5f19f129729 100644 --- a/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts +++ b/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts @@ -29,14 +29,11 @@ export async function loader({ params, request }: LoaderFunctionArgs) { }, }); - if (!run) { + if (!run || !run.organizationId) { return new Response("Not found", { status: 404 }); } - const eventRepository = resolveEventRepositoryForStore( - run.taskEventStore, - run.organizationId ?? "" - ); + const eventRepository = resolveEventRepositoryForStore(run.taskEventStore, run.organizationId); const runEvents = await eventRepository.getRunEvents( getTaskEventStoreTableForRun(run), diff --git a/apps/webapp/app/v3/runEngineHandlers.server.ts b/apps/webapp/app/v3/runEngineHandlers.server.ts index eced6c758ee..82ed633e468 100644 --- a/apps/webapp/app/v3/runEngineHandlers.server.ts +++ b/apps/webapp/app/v3/runEngineHandlers.server.ts @@ -17,10 +17,7 @@ import { QueueSizeLimitExceededError } from "~/v3/services/common.server"; import { TriggerTaskService } from "~/v3/services/triggerTask.server"; import { tracer } from "~/v3/tracer.server"; import { createExceptionPropertiesFromError } from "./eventRepository/common.server"; -import { - recordRunDebugLog, - resolveEventRepositoryForStore, -} from "./eventRepository/index.server"; +import { recordRunDebugLog, resolveEventRepositoryForStore } from "./eventRepository/index.server"; import { roomFromFriendlyRunId, socketIo } from "./handleSocketIo.server"; import { engine } from "./runEngine.server"; import { PerformTaskRunAlertsService } from "./services/alerts/performTaskRunAlerts.server"; @@ -187,9 +184,16 @@ export function registerRunEngineEventBusHandlers() { return; } + if (!taskRun.organizationId) { + logger.error("[runAttemptFailed] Task run has no organization id", { + runId: run.id, + }); + return; + } + const eventRepository = resolveEventRepositoryForStore( run.taskEventStore, - taskRun.organizationId ?? "" + taskRun.organizationId ); const [createAttemptFailedRunEventError] = await tryCatch( @@ -291,9 +295,16 @@ export function registerRunEngineEventBusHandlers() { return; } + if (!blockedRun.organizationId) { + logger.error("[cachedRunCompleted] Blocked run has no organization id", { + blockedRunId, + }); + return; + } + const eventRepository = resolveEventRepositoryForStore( blockedRun.taskEventStore, - blockedRun.organizationId ?? "" + blockedRun.organizationId ); const [completeCachedRunEventError] = await tryCatch( @@ -431,49 +442,53 @@ export function registerRunEngineEventBusHandlers() { } }); - engine.eventBus.on("runRetryScheduled", async ({ time, run, environment, retryAt, organization }) => { - try { - if (retryAt && time && time >= retryAt) { - return; - } + engine.eventBus.on( + "runRetryScheduled", + async ({ time, run, environment, retryAt, organization }) => { + try { + if (retryAt && time && time >= retryAt) { + return; + } - let retryMessage = `Retry ${typeof run.attemptNumber === "number" ? `#${run.attemptNumber - 1}` : "" + let retryMessage = `Retry ${ + typeof run.attemptNumber === "number" ? `#${run.attemptNumber - 1}` : "" } delay`; - if (run.nextMachineAfterOOM) { - retryMessage += ` after OOM`; - } + if (run.nextMachineAfterOOM) { + retryMessage += ` after OOM`; + } - const eventRepository = resolveEventRepositoryForStore( - run.taskEventStore ?? "taskEvent", - organization.id - ); + const eventRepository = resolveEventRepositoryForStore( + run.taskEventStore ?? "taskEvent", + organization.id + ); - await eventRepository.recordEvent(retryMessage, { - startTime: BigInt(time.getTime() * 1000000), - taskSlug: run.taskIdentifier, - environment, - attributes: { - properties: { - retryAt: retryAt.toISOString(), - nextMachine: run.nextMachineAfterOOM, - }, - runId: run.friendlyId, - style: { - icon: "schedule-attempt", + await eventRepository.recordEvent(retryMessage, { + startTime: BigInt(time.getTime() * 1000000), + taskSlug: run.taskIdentifier, + environment, + attributes: { + properties: { + retryAt: retryAt.toISOString(), + nextMachine: run.nextMachineAfterOOM, + }, + runId: run.friendlyId, + style: { + icon: "schedule-attempt", + }, }, - }, - context: run.traceContext as Record, - endTime: retryAt, - }); - } catch (error) { - logger.error("[runRetryScheduled] Failed to record retry event", { - error: error instanceof Error ? error.message : error, - runId: run.id, - spanId: run.spanId, - }); + context: run.traceContext as Record, + endTime: retryAt, + }); + } catch (error) { + logger.error("[runRetryScheduled] Failed to record retry event", { + error: error instanceof Error ? error.message : error, + runId: run.id, + spanId: run.spanId, + }); + } } - }); + ); engine.eventBus.on("runAttemptStarted", async ({ time, run, organization }) => { try { @@ -506,10 +521,10 @@ export function registerRunEngineEventBusHandlers() { error: e instanceof Error ? { - name: e.name, - message: e.message, - stack: e.stack, - } + name: e.name, + message: e.message, + stack: e.stack, + } : e, }); } else { @@ -518,10 +533,10 @@ export function registerRunEngineEventBusHandlers() { error: e instanceof Error ? { - name: e.name, - message: e.message, - stack: e.stack, - } + name: e.name, + message: e.message, + stack: e.stack, + } : e, }); } @@ -679,121 +694,200 @@ const QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE = "QUEUE_SIZE_LIMIT_EXCEEDED"; */ export function setupBatchQueueCallbacks() { // Item processing callback - creates a run for each batch item - engine.setBatchProcessItemCallback(async ({ batchId, friendlyId, itemIndex, item, meta, attempt, isFinalAttempt }) => { - return tracer.startActiveSpan( - "batch.processItem", - { - kind: SpanKind.INTERNAL, - attributes: { - "batch.id": friendlyId, - "batch.item_index": itemIndex, - "batch.task": item.task, - "batch.environment_id": meta.environmentId, - "batch.parent_run_id": meta.parentRunId ?? "", - "batch.attempt": attempt, - "batch.is_final_attempt": isFinalAttempt, + engine.setBatchProcessItemCallback( + async ({ batchId, friendlyId, itemIndex, item, meta, attempt, isFinalAttempt }) => { + return tracer.startActiveSpan( + "batch.processItem", + { + kind: SpanKind.INTERNAL, + attributes: { + "batch.id": friendlyId, + "batch.item_index": itemIndex, + "batch.task": item.task, + "batch.environment_id": meta.environmentId, + "batch.parent_run_id": meta.parentRunId ?? "", + "batch.attempt": attempt, + "batch.is_final_attempt": isFinalAttempt, + }, }, - }, - async (span) => { - const triggerFailedTaskService = new TriggerFailedTaskService({ - prisma, - engine, - replicaPrisma: $replica, - }); + async (span) => { + const triggerFailedTaskService = new TriggerFailedTaskService({ + prisma, + engine, + replicaPrisma: $replica, + }); - // Check for pre-marked error items (e.g. oversized payloads) - const itemError = item.options?.__error as string | undefined; - if (itemError) { - const errorCode = (item.options?.__errorCode as string) ?? "ITEM_ERROR"; + // Check for pre-marked error items (e.g. oversized payloads) + const itemError = item.options?.__error as string | undefined; + if (itemError) { + const errorCode = (item.options?.__errorCode as string) ?? "ITEM_ERROR"; + + let environment: AuthenticatedEnvironment | undefined; + try { + environment = (await findEnvironmentById(meta.environmentId)) ?? undefined; + } catch { + // Best-effort environment lookup + } + + if (environment) { + const failedRunId = await triggerFailedTaskService.call({ + taskId: item.task, + environment, + payload: item.payload ?? "{}", + payloadType: item.payloadType as string, + errorMessage: itemError, + errorCode: errorCode as TaskRunErrorCodes, + parentRunId: meta.parentRunId, + resumeParentOnCompletion: meta.resumeParentOnCompletion, + batch: { id: batchId, index: itemIndex }, + traceContext: meta.traceContext as Record | undefined, + spanParentAsLink: meta.spanParentAsLink, + }); + + if (failedRunId) { + span.setAttribute("batch.result.pre_failed", true); + span.setAttribute("batch.result.run_id", failedRunId); + span.end(); + return { success: true as const, runId: failedRunId }; + } + } + + // Fallback if TriggerFailedTaskService or environment lookup fails + span.end(); + return { success: false as const, error: itemError, errorCode }; + } let environment: AuthenticatedEnvironment | undefined; try { environment = (await findEnvironmentById(meta.environmentId)) ?? undefined; - } catch { - // Best-effort environment lookup - } - if (environment) { - const failedRunId = await triggerFailedTaskService.call({ - taskId: item.task, - environment, - payload: item.payload ?? "{}", - payloadType: item.payloadType as string, - errorMessage: itemError, - errorCode: errorCode as TaskRunErrorCodes, - parentRunId: meta.parentRunId, - resumeParentOnCompletion: meta.resumeParentOnCompletion, - batch: { id: batchId, index: itemIndex }, - traceContext: meta.traceContext as Record | undefined, - spanParentAsLink: meta.spanParentAsLink, - }); - - if (failedRunId) { - span.setAttribute("batch.result.pre_failed", true); - span.setAttribute("batch.result.run_id", failedRunId); + if (!environment) { + span.setAttribute("batch.result.error", "Environment not found"); span.end(); - return { success: true as const, runId: failedRunId }; + + return { + success: false as const, + error: "Environment not found", + errorCode: "ENVIRONMENT_NOT_FOUND", + }; } - } - // Fallback if TriggerFailedTaskService or environment lookup fails - span.end(); - return { success: false as const, error: itemError, errorCode }; - } + const triggerTaskService = new TriggerTaskService(); - let environment: AuthenticatedEnvironment | undefined; - try { - environment = (await findEnvironmentById(meta.environmentId)) ?? undefined; + // Normalize payload - for application/store (R2 paths), this passes through as-is + const payload = normalizePayload(item.payload, item.payloadType); - if (!environment) { - span.setAttribute("batch.result.error", "Environment not found"); - span.end(); + const result = await triggerTaskService.call( + item.task, + environment, + { + payload, + options: { + ...(item.options as Record), + payloadType: item.payloadType, + parentRunId: meta.parentRunId, + resumeParentOnCompletion: meta.resumeParentOnCompletion, + parentBatch: batchId, + }, + }, + { + triggerVersion: meta.triggerVersion, + traceContext: meta.traceContext as Record | undefined, + spanParentAsLink: meta.spanParentAsLink, + batchId, + batchIndex: itemIndex, + realtimeStreamsVersion: meta.realtimeStreamsVersion, + planType: meta.planType, + triggerSource: meta.parentRunId ? "sdk" : meta.triggerSource ?? "api", + triggerAction: "trigger", + }, + "V2" + ); - return { - success: false as const, - error: "Environment not found", - errorCode: "ENVIRONMENT_NOT_FOUND", - }; - } + if (result) { + span.setAttribute("batch.result.run_id", result.run.friendlyId); + span.end(); + return { success: true as const, runId: result.run.friendlyId }; + } else { + logger.error("[BatchQueue] TriggerTaskService returned undefined", { + batchId, + friendlyId, + itemIndex, + task: item.task, + environmentId: meta.environmentId, + attempt, + isFinalAttempt, + }); - const triggerTaskService = new TriggerTaskService(); + span.setAttribute("batch.result.error", "TriggerTaskService returned undefined"); + + // Only create a pre-failed run on the final attempt; otherwise let the retry mechanism handle it + if (isFinalAttempt) { + const failedRunId = await triggerFailedTaskService.call({ + taskId: item.task, + environment, + payload: item.payload, + payloadType: item.payloadType as string, + errorMessage: "TriggerTaskService returned undefined", + parentRunId: meta.parentRunId, + resumeParentOnCompletion: meta.resumeParentOnCompletion, + batch: { id: batchId, index: itemIndex }, + options: item.options as Record, + traceContext: meta.traceContext as Record | undefined, + spanParentAsLink: meta.spanParentAsLink, + errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER, + }); + + span.end(); + + if (failedRunId) { + return { success: true as const, runId: failedRunId }; + } + } else { + span.end(); + } - // Normalize payload - for application/store (R2 paths), this passes through as-is - const payload = normalizePayload(item.payload, item.payloadType); + return { + success: false as const, + error: "TriggerTaskService returned undefined", + errorCode: "TRIGGER_FAILED", + }; + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Queue-size-limit rejections are a customer-overload scenario (the + // env's queue is at its configured max). Retrying is pointless — the + // same item will fail again — and creating pre-failed TaskRuns for + // every item of every retried batch is exactly what chews through + // DB capacity when a noisy tenant fills their queue. Signal the + // BatchQueue to skip retries and skip pre-failed run creation, and + // let the completion callback collapse the per-item errors into a + // single summary row. + if (error instanceof QueueSizeLimitExceededError) { + logger.warn("[BatchQueue] Batch item rejected: queue size limit reached", { + batchId, + friendlyId, + itemIndex, + task: item.task, + environmentId: meta.environmentId, + maximumSize: error.maximumSize, + }); - const result = await triggerTaskService.call( - item.task, - environment, - { - payload, - options: { - ...(item.options as Record), - payloadType: item.payloadType, - parentRunId: meta.parentRunId, - resumeParentOnCompletion: meta.resumeParentOnCompletion, - parentBatch: batchId, - }, - }, - { - triggerVersion: meta.triggerVersion, - traceContext: meta.traceContext as Record | undefined, - spanParentAsLink: meta.spanParentAsLink, - batchId, - batchIndex: itemIndex, - realtimeStreamsVersion: meta.realtimeStreamsVersion, - planType: meta.planType, - triggerSource: meta.parentRunId ? "sdk" : meta.triggerSource ?? "api", - triggerAction: "trigger", - }, - "V2" - ); + span.setAttribute("batch.result.error", errorMessage); + span.setAttribute("batch.result.errorCode", QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE); + span.setAttribute("batch.result.skipRetries", true); + span.end(); - if (result) { - span.setAttribute("batch.result.run_id", result.run.friendlyId); - span.end(); - return { success: true as const, runId: result.run.friendlyId }; - } else { - logger.error("[BatchQueue] TriggerTaskService returned undefined", { + return { + success: false as const, + error: errorMessage, + errorCode: QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE, + skipRetries: true, + }; + } + + logger.error("[BatchQueue] Failed to trigger batch item", { batchId, friendlyId, itemIndex, @@ -801,18 +895,20 @@ export function setupBatchQueueCallbacks() { environmentId: meta.environmentId, attempt, isFinalAttempt, + error, }); - span.setAttribute("batch.result.error", "TriggerTaskService returned undefined"); + span.setAttribute("batch.result.error", errorMessage); + span.recordException(error instanceof Error ? error : new Error(String(error))); // Only create a pre-failed run on the final attempt; otherwise let the retry mechanism handle it - if (isFinalAttempt) { + if (isFinalAttempt && environment) { const failedRunId = await triggerFailedTaskService.call({ taskId: item.task, environment, payload: item.payload, payloadType: item.payloadType as string, - errorMessage: "TriggerTaskService returned undefined", + errorMessage, parentRunId: meta.parentRunId, resumeParentOnCompletion: meta.resumeParentOnCompletion, batch: { id: batchId, index: itemIndex }, @@ -831,95 +927,16 @@ export function setupBatchQueueCallbacks() { span.end(); } - return { - success: false as const, - error: "TriggerTaskService returned undefined", - errorCode: "TRIGGER_FAILED", - }; - } - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - - // Queue-size-limit rejections are a customer-overload scenario (the - // env's queue is at its configured max). Retrying is pointless — the - // same item will fail again — and creating pre-failed TaskRuns for - // every item of every retried batch is exactly what chews through - // DB capacity when a noisy tenant fills their queue. Signal the - // BatchQueue to skip retries and skip pre-failed run creation, and - // let the completion callback collapse the per-item errors into a - // single summary row. - if (error instanceof QueueSizeLimitExceededError) { - logger.warn("[BatchQueue] Batch item rejected: queue size limit reached", { - batchId, - friendlyId, - itemIndex, - task: item.task, - environmentId: meta.environmentId, - maximumSize: error.maximumSize, - }); - - span.setAttribute("batch.result.error", errorMessage); - span.setAttribute("batch.result.errorCode", QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE); - span.setAttribute("batch.result.skipRetries", true); - span.end(); - return { success: false as const, error: errorMessage, - errorCode: QUEUE_SIZE_LIMIT_EXCEEDED_ERROR_CODE, - skipRetries: true, + errorCode: "TRIGGER_ERROR", }; } - - logger.error("[BatchQueue] Failed to trigger batch item", { - batchId, - friendlyId, - itemIndex, - task: item.task, - environmentId: meta.environmentId, - attempt, - isFinalAttempt, - error, - }); - - span.setAttribute("batch.result.error", errorMessage); - span.recordException(error instanceof Error ? error : new Error(String(error))); - - // Only create a pre-failed run on the final attempt; otherwise let the retry mechanism handle it - if (isFinalAttempt && environment) { - const failedRunId = await triggerFailedTaskService.call({ - taskId: item.task, - environment, - payload: item.payload, - payloadType: item.payloadType as string, - errorMessage, - parentRunId: meta.parentRunId, - resumeParentOnCompletion: meta.resumeParentOnCompletion, - batch: { id: batchId, index: itemIndex }, - options: item.options as Record, - traceContext: meta.traceContext as Record | undefined, - spanParentAsLink: meta.spanParentAsLink, - errorCode: TaskRunErrorCodes.BATCH_ITEM_COULD_NOT_TRIGGER, - }); - - span.end(); - - if (failedRunId) { - return { success: true as const, runId: failedRunId }; - } - } else { - span.end(); - } - - return { - success: false as const, - error: errorMessage, - errorCode: "TRIGGER_ERROR", - }; } - } - ); - }); + ); + } + ); // Batch completion callback - updates Postgres with results engine.setBatchCompletionCallback(async (result: CompleteBatchResult) => { From c28ee490a9e533214a8857268e8b4af6e053f0be Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 17 Apr 2026 12:41:02 +0100 Subject: [PATCH 29/32] Test for deterministic data store resolution if there are multiple entries --- .../organizationDataStoresRegistry.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/apps/webapp/test/organizationDataStoresRegistry.test.ts b/apps/webapp/test/organizationDataStoresRegistry.test.ts index 44fe851a404..f8ff9dde41f 100644 --- a/apps/webapp/test/organizationDataStoresRegistry.test.ts +++ b/apps/webapp/test/organizationDataStoresRegistry.test.ts @@ -117,6 +117,39 @@ describe("OrganizationDataStoresRegistry", () => { expect(z?.url).toBe(TEST_URL); }); + postgresTest( + "when an org appears in multiple data stores, first row by id asc wins", + async ({ prisma }) => { + const registry = new OrganizationDataStoresRegistry(prisma); + const sharedOrg = "org-dup-overlap"; + + await registry.addDataStore({ + key: "dup-overlap-first", + kind: "CLICKHOUSE", + organizationIds: [sharedOrg], + config: ClickhouseConnectionSchema.parse({ url: TEST_URL }), + }); + await registry.addDataStore({ + key: "dup-overlap-second", + kind: "CLICKHOUSE", + organizationIds: [sharedOrg], + config: ClickhouseConnectionSchema.parse({ url: TEST_URL_2 }), + }); + + const [winner] = await prisma.organizationDataStore.findMany({ + where: { key: { in: ["dup-overlap-first", "dup-overlap-second"] } }, + orderBy: { id: "asc" }, + }); + expect(winner).toBeDefined(); + + await registry.loadFromDatabase(); + + const expectedUrl = + winner!.key === "dup-overlap-first" ? TEST_URL : TEST_URL_2; + expect(registry.get(sharedOrg, "CLICKHOUSE")?.url).toBe(expectedUrl); + } + ); + postgresTest("updateDataStore updates organizationIds and rotates the secret", async ({ prisma }) => { const registry = new OrganizationDataStoresRegistry(prisma); From 6b3fcd3ddaa472a853a4d986b2f8b3bed209a8a1 Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 17 Apr 2026 13:13:00 +0100 Subject: [PATCH 30/32] Separate the clickhouseFactoryInstance so the db client isn't pulled into tests --- .../app/presenters/v3/ApiRunListPresenter.server.ts | 2 +- .../v3/CreateBulkActionPresenter.server.ts | 2 +- .../app/presenters/v3/RunTagListPresenter.server.ts | 2 +- .../app/presenters/v3/TaskListPresenter.server.ts | 2 +- .../app/presenters/v3/UsagePresenter.server.ts | 2 +- .../presenters/v3/ViewSchedulePresenter.server.ts | 2 +- .../app/presenters/v3/WaitpointPresenter.server.ts | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../route.tsx | 2 +- .../routes/admin.api.v1.runs-replication.create.ts | 2 +- .../routes/admin.api.v1.runs-replication.start.ts | 2 +- apps/webapp/app/routes/api.v1.prompts.$slug.ts | 2 +- .../app/routes/api.v1.prompts.$slug.versions.ts | 2 +- apps/webapp/app/routes/api.v1.prompts._index.ts | 2 +- ...ects.$projectParam.env.$envParam.logs.$logId.tsx | 2 +- ...lug.projects.$projectParam.env.$envParam.logs.ts | 2 +- ...env.$envParam.prompts.$promptSlug.generations.ts | 2 +- .../services/clickhouse/clickhouseFactory.server.ts | 10 ---------- .../clickhouse/clickhouseFactoryInstance.server.ts | 13 +++++++++++++ apps/webapp/app/services/queryService.server.ts | 2 +- .../app/services/runsReplicationInstance.server.ts | 2 +- apps/webapp/app/v3/eventRepository/index.server.ts | 2 +- apps/webapp/app/v3/otlpExporter.server.ts | 6 ++---- .../services/alerts/errorAlertEvaluator.server.ts | 2 +- .../app/v3/services/bulk/BulkActionV2.server.ts | 2 +- 35 files changed, 47 insertions(+), 46 deletions(-) create mode 100644 apps/webapp/app/services/clickhouse/clickhouseFactoryInstance.server.ts diff --git a/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts index 357d66501b8..1d35a30e4f2 100644 --- a/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts @@ -9,7 +9,7 @@ import { type Project, type RuntimeEnvironment, type TaskRunStatus } from "@trig import assertNever from "assert-never"; import { z } from "zod"; import { API_VERSIONS, RunStatusUnspecifiedApiVersion } from "~/api/versions"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { logger } from "~/services/logger.server"; import { CoercedDate } from "~/utils/zod"; import { ServiceValidationError } from "~/v3/services/baseService.server"; diff --git a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts index eeb5b3d871e..c3c62cd5d95 100644 --- a/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/CreateBulkActionPresenter.server.ts @@ -1,6 +1,6 @@ import { type PrismaClient } from "@trigger.dev/database"; import { CreateBulkActionSearchParams } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; import { getRunFiltersFromRequest } from "../RunFilters.server"; import { BasePresenter } from "./basePresenter.server"; diff --git a/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts b/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts index 44d6f2a0747..c4b524ec329 100644 --- a/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/RunTagListPresenter.server.ts @@ -1,6 +1,6 @@ import { RunsRepository } from "~/services/runsRepository/runsRepository.server"; import { BasePresenter } from "./basePresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { type PrismaClient } from "@trigger.dev/database"; import { timeFilters } from "~/components/runs/v3/SharedFilters"; diff --git a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts index 8a41d0ef630..2ee5d7a0674 100644 --- a/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/TaskListPresenter.server.ts @@ -4,7 +4,7 @@ import { type TaskTriggerSource, } from "@trigger.dev/database"; import { $replica } from "~/db.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { type AverageDurations, ClickHouseEnvironmentMetricsRepository, diff --git a/apps/webapp/app/presenters/v3/UsagePresenter.server.ts b/apps/webapp/app/presenters/v3/UsagePresenter.server.ts index d312088b6d9..f04e53496a2 100644 --- a/apps/webapp/app/presenters/v3/UsagePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/UsagePresenter.server.ts @@ -4,7 +4,7 @@ import { getUsage, getUsageSeries } from "~/services/platform.v3.server"; import { createTimeSeriesData } from "~/utils/graphs"; import { BasePresenter } from "./basePresenter.server"; import { DataPoint, linear } from "regression"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; type Options = { organizationId: string; diff --git a/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts b/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts index 5341568e6f7..dbb1123c488 100644 --- a/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts +++ b/apps/webapp/app/presenters/v3/ViewSchedulePresenter.server.ts @@ -1,7 +1,7 @@ import { ScheduleObject } from "@trigger.dev/core/v3"; import { PrismaClient, prisma } from "~/db.server"; import { displayableEnvironment } from "~/models/runtimeEnvironment.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { nextScheduledTimestamps } from "~/v3/utils/calculateNextSchedule.server"; import { NextRunListPresenter } from "./NextRunListPresenter.server"; import { scheduleWhereClause } from "~/models/schedules.server"; diff --git a/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts b/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts index dc9bf3d1ef0..7877c2cc0c8 100644 --- a/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts +++ b/apps/webapp/app/presenters/v3/WaitpointPresenter.server.ts @@ -1,5 +1,5 @@ import { isWaitpointOutputTimeout, prettyPrintPacket } from "@trigger.dev/core/v3"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; import { logger } from "~/services/logger.server"; import { BasePresenter } from "./basePresenter.server"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx index f7866565c7d..a7c07965d69 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx @@ -31,7 +31,7 @@ import { MetricDashboardPresenter, } from "~/presenters/v3/MetricDashboardPresenter.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUser } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { EnvironmentParamSchema } from "~/utils/pathBuilder"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx index a63c3fe814c..5108f41ba03 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors.$fingerprint/route.tsx @@ -31,7 +31,7 @@ import { type ErrorGroupSummary, } from "~/presenters/v3/ErrorGroupPresenter.server"; import { type NextRunList } from "~/presenters/v3/NextRunListPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUser, requireUserId } from "~/services/session.server"; import { EnvironmentParamSchema, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx index 1c9929b2958..a3b7c92bd64 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.errors._index/route.tsx @@ -70,7 +70,7 @@ import { type ErrorOccurrences, type ErrorsList as ErrorsListData, } from "~/presenters/v3/ErrorsListPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; import { requireUser } from "~/services/session.server"; import { formatNumberCompact } from "~/utils/numberFormatter"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx index 02401f043b8..6147b1df0ad 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs/route.tsx @@ -16,7 +16,7 @@ import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { LogsListPresenter, LogEntry } from "~/presenters/v3/LogsListPresenter.server"; import type { LogLevel } from "~/utils/logUtils"; import { $replica, prisma } from "~/db.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { NavBar, PageTitle } from "~/components/primitives/PageHeader"; import { PageBody, PageContainer } from "~/components/layout/AppLayout"; import { Suspense, useCallback, useEffect, useMemo, useRef, useState, useTransition } from "react"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx index 813879cf24f..f4248aa64b6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.$modelId/route.tsx @@ -28,7 +28,7 @@ import type { QueryWidgetConfig } from "~/components/metrics/QueryWidget"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { ModelRegistryPresenter } from "~/presenters/v3/ModelRegistryPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUserId } from "~/services/session.server"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx index 29c2e4f6c8c..10c85601712 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models._index/route.tsx @@ -71,7 +71,7 @@ import { type PopularModel, ModelRegistryPresenter, } from "~/presenters/v3/ModelRegistryPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUserId } from "~/services/session.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx index 1306eb91943..1f8748f08cf 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.models.compare/route.tsx @@ -20,7 +20,7 @@ import { type ModelComparisonItem, ModelRegistryPresenter, } from "~/presenters/v3/ModelRegistryPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUserId } from "~/services/session.server"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx index de43ea46b23..7201b94a16e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx @@ -70,7 +70,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { type GenerationRow, PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; import { SpanView } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.spans.$spanParam/route"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { getResizableSnapshot } from "~/services/resizablePanel.server"; import { requireUserId } from "~/services/session.server"; import { PromptService } from "~/v3/services/promptService.server"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx index b44c5954ffe..5d3f36a06f1 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts._index/route.tsx @@ -22,7 +22,7 @@ import { useProject } from "~/hooks/useProject"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUserId } from "~/services/session.server"; import { docsPath, EnvironmentParamSchema, v3PromptsPath } from "~/utils/pathBuilder"; import { LinkButton } from "~/components/primitives/Buttons"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx index 2f4e7937727..712a39a7820 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam/route.tsx @@ -92,7 +92,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server"; import { RunEnvironmentMismatchError, RunPresenter } from "~/presenters/v3/RunPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { getImpersonationId } from "~/services/impersonation.server"; import { logger } from "~/services/logger.server"; import { getResizableSnapshot } from "~/services/resizablePanel.server"; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx index 28a3ef46a77..cfe83acbbb4 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs._index/route.tsx @@ -44,7 +44,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; import { NextRunListPresenter } from "~/presenters/v3/NextRunListPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { setRootOnlyFilterPreference, uiPreferencesStorage, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx index d6e866184a5..a611ffbcf89 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.test.tasks.$taskParam/route.tsx @@ -74,7 +74,7 @@ import { Dialog, DialogContent, DialogHeader, DialogTrigger } from "~/components import { DialogClose, DialogDescription } from "@radix-ui/react-dialog"; import { FormButtons } from "~/components/primitives/FormButtons"; import { $replica } from "~/db.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { RegionsPresenter, type Region } from "~/presenters/v3/RegionsPresenter.server"; import { TestSidebarTabs } from "./TestSidebarTabs"; import { AIPayloadTabContent } from "./AIPayloadTabContent"; diff --git a/apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts b/apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts index 870c5891a13..0026c66bfda 100644 --- a/apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts +++ b/apps/webapp/app/routes/admin.api.v1.runs-replication.create.ts @@ -2,7 +2,7 @@ import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; import { z } from "zod"; import { env } from "~/env.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { RunsReplicationService } from "~/services/runsReplicationService.server"; import { getRunsReplicationGlobal, diff --git a/apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts b/apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts index 7291a275b19..d67ca3a1ae3 100644 --- a/apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts +++ b/apps/webapp/app/routes/admin.api.v1.runs-replication.start.ts @@ -1,6 +1,6 @@ import { ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { requireAdminApiRequest } from "~/services/personalAccessToken.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { getRunsReplicationGlobal } from "~/services/runsReplicationGlobal.server"; import { runsReplicationInstance } from "~/services/runsReplicationInstance.server"; diff --git a/apps/webapp/app/routes/api.v1.prompts.$slug.ts b/apps/webapp/app/routes/api.v1.prompts.$slug.ts index 919133d3cbd..5b429211720 100644 --- a/apps/webapp/app/routes/api.v1.prompts.$slug.ts +++ b/apps/webapp/app/routes/api.v1.prompts.$slug.ts @@ -2,7 +2,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { createActionApiRoute, createLoaderApiRoute, diff --git a/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts b/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts index 8f953d1a48c..d5c74b0fed5 100644 --- a/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts +++ b/apps/webapp/app/routes/api.v1.prompts.$slug.versions.ts @@ -2,7 +2,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; const ParamsSchema = z.object({ diff --git a/apps/webapp/app/routes/api.v1.prompts._index.ts b/apps/webapp/app/routes/api.v1.prompts._index.ts index a6ad065c245..be914e67df9 100644 --- a/apps/webapp/app/routes/api.v1.prompts._index.ts +++ b/apps/webapp/app/routes/api.v1.prompts._index.ts @@ -1,6 +1,6 @@ import { json } from "@remix-run/server-runtime"; import { PromptPresenter } from "~/presenters/v3/PromptPresenter.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; export const loader = createLoaderApiRoute( diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx index ae552e96eb9..f4d34907042 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.$logId.tsx @@ -1,7 +1,7 @@ import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { typedjson } from "remix-typedjson"; import { z } from "zod"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { requireUserId } from "~/services/session.server"; import { LogDetailPresenter } from "~/presenters/v3/LogDetailPresenter.server"; import { findProjectBySlug } from "~/models/project.server"; diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts index 7bb0db0d54b..38b7dd390f8 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.logs.ts @@ -6,7 +6,7 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { LogsListPresenter, type LogLevel, LogsListOptionsSchema } from "~/presenters/v3/LogsListPresenter.server"; import { $replica } from "~/db.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { getCurrentPlan } from "~/services/platform.v3.server"; // Valid log levels for filtering diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts index 5188d8ccdfe..e468438cdc1 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug.generations.ts @@ -6,7 +6,7 @@ import { EnvironmentParamSchema } from "~/utils/pathBuilder"; import { parsePeriodToMs } from "~/utils/periods"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { PromptPresenter, type GenerationRow, diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index 92df7de2b54..deeddd5e167 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -3,7 +3,6 @@ import { createHash } from "crypto"; import { ClickhouseEventRepository } from "~/v3/eventRepository/clickhouseEventRepository.server"; import { env } from "~/env.server"; import { singleton } from "~/utils/singleton"; -import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; import type { OrganizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistry.server"; import { type IEventRepository } from "~/v3/eventRepository/eventRepository.types"; @@ -281,15 +280,6 @@ export class ClickhouseFactory { } } -// --------------------------------------------------------------------------- -// Singleton factory instance -// --------------------------------------------------------------------------- - -export const clickhouseFactory = singleton( - "clickhouseFactory", - () => new ClickhouseFactory(organizationDataStoresRegistry) -); - /** * Get admin ClickHouse client for cross-organization queries. * Only use for admin tools and analytics that need to query across all orgs. diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactoryInstance.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactoryInstance.server.ts new file mode 100644 index 00000000000..6795df5f644 --- /dev/null +++ b/apps/webapp/app/services/clickhouse/clickhouseFactoryInstance.server.ts @@ -0,0 +1,13 @@ +import { organizationDataStoresRegistry } from "~/services/dataStores/organizationDataStoresRegistryInstance.server"; +import { singleton } from "~/utils/singleton"; +import { ClickhouseFactory } from "./clickhouseFactory.server"; + +/** + * Production singleton wired to the global organization data-stores registry. + * Import this only from app/runtime code — not from tests that construct a + * {@link ClickhouseFactory} with a stub registry (see `clickhouseFactory.server.ts`). + */ +export const clickhouseFactory = singleton( + "clickhouseFactory", + () => new ClickhouseFactory(organizationDataStoresRegistry) +); diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index 214232ebdf5..4a576c2bf34 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -11,7 +11,7 @@ import type { TableSchema, WhereClauseCondition } from "@internal/tsql"; import { z } from "zod"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; -import { clickhouseFactory } from "./clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "./clickhouse/clickhouseFactoryInstance.server"; import { queryConcurrencyLimiter, DEFAULT_ORG_CONCURRENCY_LIMIT, diff --git a/apps/webapp/app/services/runsReplicationInstance.server.ts b/apps/webapp/app/services/runsReplicationInstance.server.ts index 1a507aafd74..d5071e2d2b8 100644 --- a/apps/webapp/app/services/runsReplicationInstance.server.ts +++ b/apps/webapp/app/services/runsReplicationInstance.server.ts @@ -1,6 +1,6 @@ import invariant from "tiny-invariant"; import { env } from "~/env.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { singleton } from "~/utils/singleton"; import { meter, provider } from "~/v3/tracer.server"; import { RunsReplicationService } from "./runsReplicationService.server"; diff --git a/apps/webapp/app/v3/eventRepository/index.server.ts b/apps/webapp/app/v3/eventRepository/index.server.ts index c2e772cc127..9c30f011a73 100644 --- a/apps/webapp/app/v3/eventRepository/index.server.ts +++ b/apps/webapp/app/v3/eventRepository/index.server.ts @@ -6,7 +6,7 @@ import { logger } from "~/services/logger.server"; import { FEATURE_FLAG } from "../featureFlags"; import { flag } from "../featureFlags.server"; import { getTaskEventStore } from "../taskEventStore.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; export const EVENT_STORE_TYPES = { POSTGRES: "postgres", diff --git a/apps/webapp/app/v3/otlpExporter.server.ts b/apps/webapp/app/v3/otlpExporter.server.ts index 37c5c2840bc..788e7339834 100644 --- a/apps/webapp/app/v3/otlpExporter.server.ts +++ b/apps/webapp/app/v3/otlpExporter.server.ts @@ -20,10 +20,8 @@ import { } from "@trigger.dev/otlp-importer"; import type { MetricsV1Input } from "@internal/clickhouse"; import { logger } from "~/services/logger.server"; -import { - clickhouseFactory, - type ClickhouseFactory, -} from "~/services/clickhouse/clickhouseFactory.server"; +import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { generateSpanId } from "./eventRepository/common.server"; import type { diff --git a/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts b/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts index 40fe2d5f10d..cd68258c015 100644 --- a/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts +++ b/apps/webapp/app/v3/services/alerts/errorAlertEvaluator.server.ts @@ -7,7 +7,7 @@ import { } from "@trigger.dev/database"; import { $replica, prisma } from "~/db.server"; import { ErrorAlertConfig } from "~/models/projectAlert.server"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { logger } from "~/services/logger.server"; import { alertsWorker } from "~/v3/alertsWorker.server"; diff --git a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts index cf1f80165dc..21f5d39db91 100644 --- a/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts +++ b/apps/webapp/app/v3/services/bulk/BulkActionV2.server.ts @@ -7,7 +7,7 @@ import { } from "@trigger.dev/database"; import { getRunFiltersFromRequest } from "~/presenters/RunFilters.server"; import { type CreateBulkActionPayload } from "~/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.bulkaction"; -import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactory.server"; +import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server"; import { parseRunListInputOptions, type RunListInputFilters, From d55216b2eedecac9337c326b3a90230ea521a6bb Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 17 Apr 2026 14:35:12 +0100 Subject: [PATCH 31/32] Use the same ClickHouse settings for org specific clients --- .../clickhouse/clickhouseFactory.server.ts | 97 ++++++++++++++----- 1 file changed, 73 insertions(+), 24 deletions(-) diff --git a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts index deeddd5e167..0159da4fc12 100644 --- a/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts +++ b/apps/webapp/app/services/clickhouse/clickhouseFactory.server.ts @@ -36,6 +36,21 @@ const defaultLogsClickhouseClient = singleton( initializeLogsClickhouseClient ); +function getLogsListClickhouseSettings() { + return { + max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), + max_bytes_before_external_sort: + env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), + max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, + ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { + max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), + }), + ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { + max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, + }), + }; +} + function initializeLogsClickhouseClient() { if (!env.LOGS_CLICKHOUSE_URL) { throw new Error("LOGS_CLICKHOUSE_URL is not set"); @@ -54,18 +69,7 @@ function initializeLogsClickhouseClient() { logLevel: env.CLICKHOUSE_LOG_LEVEL, compression: { request: true }, maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - clickhouseSettings: { - max_memory_usage: env.CLICKHOUSE_LOGS_LIST_MAX_MEMORY_USAGE.toString(), - max_bytes_before_external_sort: - env.CLICKHOUSE_LOGS_LIST_MAX_BYTES_BEFORE_EXTERNAL_SORT.toString(), - max_threads: env.CLICKHOUSE_LOGS_LIST_MAX_THREADS, - ...(env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ && { - max_rows_to_read: env.CLICKHOUSE_LOGS_LIST_MAX_ROWS_TO_READ.toString(), - }), - ...(env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME && { - max_execution_time: env.CLICKHOUSE_LOGS_LIST_MAX_EXECUTION_TIME, - }), - }, + clickhouseSettings: getLogsListClickhouseSettings(), }); } @@ -163,18 +167,63 @@ export type ClientType = "standard" | "events" | "replication" | "logs" | "query function buildOrgClickhouseClient(url: string, clientType: ClientType): ClickHouse { const parsed = new URL(url); parsed.searchParams.delete("secure"); - - return new ClickHouse({ - url: parsed.toString(), - name: `org-clickhouse-${clientType}`, - keepAlive: { - enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", - idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, - }, - logLevel: env.CLICKHOUSE_LOG_LEVEL, - compression: { request: true }, - maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, - }); + const name = `org-clickhouse-${clientType}`; + + switch (clientType) { + case "events": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.EVENTS_CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.EVENTS_CLICKHOUSE_LOG_LEVEL, + compression: { + request: env.EVENTS_CLICKHOUSE_COMPRESSION_REQUEST === "1", + }, + maxOpenConnections: env.EVENTS_CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); + case "replication": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.RUN_REPLICATION_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.RUN_REPLICATION_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.RUN_REPLICATION_CLICKHOUSE_LOG_LEVEL, + compression: { request: true }, + maxOpenConnections: env.RUN_REPLICATION_MAX_OPEN_CONNECTIONS, + }); + case "logs": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { request: true }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + clickhouseSettings: getLogsListClickhouseSettings(), + }); + case "standard": + case "query": + case "admin": + return new ClickHouse({ + url: parsed.toString(), + name, + keepAlive: { + enabled: env.CLICKHOUSE_KEEP_ALIVE_ENABLED === "1", + idleSocketTtl: env.CLICKHOUSE_KEEP_ALIVE_IDLE_SOCKET_TTL_MS, + }, + logLevel: env.CLICKHOUSE_LOG_LEVEL, + compression: { request: true }, + maxOpenConnections: env.CLICKHOUSE_MAX_OPEN_CONNECTIONS, + }); + } } // --------------------------------------------------------------------------- From 9a7fe810e8d5d6eb7d270c5da04c4677e1bfca6f Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Fri, 17 Apr 2026 14:57:27 +0100 Subject: [PATCH 32/32] Update apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../dataStores/organizationDataStoresRegistry.server.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts index 838dd5ae5b8..dff741cab0f 100644 --- a/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts +++ b/apps/webapp/app/services/dataStores/organizationDataStoresRegistry.server.ts @@ -132,10 +132,11 @@ export class OrganizationDataStoresRegistry { data: { key, organizationIds, - kind: "CLICKHOUSE", + kind, config: { version: 1, data: { secretKey } }, }, }); + } async updateDataStore({