diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts index c5085517182..aa76cff4e51 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.test.ts @@ -4,8 +4,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { normalizeConditionRouterIds } from './builders' -const { mockValidateSelectorIds } = vi.hoisted(() => ({ +const { mockValidateSelectorIds, mockGetModelOptions } = vi.hoisted(() => ({ mockValidateSelectorIds: vi.fn(), + mockGetModelOptions: vi.fn(() => []), })) const conditionBlockConfig = { @@ -26,7 +27,24 @@ const routerBlockConfig = { type: 'router_v2', name: 'Router', outputs: {}, - subBlocks: [{ id: 'routes', type: 'router-input' }], + subBlocks: [ + { id: 'routes', type: 'router-input' }, + { id: 'model', type: 'combobox', options: mockGetModelOptions }, + ], +} + +const agentBlockConfig = { + type: 'agent', + name: 'Agent', + outputs: {}, + subBlocks: [{ id: 'model', type: 'combobox', options: mockGetModelOptions }], +} + +const huggingfaceBlockConfig = { + type: 'huggingface', + name: 'HuggingFace', + outputs: {}, + subBlocks: [{ id: 'model', type: 'short-input' }], } vi.mock('@/blocks/registry', () => ({ @@ -37,7 +55,15 @@ vi.mock('@/blocks/registry', () => ({ ? oauthBlockConfig : type === 'router_v2' ? routerBlockConfig - : undefined, + : type === 'agent' + ? agentBlockConfig + : type === 'huggingface' + ? huggingfaceBlockConfig + : undefined, +})) + +vi.mock('@/blocks/utils', () => ({ + getModelOptions: mockGetModelOptions, })) vi.mock('@/lib/copilot/validation/selector-validator', () => ({ @@ -83,6 +109,105 @@ describe('validateInputsForBlock', () => { expect(result.errors).toHaveLength(1) expect(result.errors[0]?.error).toContain('expected a JSON array') }) + + it('accepts known agent model ids', () => { + const result = validateInputsForBlock('agent', { model: 'claude-sonnet-4-6' }, 'agent-1') + + expect(result.errors).toHaveLength(0) + expect(result.validInputs.model).toBe('claude-sonnet-4-6') + }) + + it('rejects hallucinated agent model ids that match a static provider pattern', () => { + const result = validateInputsForBlock('agent', { model: 'claude-sonnet-4.6' }, 'agent-1') + + expect(result.validInputs.model).toBeUndefined() + expect(result.errors).toHaveLength(1) + expect(result.errors[0]?.field).toBe('model') + expect(result.errors[0]?.error).toContain('Unknown model id') + expect(result.errors[0]?.error).toContain('claude-sonnet-4-6') + }) + + it('rejects legacy claude-4.5-haiku style ids', () => { + const result = validateInputsForBlock('agent', { model: 'claude-4.5-haiku' }, 'agent-1') + + expect(result.errors).toHaveLength(1) + expect(result.errors[0]?.error).toContain('Unknown model id') + }) + + it('allows empty model values', () => { + const result = validateInputsForBlock('agent', { model: '' }, 'agent-1') + + expect(result.errors).toHaveLength(0) + expect(result.validInputs.model).toBe('') + }) + + it('allows custom ollama-prefixed model ids', () => { + const result = validateInputsForBlock('agent', { model: 'ollama/my-private-model' }, 'agent-1') + + expect(result.errors).toHaveLength(0) + expect(result.validInputs.model).toBe('ollama/my-private-model') + }) + + it('validates the model field on router_v2 blocks too', () => { + const valid = validateInputsForBlock('router_v2', { model: 'claude-sonnet-4-6' }, 'router-1') + expect(valid.errors).toHaveLength(0) + expect(valid.validInputs.model).toBe('claude-sonnet-4-6') + + const invalid = validateInputsForBlock('router_v2', { model: 'claude-sonnet-4.6' }, 'router-1') + expect(invalid.validInputs.model).toBeUndefined() + expect(invalid.errors).toHaveLength(1) + expect(invalid.errors[0]?.blockType).toBe('router_v2') + expect(invalid.errors[0]?.field).toBe('model') + expect(invalid.errors[0]?.error).toContain('Unknown model id') + }) + + it("does not apply model validation to blocks whose model field is not Sim's catalog", () => { + const result = validateInputsForBlock( + 'huggingface', + { model: 'mistralai/Mistral-7B-Instruct-v0.3' }, + 'hf-1' + ) + + expect(result.errors).toHaveLength(0) + expect(result.validInputs.model).toBe('mistralai/Mistral-7B-Instruct-v0.3') + }) + + it('rejects a bare Ollama-style tag without the provider prefix', () => { + const result = validateInputsForBlock('agent', { model: 'llama3.1:8b' }, 'agent-1') + + expect(result.validInputs.model).toBeUndefined() + expect(result.errors).toHaveLength(1) + expect(result.errors[0]?.error).toContain('Unknown model id') + expect(result.errors[0]?.error).toContain('ollama/') + }) + + it('rejects date-pinned ids that are not literally in the catalog', () => { + const result = validateInputsForBlock( + 'agent', + { model: 'claude-sonnet-4-5-20250929' }, + 'agent-1' + ) + + expect(result.validInputs.model).toBeUndefined() + expect(result.errors).toHaveLength(1) + expect(result.errors[0]?.error).toContain('Unknown model id') + }) + + it('trims whitespace around catalog model ids and stores the trimmed value', () => { + const result = validateInputsForBlock('agent', { model: ' gpt-5.4 ' }, 'agent-1') + + expect(result.errors).toHaveLength(0) + expect(result.validInputs.model).toBe('gpt-5.4') + }) + + it('rejects a pattern-matching but uncataloged id even with surrounding whitespace', () => { + const result = validateInputsForBlock('agent', { model: ' gpt-100-ultra ' }, 'agent-1') + + expect(result.validInputs.model).toBeUndefined() + expect(result.errors).toHaveLength(1) + expect(result.errors[0]?.error).toContain('gpt-100-ultra') + expect(result.errors[0]?.error).not.toMatch(/\s{2,}/) + }) }) describe('normalizeConditionRouterIds', () => { diff --git a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts index efab8a24b3e..2c182cc839c 100644 --- a/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts +++ b/apps/sim/lib/copilot/tools/server/workflow/edit-workflow/validation.ts @@ -3,7 +3,9 @@ import { validateSelectorIds } from '@/lib/copilot/validation/selector-validator import type { PermissionGroupConfig } from '@/lib/permission-groups/types' import { getBlock } from '@/blocks/registry' import type { SubBlockConfig } from '@/blocks/types' +import { getModelOptions } from '@/blocks/utils' import { EDGE, normalizeName } from '@/executor/constants' +import { isKnownModelId, suggestModelIdsForUnknownModel } from '@/providers/models' import { TRIGGER_RUNTIME_SUBBLOCK_IDS } from '@/triggers/constants' import type { EdgeHandleValidationResult, @@ -350,9 +352,31 @@ export function validateValueForSubBlockType( case 'short-input': case 'long-input': case 'combobox': { - // Should be string (combobox allows custom values) + const usesProviderCatalog = + fieldName === 'model' && subBlockConfig.options === getModelOptions + + if (usesProviderCatalog) { + const stringValue = typeof value === 'string' ? value : String(value) + const trimmed = stringValue.trim() + if (trimmed !== '' && !isKnownModelId(trimmed)) { + const suggestions = suggestModelIdsForUnknownModel(trimmed) + const suggestionText = + suggestions.length > 0 ? ` Valid options include: ${suggestions.join(', ')}.` : '' + return { + valid: false, + error: { + blockId, + blockType, + field: fieldName, + value, + error: `Unknown model id "${trimmed}" for block "${blockType}". Read components/blocks/${blockType}.json (the model.options array) for valid ids; prefer entries with recommended: true and avoid deprecated: true. For user-configured models (Ollama, vLLM, OpenRouter, Fireworks), prefix the id with the provider slash, e.g. "ollama/llama3.1:8b".${suggestionText}`, + }, + } + } + return { valid: true, value: trimmed } + } + if (typeof value !== 'string' && typeof value !== 'number') { - // Convert to string but don't error return { valid: true, value: String(value) } } return { valid: true, value } diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts index e0c05768800..4dc2d3458f1 100644 --- a/apps/sim/lib/copilot/vfs/serializers.ts +++ b/apps/sim/lib/copilot/vfs/serializers.ts @@ -2,7 +2,7 @@ import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions' import { isHosted } from '@/lib/core/config/feature-flags' import { isSubBlockHidden } from '@/lib/workflows/subblocks/visibility' import type { BlockConfig, SubBlockConfig } from '@/blocks/types' -import { PROVIDER_DEFINITIONS } from '@/providers/models' +import { DYNAMIC_MODEL_PROVIDERS, PROVIDER_DEFINITIONS } from '@/providers/models' import type { ToolConfig } from '@/tools/types' /** @@ -320,24 +320,38 @@ export function serializeTableMeta(table: { * Excludes dynamic providers (ollama, vllm, openrouter) whose models are user-configured. * Includes provider ID and whether the model is hosted by Sim (no API key required). */ -function getStaticModelOptionsForVFS(): Array<{ +interface StaticModelOption { id: string provider: string hosted: boolean -}> { + recommended?: boolean + speedOptimized?: boolean + deprecated?: boolean +} + +const DYNAMIC_PROVIDERS_NOTE = { + note: 'The options array above lists Sim\'s static provider catalog. These providers also accept user-configured models that are NOT enumerated here: the user may have additional ids available at runtime (e.g. local Ollama tags). To reference one, prefix the model id with the provider slash below — for example "ollama/llama3.1:8b" instead of the bare "llama3.1:8b". The server rejects bare ids that are not in the catalog; always use the prefix for user-configured models.', + prefixes: DYNAMIC_MODEL_PROVIDERS.map((p) => `${p}/`), +} as const + +function getStaticModelOptionsForVFS(): StaticModelOption[] { const hostedProviders = new Set(['openai', 'anthropic', 'google']) - const dynamicProviders = new Set(['ollama', 'vllm', 'openrouter', 'fireworks']) + const dynamicProviders = new Set(DYNAMIC_MODEL_PROVIDERS) - const models: Array<{ id: string; provider: string; hosted: boolean }> = [] + const models: StaticModelOption[] = [] for (const [providerId, def] of Object.entries(PROVIDER_DEFINITIONS)) { if (dynamicProviders.has(providerId)) continue for (const model of def.models) { - models.push({ + const option: StaticModelOption = { id: model.id, provider: providerId, hosted: hostedProviders.has(providerId), - }) + } + if (model.recommended) option.recommended = true + if (model.speedOptimized) option.speedOptimized = true + if (model.deprecated) option.deprecated = true + models.push(option) } } @@ -378,9 +392,9 @@ export function serializeBlockSchema(block: BlockConfig): string { .map((sb) => { const serialized = serializeSubBlock(sb) - // For model comboboxes with function options, inject static model data with hosting info if (sb.id === 'model' && sb.type === 'combobox' && typeof sb.options === 'function') { serialized.options = getStaticModelOptionsForVFS() + serialized.dynamicProviders = DYNAMIC_PROVIDERS_NOTE } return serialized diff --git a/apps/sim/providers/models.ts b/apps/sim/providers/models.ts index 3d8cd0f8cc6..0d265f620fd 100644 --- a/apps/sim/providers/models.ts +++ b/apps/sim/providers/models.ts @@ -59,6 +59,9 @@ export interface ModelDefinition { contextWindow?: number /** ISO date string (YYYY-MM-DD) when the model was first publicly released */ releaseDate?: string + recommended?: boolean + speedOptimized?: boolean + deprecated?: boolean } export interface ProviderDefinition { @@ -216,6 +219,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 1050000, releaseDate: '2026-03-05', + recommended: true, }, { id: 'gpt-5.4-mini', @@ -256,6 +260,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 400000, releaseDate: '2026-03-17', + speedOptimized: true, }, // GPT-5.2 family { @@ -504,6 +509,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 128000, releaseDate: '2024-05-13', + deprecated: true, }, ], }, @@ -537,6 +543,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 1000000, releaseDate: '2026-04-16', + recommended: true, }, { id: 'claude-opus-4-6', @@ -577,6 +584,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 1000000, releaseDate: '2026-02-17', + recommended: true, }, { id: 'claude-opus-4-5', @@ -695,6 +703,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 200000, releaseDate: '2025-10-15', + speedOptimized: true, }, { id: 'claude-3-haiku-20240307', @@ -710,6 +719,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 200000, releaseDate: '2024-03-07', + deprecated: true, }, ], }, @@ -1170,6 +1180,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 1048576, releaseDate: '2026-02-19', + recommended: true, }, { id: 'gemini-3.1-flash-lite-preview', @@ -1253,6 +1264,7 @@ export const PROVIDER_DEFINITIONS: Record = { }, contextWindow: 1048576, releaseDate: '2025-06-17', + speedOptimized: true, }, { id: 'gemini-2.0-flash', @@ -2697,6 +2709,58 @@ export function getProviderModels(providerId: string): string[] { return PROVIDER_DEFINITIONS[providerId]?.models.map((m) => m.id) || [] } +export const DYNAMIC_MODEL_PROVIDERS = ['ollama', 'vllm', 'openrouter', 'fireworks'] as const + +function getAllStaticModelIds(): string[] { + const ids: string[] = [] + for (const [providerId, provider] of Object.entries(PROVIDER_DEFINITIONS)) { + if ((DYNAMIC_MODEL_PROVIDERS as readonly string[]).includes(providerId)) continue + for (const model of provider.models) ids.push(model.id) + } + return ids +} + +const STATIC_MODEL_ID_SET = new Set(getAllStaticModelIds().map((id) => id.toLowerCase())) + +export function isKnownModelId(modelId: string): boolean { + if (!modelId || typeof modelId !== 'string') return false + const trimmed = modelId.trim() + if (!trimmed) return false + + if (STATIC_MODEL_ID_SET.has(trimmed.toLowerCase())) return true + + const lowered = trimmed.toLowerCase() + for (const provider of DYNAMIC_MODEL_PROVIDERS) { + if (lowered.startsWith(`${provider}/`)) return true + } + + return false +} + +function getRecommendedModels(): string[] { + const models: string[] = [] + for (const [providerId, provider] of Object.entries(PROVIDER_DEFINITIONS)) { + if ((DYNAMIC_MODEL_PROVIDERS as readonly string[]).includes(providerId)) continue + for (const model of provider.models) { + if (model.recommended) models.push(model.id) + } + } + return models +} + +export function suggestModelIdsForUnknownModel(_modelId: string, limit = 5): string[] { + const recommended = getRecommendedModels() + if (recommended.length > 0) return recommended.slice(0, limit) + + return [ + PROVIDER_DEFINITIONS.anthropic.defaultModel, + PROVIDER_DEFINITIONS.openai.defaultModel, + PROVIDER_DEFINITIONS.google.defaultModel, + ] + .filter(Boolean) + .slice(0, limit) +} + export function getBaseModelProviders(): Record { return Object.entries(PROVIDER_DEFINITIONS) .filter(([providerId]) => !['ollama', 'vllm', 'openrouter'].includes(providerId))