diff --git a/drift-proposals/gemini-gemini-3.5-transcribe-live-new-family.md b/drift-proposals/gemini-gemini-3.5-transcribe-live-new-family.md new file mode 100644 index 00000000..d7a29d60 --- /dev/null +++ b/drift-proposals/gemini-gemini-3.5-transcribe-live-new-family.md @@ -0,0 +1,53 @@ +# New / unclassified model family: gemini-3.5-transcribe-live + +Provider: gemini +Detected: 2026-08-27 +Status: RESOLVED — decision recorded below and applied to the registry + +This model family appeared in a live /models listing but matches no classification rule (include, exclude, -preview, gemma). drift-sync never silently classifies a new family. + +## Decision + + + + + +Decision: EXCLUDE (applied — excludeFamilies.gemini in +`src/__tests__/drift/model-registry.ts`, with the `excludeFamilies.gemini` +re-pin in `src/__tests__/drift/logic-pin.test.ts`). + +Rationale: wrong modality — a bidirectional streaming SPEECH-TO-TEXT surface. +This is the Live-API half of the same GA line as `gemini-3.5-transcribe` (one +model, two surfaces, two distinct normalized families, so both are enumerated). +Google's release note calls it "Low-latency, bidirectional streaming +speech-to-text over WebSockets" with "interim and finalized transcription +events", and the model card gives it the same input `Audio` -> output `Text, +Word annotations` signature as its unary sibling. + +TWO INDEPENDENT capability facts were observed live, not inferred from the id: + + 1. It DECLARES `bidiGenerateContent`. The Gemini Live leg's discovery + (`fetchLiveCapableModels` in `ws-gemini-live.drift.ts`) filters SOLELY on + that declared method and applies no name heuristic, and it selected this id + — so the declaration is present. + + 2. It CANNOT emit AUDIO. Google refused the resulting session out of band: + `code=1007 reason="The requested combination of response modalities (AUDIO) + is not supported by the model. models/gemini-3.5-transcribe-live"` + (drift run 33296393200, 2026-08-30). + +So it is a bidi Live surface that emits TEXT and cannot emit AUDIO — a streaming +transcriber, not a native-audio conversational model. It belongs to the realtime +canary's domain alongside the already-excluded `gemini-live`, and can never be +text-generation drift. Mirrors the OpenAI `gpt-live-transcribe` decision (PR +#343). + +The same 1007 is ALSO the diagnosis for the Gemini Live leg's own failure; that +half is fixed in `ws-gemini-live.drift.ts`, not here. diff --git a/drift-proposals/gemini-gemini-3.5-transcribe-new-family.md b/drift-proposals/gemini-gemini-3.5-transcribe-new-family.md new file mode 100644 index 00000000..384aa33d --- /dev/null +++ b/drift-proposals/gemini-gemini-3.5-transcribe-new-family.md @@ -0,0 +1,43 @@ +# New / unclassified model family: gemini-3.5-transcribe + +Provider: gemini +Detected: 2026-08-27 +Status: RESOLVED — decision recorded below and applied to the registry + +This model family appeared in a live /models listing but matches no classification rule (include, exclude, -preview, gemma). drift-sync never silently classifies a new family. + +## Decision + + + + + +Decision: EXCLUDE (applied — excludeFamilies.gemini in +`src/__tests__/drift/model-registry.ts`, with the `excludeFamilies.gemini` +re-pin in `src/__tests__/drift/logic-pin.test.ts`). + +Rationale: wrong modality — speech->text, not text generation. Google's model +card for the Gemini 3.5 Transcribe line (GA 2026-08-26) gives this id the +signature input `Audio (up to 1 hour)` -> output `Text, Word annotations`, and +the release note describes it as "High-accuracy, low-latency non-streaming +speech-to-text". It emits no chat completion of its own: the text it returns is +a transcript of the caller's audio, with speaker diarization and word-level +timestamps. That is the audio canary's domain, not this text check. + +NOT rejected for the "transcribe" substring. The deciding evidence is the +provider-declared input/output signature, the same instrument used for +`gemini-3.7-flash-video-understanding-eap` (declared access status, not the +"video" in its name) and against which the native-audio misclassification was +caught. Mirrors the OpenAI `gpt-transcribe` / `gpt-live-transcribe` decisions +(PR #343). + +Membership in `excludeFamilies` is a CLASSIFICATION for the `/models` listing +check in `models.drift.ts` and nothing more: it says the family is accounted +for, not which endpoints aimock implements for it. diff --git a/drift-proposals/gemini-gemini-omni-1.1-flash-new-family.md b/drift-proposals/gemini-gemini-omni-1.1-flash-new-family.md new file mode 100644 index 00000000..a1ccc6da --- /dev/null +++ b/drift-proposals/gemini-gemini-omni-1.1-flash-new-family.md @@ -0,0 +1,44 @@ +# New / unclassified model family: gemini-omni-1.1-flash + +Provider: gemini +Detected: 2026-08-28 +Status: RESOLVED — decision recorded below and applied to the registry + +This model family appeared in a live /models listing but matches no classification rule (include, exclude, -preview, gemma). drift-sync never silently classifies a new family. + +## Decision + + + + + +Decision: EXCLUDE (applied — excludeFamilies.gemini in +`src/__tests__/drift/model-registry.ts`, with the `excludeFamilies.gemini` +re-pin in `src/__tests__/drift/logic-pin.test.ts`). + +Rationale: non-text generative media. Google's 2026-08-27 release note announces +`gemini-omni-1.1-flash` as "the GA version of our fast, conversational video +generation and editing model", and its model-card signature is input `Video with +audio` -> output `Video with audio` (scene extension to 40s, keyframe +interpolation, 360p-4K upscale). It produces no text turn at all, so it cannot +be text-generation drift. Same category as the already-excluded `veo-*`, +`lyria-*` and `imagen-*` entries. + +NOT classified from the "omni" substring — which would have argued the OPPOSITE. +"omni" names OpenAI's TEXT-capable omni line (`omni-moderation`, and the `gpt-4o` +"omni" lineage), so a name-shaped rule here would have read this as a +text-capable multimodal chat tier and INCLUDED it. The provider-declared +input/output signature is what decides it, exactly as with +`gemini-3.7-flash-video-understanding-eap`. + +Its `-preview` sibling `gemini-omni-flash-preview` is already auto-excluded by +the PREVIEW_FAMILY rule. This GA id carries no trailing `-preview` token, so +that rule cannot reach it and it must be enumerated (mirrors the +`-preview-tts` / `-preview-customtools` entries). diff --git a/src/__tests__/drift/logic-pin.test.ts b/src/__tests__/drift/logic-pin.test.ts index ff720476..ef86ec3c 100644 --- a/src/__tests__/drift/logic-pin.test.ts +++ b/src/__tests__/drift/logic-pin.test.ts @@ -441,7 +441,7 @@ const DATA_FROZEN: Record string[]; pin: string }> = { }, "excludeFamilies.gemini": { members: () => [...excludeFamilies.gemini].sort(), - pin: "3c8fb0b7cee776619467afd34407bde69e30b063ae263da4871096e7a7afca1d", + pin: "8afbca596165c15f956e94fe0a358adf2144c8f3c5f93b6599ddb81e0eb2ecfa", }, // The realtime canary's seed sets, previously pinned NOWHERE. An edit to // either one was invisible to every guard in the repo: adding a family to diff --git a/src/__tests__/drift/model-registry.ts b/src/__tests__/drift/model-registry.ts index f96c7a9f..851963dd 100644 --- a/src/__tests__/drift/model-registry.ts +++ b/src/__tests__/drift/model-registry.ts @@ -287,6 +287,55 @@ export const excludeFamilies: Record> = { "gemini-2.0-flash-thinking-exp", // Live/full-duplex voice — owned by the realtime canary, not this text check "gemini-live", + // The Gemini 3.5 Transcribe line, GA 2026-08-26, first observed by the daily + // /models canary on 2026-08-27 (that later date is the `Detected:` stamp in + // the notes under drift-proposals/ — two dates for two events, not a + // disagreement). Two ids ship as ONE model on TWO surfaces, and they are two + // distinct normalized families, so both are enumerated. + // + // Classified on DECLARED CAPABILITY, not on the "transcribe" substring. + // Google's model card gives both the same signature — input `Audio (up to + // 1 hour)`, output `Text, Word annotations` — i.e. speech->text only. They + // emit no chat completion of their own; the text they return is a + // transcript of the caller's audio. That is the realtime/audio canary's + // domain (voice-models.ts), exactly like `gemini-live` above, and it can + // never be TEXT-GENERATION drift. + // + // `-live` carries a second, independently observed capability fact. The Live + // leg's discovery filters SOLELY on declared `bidiGenerateContent` (no name + // heuristic — see ws-gemini-live.drift.ts), and it selected this id, so the + // model does declare that method. Google then refused the session out of + // band: `code=1007 reason="The requested combination of response modalities + // (AUDIO) is not supported by the model. models/gemini-3.5-transcribe-live"` + // (run 33296393200). So it is a bidi Live surface that emits TEXT and cannot + // emit AUDIO — a streaming transcriber, not a native-audio model. + // + // NOTE membership here is a CLASSIFICATION for the `/models` listing check + // in models.drift.ts and nothing more: it says the family is accounted for, + // not which endpoints aimock implements for it. Mirrors the OpenAI + // `gpt-transcribe` / `gpt-live-transcribe` pair above. + // + // Decision: EXCLUDE, recorded in + // drift-proposals/gemini-gemini-3.5-transcribe-new-family.md and + // drift-proposals/gemini-gemini-3.5-transcribe-live-new-family.md. + "gemini-3.5-transcribe", + "gemini-3.5-transcribe-live", + // Gemini Omni Flash, GA 2026-08-27. Non-text generative media, in the same + // category as the veo-* / lyria-* / imagen-* entries. + // + // Classified on DECLARED CAPABILITY, not on the "omni" substring — "omni" + // names OpenAI's TEXT-capable omni line, so the substring would have argued + // the opposite. Google's release note calls it "our fast, conversational + // video generation and editing model", and its model-card signature is + // input `Video with audio` -> output `Video with audio` (extension, + // keyframe interpolation, 360p-4K). It produces no text turn, so it cannot + // be text-generation drift. Its `-preview` sibling `gemini-omni-flash-preview` + // is already auto-excluded by PREVIEW_FAMILY; this GA id carries no + // `-preview` token, so that rule cannot reach it and it must be enumerated. + // + // Decision: EXCLUDE, recorded in + // drift-proposals/gemini-gemini-omni-1.1-flash-new-family.md. + "gemini-omni-1.1-flash", // Restricted EARLY-ACCESS (EAP) surface, not a GA tier. Google's own live // /models entry declares `displayName` AND `description` as // "[Confidential] Gemini 3.7 Flash Video Understanding EAP" — the only one diff --git a/src/__tests__/drift/models.drift.ts b/src/__tests__/drift/models.drift.ts index b03a13f9..648b4a50 100644 --- a/src/__tests__/drift/models.drift.ts +++ b/src/__tests__/drift/models.drift.ts @@ -402,6 +402,11 @@ describe("full live /models wave is fully classified (2026-07-16 drift)", () => // image / audio / tts / video / music / robotics / embeddings "gemini-2.5-flash-image", "gemini-2.5-flash-native-audio-latest", + // 2026-08-26/27 GA line (run 33296393200): speech-to-text pair + the + // conversational video-generation tier — all explicit excludes + "gemini-3.5-transcribe", + "gemini-3.5-transcribe-live", + "gemini-omni-1.1-flash", "gemini-2.5-flash-preview-tts", // explicit exclude (-preview-tts) "gemini-2.5-pro-preview-tts", // explicit exclude (-preview-tts) "gemini-3-pro-image", diff --git a/src/__tests__/drift/providers.ts b/src/__tests__/drift/providers.ts index 68100f53..d382e851 100644 --- a/src/__tests__/drift/providers.ts +++ b/src/__tests__/drift/providers.ts @@ -118,10 +118,23 @@ export interface LiveModelEntry { * model. */ export function selectLiveModel(models: LiveModelEntry[], preferred: string[] = []): string | null { - const liveIds = models + const liveIds = liveModelIds(models); + return preferred.find((p) => liveIds.includes(p)) ?? liveIds[0] ?? null; +} + +/** + * The usable (non-deprecated, non-empty) ids from a `/models` listing, in the + * provider's own order. + * + * Shared by {@link selectLiveModel} — which takes the first — and by + * {@link resolveLiveModelCandidates}, which keeps the whole ordered list for a + * leg that must try more than one because the LISTING alone cannot settle the + * capability it needs (see that function's note). + */ +function liveModelIds(models: LiveModelEntry[]): string[] { + return models .filter((m) => typeof m.id === "string" && m.id.length > 0 && m.deprecated !== true) .map((m) => m.id); - return preferred.find((p) => liveIds.includes(p)) ?? liveIds[0] ?? null; } /** @@ -172,9 +185,60 @@ export function resolveLiveModel( return promise; } -/** Test-only: clear the {@link resolveLiveModel} memo cache between cases. */ +/** + * Outcome of resolving the FULL ordered candidate list for a leg — the same + * three-way classification as {@link ResolvedModel}, carrying every usable id + * instead of just the first. + */ +export type ResolvedCandidates = { models: string[] } | { infra: number } | { unavailable: true }; + +/** Per-key memo for {@link resolveLiveModelCandidates}. */ +const resolvedCandidatesCache = new Map>(); + +/** + * Resolve EVERY usable model from a provider's listing, MEMOIZED per `key`, + * with the same infra/unavailable classification as {@link resolveLiveModel}. + * + * WHY A LIST. `resolveLiveModel` takes the first usable id, which is right + * whenever the listing filter already settles the capability the leg needs. It + * is NOT right when the listing cannot express that capability. Gemini Live is + * that case: `/models` declares `bidiGenerateContent` but says nothing about + * which RESPONSE MODALITY a model serves, and the two are genuinely + * independent — `gemini-3.5-transcribe-live` declares `bidiGenerateContent` and + * yet emits only TEXT, so a leg driving AUDIO must be able to move past it to + * the next candidate. The provider's own refusal is the only authority on that + * capability, so the leg learns it by asking (see `driveGeminiLiveAudio` in + * ws-providers.ts) — never by pattern-matching the model NAME. + */ +export function resolveLiveModelCandidates( + key: string, + fetchListing: () => Promise<{ status: number; models: LiveModelEntry[] }>, +): Promise { + const cached = resolvedCandidatesCache.get(key); + if (cached) return cached; + const promise = (async (): Promise => { + try { + const { status, models } = await fetchListing(); + if (isInfraSkip(status)) return { infra: status }; + if (status >= 400) return { unavailable: true }; + const ids = liveModelIds(models); + return ids.length > 0 ? { models: ids } : { unavailable: true }; + } catch (err) { + if (err instanceof InfraError) return { infra: err.status }; + throw err; + } + })(); + resolvedCandidatesCache.set(key, promise); + return promise; +} + +/** + * Test-only: clear the {@link resolveLiveModel} and + * {@link resolveLiveModelCandidates} memo caches between cases. + */ export function __resetResolveLiveModelCache(): void { resolvedModelCache.clear(); + resolvedCandidatesCache.clear(); } // --------------------------------------------------------------------------- diff --git a/src/__tests__/drift/text-drift.test.ts b/src/__tests__/drift/text-drift.test.ts index 7bb39e51..e91beaa0 100644 --- a/src/__tests__/drift/text-drift.test.ts +++ b/src/__tests__/drift/text-drift.test.ts @@ -127,3 +127,86 @@ describe("openai transcription line is classified as EXCLUDED (PR #343)", () => ).toEqual([]); }); }); + +// --------------------------------------------------------------------------- +// The 2026-08-26/27 Gemini transcription + omni-video line — BEHAVIOURAL +// coverage of the classification, in the same shape as the OpenAI block above. +// +// `gemini-3.5-transcribe`, `gemini-3.5-transcribe-live` and +// `gemini-omni-1.1-flash` were classified EXCLUDE in model-registry.ts on the +// provider's own DECLARED capabilities (see the rationale comment beside the +// entries, and drift-proposals/). Without the assertions below the only thing +// that would redden if an entry were dropped is the `excludeFamilies.gemini` +// membership CHECKSUM in logic-pin.test.ts — which says "the data moved" and +// nothing about what the classification MEANS. +// +// The `gemini-3.5-transcribe` / `gemini-3.5-transcribe-live` pair is asserted +// in BOTH directions on purpose: the first key is a strict PREFIX of the +// second, the same substring/prefix hazard the OpenAI `gpt-live` block exists +// for. They are DIFFERENT families and both must be classified on their own +// entry, not by one swallowing the other. +// --------------------------------------------------------------------------- + +describe("gemini transcription + omni-video line is classified as EXCLUDED", () => { + it("gemini-3.5-transcribe is EXCLUDED in a /models-shaped payload", () => { + expect(isClassifiedFamily("gemini-3.5-transcribe", "gemini")).toBe(true); + expect( + unclassifiedFamilies( + [ + "gemini-3.5-flash", // include, for a realistic mixed listing + "gemini-3.5-transcribe", + "gemini-3.5-transcribe-2026-08-26", // dated snapshot collapses onto the family + ], + "gemini", + ), + ).toEqual([]); + }); + + it("gemini-3.5-transcribe-live is EXCLUDED in a /models-shaped payload", () => { + expect(isClassifiedFamily("gemini-3.5-transcribe-live", "gemini")).toBe(true); + expect( + unclassifiedFamilies( + [ + "gemini-3.5-flash", + "gemini-3.5-transcribe-live", + "gemini-3.5-transcribe-live-2026-08-26", + "gemini-live", // the pre-existing full-duplex Live surface + ], + "gemini", + ), + ).toEqual([]); + }); + + it("gemini-omni-1.1-flash is EXCLUDED in a /models-shaped payload", () => { + expect(isClassifiedFamily("gemini-omni-1.1-flash", "gemini")).toBe(true); + expect( + unclassifiedFamilies( + [ + "gemini-3.5-flash", + "gemini-omni-1.1-flash", + "gemini-omni-1.1-flash-2026-08-27", + "gemini-omni-flash-preview", // sibling preview tier, excluded by pattern + ], + "gemini", + ), + ).toEqual([]); + }); + + it("neither transcribe key classifies the other, nor an unrelated extension", () => { + // The two entries are distinct families; a `startsWith`-shaped classification + // bug would let the shorter key classify the longer one (or vice versa) and + // silently swallow a family the canary exists to report. + expect(normalizeModelFamily("gemini-3.5-transcribe-live", "gemini")).toBe( + "gemini-3.5-transcribe-live", + ); + // NEGATIVE CONTROL: an id that merely EXTENDS an excluded key is still a new + // family and must be reported. Without this, `toEqual([])` above is also + // what a neutered `unclassifiedFamilies` would produce. + expect(unclassifiedFamilies(["gemini-3.5-transcribe-diarize"], "gemini")).toEqual([ + "gemini-3.5-transcribe-diarize", + ]); + expect(unclassifiedFamilies(["gemini-omni-1.1-pro"], "gemini")).toEqual([ + "gemini-omni-1.1-pro", + ]); + }); +}); diff --git a/src/__tests__/drift/ws-gemini-live-modality.test.ts b/src/__tests__/drift/ws-gemini-live-modality.test.ts index df8c3a95..5435063f 100644 --- a/src/__tests__/drift/ws-gemini-live-modality.test.ts +++ b/src/__tests__/drift/ws-gemini-live-modality.test.ts @@ -56,6 +56,8 @@ import { classifyGeminiMessage, GEMINI_LIVE_RESPONSE_MODALITIES, WSClosedError, + driveGeminiLiveAudio, + isModalityRefusal, } from "./ws-providers.js"; import { extractShape, compareSSESequences } from "./schema.js"; import { @@ -143,6 +145,12 @@ interface FakeProviderOptions { muteFromStart?: boolean; /** Send `setupComplete`, then ignore the client's turn entirely. */ muteAfterSetup?: boolean; + /** + * Models that serve TEXT and refuse AUDIO — the real + * `gemini-3.5-transcribe-live` shape. Any model NOT listed here behaves like a + * native-audio model: AUDIO only, TEXT refused. + */ + audioUnsupportedModels?: string[]; } /** The client-to-server messages the fake provider understands. */ @@ -228,10 +236,16 @@ function startFakeProvider(options: FakeProviderOptions = {}): Promise d.path).join(" ")).toContain("inlineData"); }); }); + +// --------------------------------------------------------------------------- +// The MIRROR-IMAGE refusal: a bidiGenerateContent model that refuses AUDIO +// +// THE BUG. Everything above rests on a premise this file used to state as fact: +// "every model exposing `bidiGenerateContent` is a native-audio model that +// supports only `AUDIO`". Google falsified it on 2026-08-26 by shipping +// `gemini-3.5-transcribe-live` — a streaming SPEECH-TO-TEXT model that declares +// `bidiGenerateContent` and emits only TEXT. The live leg's discovery filters +// on that declared method alone (correctly: the model NAME carries no +// capability signal), so the listing handed it to the probe, the probe asked +// for AUDIO, and the provider refused with the mirror image of the frame at the +// top of this file, recovered verbatim from the drift-report artifact of run +// 33296393200: +// +// code=1007 reason="The requested combination of response modalities (AUDIO) +// is not supported by the model. models/gemini-3.5-transcribe-live" +// +// The whole Gemini Live leg went red on that refusal every run from 2026-08-27. +// +// THE FIX is not a name filter and not a hardcoded pin — both are the mistakes +// this leg has already made. Response modality is a per-model capability the +// `/models` listing does not express, so the ONLY authority on it is the +// provider, and `driveGeminiLiveAudio` asks: it walks the listing's candidates +// in order and a 1007 modality refusal disqualifies that candidate and advances +// to the next. +// +// THE PROOF, against the same local TLS server, with the per-model rule enforced +// by the "provider" rather than by these expectations: +// +// RED — driving candidate[0] directly (the pre-fix behaviour) fails with +// exactly the 1007 frame above. +// GREEN — `driveGeminiLiveAudio` over the same ordered candidates reports that +// refusal, advances, and completes a real AUDIO turn on candidate[1]. +// --------------------------------------------------------------------------- + +/** The streaming transcriber the live listing now hands the probe first. */ +const TRANSCRIBE_LIVE_MODEL = "models/gemini-3.5-transcribe-live"; + +/** A native-audio model, later in the same listing — the one that serves AUDIO. */ +const NATIVE_AUDIO_MODEL = "models/gemini-2.5-flash-native-audio-latest"; + +/** + * The CLOSE reason Google sent, copied verbatim from the `drift-report` + * artifact of run 33296393200 rather than generated by this file's + * expectations. + */ +const OBSERVED_AUDIO_1007_REASON = + "The requested combination of response modalities (AUDIO) is not supported by the model. " + + TRANSCRIBE_LIVE_MODEL; + +describe("a bidiGenerateContent model that refuses AUDIO", () => { + /** The listing order the live run saw: the transcriber first. */ + const candidates = [TRANSCRIBE_LIVE_MODEL, NATIVE_AUDIO_MODEL]; + + it("RED: driving the first candidate directly is refused with the observed 1007", async () => { + const provider = await startFakeProvider({ + audioUnsupportedModels: [TRANSCRIBE_LIVE_MODEL], + }); + try { + const err = await geminiLiveWS( + { apiKey: "test-key" }, + AUDIO_PROMPT, + undefined, + TRANSCRIBE_LIVE_MODEL, + { host: "localhost", port: provider.port, ca: cert, waitMs: 2000 }, + ).then( + () => null, + (e: unknown) => e, + ); + + expect(err).toBeInstanceOf(WSClosedError); + const closed = err as WSClosedError; + expect(closed.code).toBe(1007); + // The bytes the live provider sent — not a paraphrase. + expect(closed.reason).toBe(OBSERVED_AUDIO_1007_REASON); + } finally { + provider.stop(); + } + }); + + it("GREEN: driveGeminiLiveAudio advances past the refusal and completes an AUDIO turn", async () => { + const provider = await startFakeProvider({ + audioUnsupportedModels: [TRANSCRIBE_LIVE_MODEL], + }); + try { + const drive = await driveGeminiLiveAudio( + { apiKey: "test-key" }, + AUDIO_PROMPT, + undefined, + candidates, + { host: "localhost", port: provider.port, ca: cert, waitMs: 2000 }, + ); + + expect(drive).not.toBeNull(); + // It moved to the model that actually serves AUDIO... + expect(drive!.model).toBe(NATIVE_AUDIO_MODEL); + // ...and it did not swallow the refusal — the provider's own words are + // carried out for the skip/diagnostic message. + expect(drive!.refused).toEqual([ + { model: TRANSCRIBE_LIVE_MODEL, reason: OBSERVED_AUDIO_1007_REASON }, + ]); + + // A REAL audio turn, not merely "no throw": the leg grades on these counts. + const turn = summarizeGeminiLiveTurn(drive!.result.rawMessages); + expect(turn.setupCompleteCount).toBe(1); + expect(turn.audioPartCount).toBeGreaterThan(0); + expect(turn.textPartCount).toBe(0); + expect(turn.turnCompleteCount).toBe(1); + + // The provider saw BOTH setups, in listing order — the advance happened on + // the wire, not by this test agreeing with itself. + const setupModels = provider.received + .filter((m) => m.setup) + .map((m) => m.setup!.model) + .filter((m): m is string => typeof m === "string"); + expect(setupModels).toEqual(candidates); + } finally { + provider.stop(); + } + }); + + it("returns null — an honest skip — when EVERY candidate refuses AUDIO", async () => { + // The listing exposes only transcribers. That is unavailability, not drift: + // there is no model this leg can grade an AUDIO turn against. + const provider = await startFakeProvider({ audioUnsupportedModels: candidates }); + try { + const drive = await driveGeminiLiveAudio( + { apiKey: "test-key" }, + AUDIO_PROMPT, + undefined, + candidates, + { host: "localhost", port: provider.port, ca: cert, waitMs: 2000 }, + ); + expect(drive).toBeNull(); + } finally { + provider.stop(); + } + }); + + it("does NOT swallow a non-modality failure — a mute provider still fails", async () => { + // NEGATIVE CONTROL for the catch above. `driveGeminiLiveAudio` recovers from + // exactly one condition; a helper that walked candidates on ANY error would + // turn every real live failure into a silent `null` skip and hollow out the + // leg. A mute provider is the failure this file already reproduces. + const provider = await startFakeProvider({ muteAfterSetup: true }); + try { + await expect( + driveGeminiLiveAudio({ apiKey: "test-key" }, AUDIO_PROMPT, undefined, candidates, { + host: "localhost", + port: provider.port, + ca: cert, + waitMs: 1000, + }), + ).rejects.toThrow(/turn model=/); + } finally { + provider.stop(); + } + }); + + it("isModalityRefusal keys on the refusal sentence, never on the model name", () => { + // Modality-agnostic by construction: the TEXT refusal at the top of this + // file and the AUDIO refusal above are the same sentence, so one predicate + // must recognize both. + expect(isModalityRefusal(new WSClosedError("x", 1007, OBSERVED_1007_REASON))).toBe(true); + expect(isModalityRefusal(new WSClosedError("x", 1007, OBSERVED_AUDIO_1007_REASON))).toBe(true); + // A 1007 that is NOT a modality refusal is a real protocol error and must + // propagate — the fake provider sends exactly this one for malformed JSON. + expect(isModalityRefusal(new WSClosedError("x", 1007, "Malformed JSON"))).toBe(false); + // Right sentence, wrong code — still not a modality refusal. + expect(isModalityRefusal(new WSClosedError("x", 1011, OBSERVED_AUDIO_1007_REASON))).toBe(false); + expect(isModalityRefusal(new Error(OBSERVED_AUDIO_1007_REASON))).toBe(false); + }); +}); diff --git a/src/__tests__/drift/ws-gemini-live.drift.ts b/src/__tests__/drift/ws-gemini-live.drift.ts index a4ad7236..f4ee20cf 100644 --- a/src/__tests__/drift/ws-gemini-live.drift.ts +++ b/src/__tests__/drift/ws-gemini-live.drift.ts @@ -4,38 +4,57 @@ * Three-way comparison: SDK types × real API (WS) × aimock output (WS). * * MODALITY. The Live API permits exactly ONE response modality per session, and - * every model exposing `bidiGenerateContent` is a native-audio model that - * supports only `AUDIO` — Google's own capabilities guide states the native - * audio models "only support `AUDIO` response modality". A session requesting - * `TEXT` is refused out-of-band with an RFC 6455 CLOSE frame, observed verbatim - * from the live endpoint as + * WHICH one a model serves is a PER-MODEL capability. A native-audio model + * serves only `AUDIO` — Google's capabilities guide states the native audio + * models "only support `AUDIO` response modality" — and refuses `TEXT` + * out-of-band with an RFC 6455 CLOSE frame, observed verbatim from the live + * endpoint as * * code=1007 reason="The requested combination of response modalities (TEXT) * is not supported by the model. models/gemini-3.1-flash-live-preview" * - * so this leg drives the AUDIO shape — the shape the model actually emits — + * so this leg drives the AUDIO shape — the shape those models actually emit — * and grades the audio event sequence (`inlineData` parts + `turnComplete`) * plus the modality-independent `toolCall`. The TEXT shape is NOT expressible - * against any live Live model; aimock's TEXT path stays covered mock-only by + * against a native-audio model; aimock's TEXT path stays covered mock-only by * ws-gemini-live.test.ts, without live triangulation. * + * `bidiGenerateContent` DOES NOT IMPLY AUDIO, though — this header used to say + * it did. Google shipped `gemini-3.5-transcribe-live` on 2026-08-26: a streaming + * speech-to-text model that declares `bidiGenerateContent` and emits only TEXT. + * The listing offered it first, this leg asked for AUDIO, and the provider sent + * the mirror-image refusal + * + * code=1007 reason="The requested combination of response modalities (AUDIO) + * is not supported by the model. models/gemini-3.5-transcribe-live" + * + * reddening the leg on every run from 2026-08-27 (run 33296393200). + * * MODEL SELECTION is SELF-HEALING rather than a hardcoded pin plus a * human-maintained `.skip` cycle: `fetchLiveCapableModels` + providers.ts's - * shared `resolveLiveModel` query the live model listing on every run and - * resolve to exactly one of: - * - `{ model }` — a `bidiGenerateContent` model exists; the drift tests - * below run for real against it. + * shared `resolveLiveModelCandidates` query the live model listing on every run + * and resolve to exactly one of: + * - `{ models }` — the ordered `bidiGenerateContent` candidates; the + * drift tests below run for real against the first that + * actually serves AUDIO. * - `{ unavailable }` — the listing exposes no Live-capable model at all — * an HONEST SKIP, not a failure. * - `{ infra }` — the listing hit an auth/rate-limit/5xx condition — * also an honest skip (never a hard-fail). * - * Selection keys on the listing's declared `bidiGenerateContent` support ONLY. - * It must never re-derive a capability from the model NAME: a `"native-audio"` - * name-substring filter silently mis-classified `gemini-3.1-flash-live-preview` - * — a native-audio model whose name lacks that substring — as text-capable, and - * the probe then requested TEXT on it. Google names models freely; the listing - * is the only authority. + * Selection keys on the listing's declared `bidiGenerateContent` support ONLY, + * and never re-derives a capability from the model NAME. That rule has been + * earned twice: a `"native-audio"` name-substring filter silently + * mis-classified `gemini-3.1-flash-live-preview` — a native-audio model whose + * name lacks that substring — as text-capable, and a `"transcribe"` filter + * would be the same mistake wearing the opposite sign. Google names models + * freely; only the provider is an authority on capability. + * + * Since the LISTING cannot express response modality, the leg gets that one + * capability from the provider by ASKING: `driveGeminiLiveAudio` walks the + * candidates in listing order and a 1007 modality refusal disqualifies that + * candidate and advances to the next. A listing whose Live models ALL refuse + * AUDIO is an honest skip — there is no AUDIO turn to grade — not drift. * * A WS handshake-level infra status (401/403/429/5xx) is likewise an honest * skip via `WSHandshakeError` + `isInfraSkip` (ws-providers.ts, shared with @@ -52,13 +71,15 @@ import { geminiLiveToolCallEventShapes, } from "./sdk-shapes.js"; import { - geminiLiveWS, + driveGeminiLiveAudio, summarizeGeminiLiveTurn, GEMINI_LIVE_RESPONSE_MODALITIES, WSHandshakeError, + type WSResult, } from "./ws-providers.js"; import { resolveLiveModel, + resolveLiveModelCandidates, isInfraSkip, __resetResolveLiveModelCache, type LiveModelEntry, @@ -100,9 +121,13 @@ afterAll(async () => { * That declared method is the ONLY selection criterion. No model-name heuristic * is applied: names carry no reliable capability signal (see this file's header * — a `"native-audio"` substring filter mis-classified a native-audio model as - * text-capable because Google left that substring out of its name), and since - * the leg drives the universally-supported AUDIO modality it needs no capability - * inference beyond "speaks the Live protocol". + * text-capable because Google left that substring out of its name). + * + * The filter yields "speaks the Live protocol" and nothing more. It notably does + * NOT yield "serves AUDIO": the listing exposes no response-modality field, and + * `gemini-3.5-transcribe-live` declares this method while emitting only TEXT. + * That remaining capability is resolved from the provider's own refusal, in + * `driveGeminiLiveAudio` — never from the id. * * A non-2xx status is surfaced so `resolveLiveModel`/`isInfraSkip` can classify * an auth/rate-limit/5xx listing failure as an honest skip rather than this @@ -124,11 +149,84 @@ async function fetchLiveCapableModels( } /** - * Resolve the live Live-capable model, memoized per the leg's cache key so all + * Resolve EVERY Live-capable candidate, memoized per the leg's cache key so all * tests in this file make exactly one listing call. + * + * A LIST, not one id, because `bidiGenerateContent` does not settle the + * RESPONSE MODALITY this leg needs and the listing expresses no other signal — + * see `driveGeminiLiveAudio` in ws-providers.ts, and the header above. */ -function getLiveCapableModel(apiKey: string) { - return resolveLiveModel("gemini-live", () => fetchLiveCapableModels(apiKey)); +function getLiveCapableModels(apiKey: string) { + return resolveLiveModelCandidates("gemini-live", () => fetchLiveCapableModels(apiKey)); +} + +/** A resolved real AUDIO turn, or `null` when the leg must skip honestly. */ +type RealAudioTurn = { model: string; realResult: WSResult } | null; + +/** + * Drive the real AUDIO turn both tests below compare against, converting every + * honest-skip condition into `null` (with the warning that carries the signal) + * and letting every genuine failure propagate. + * + * Shared so the two legs cannot drift apart on WHICH conditions are a skip — + * they are: an infra listing status, a listing with no Live-capable model, a WS + * handshake infra status, and a listing whose Live-capable models ALL refuse + * AUDIO. That last one is new: `gemini-3.5-transcribe-live` declares + * `bidiGenerateContent` and serves only TEXT, so "the listing has a Live model" + * no longer implies "this leg has something it can grade". + */ +async function driveRealAudioTurn( + config: { apiKey: string }, + ctx: { skip: () => void }, + prompt: string, + tools?: object[], +): Promise { + const resolved = await getLiveCapableModels(config.apiKey); + if ("infra" in resolved) { + console.warn(`[gemini-live drift] listing infra status ${resolved.infra} — skipping`); + ctx.skip(); + return null; + } + if ("unavailable" in resolved) { + console.warn("[gemini-live drift] listing exposes no bidiGenerateContent model — skipping"); + ctx.skip(); + return null; + } + + // A WS handshake-level infra status (auth/rate-limit/5xx) is an HONEST SKIP, + // never a hard-fail that would quarantine the shared drift baseline (mirrors + // ws-responses.drift.ts's handling of the same error type). + let drive; + try { + drive = await driveGeminiLiveAudio(config, prompt, tools, resolved.models); + } catch (err) { + if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { + console.warn(`[gemini-live drift] WS handshake infra status ${err.status} — skipping`); + ctx.skip(); + return null; + } + throw err; + } + + if (!drive) { + // Every Live-capable model in the listing is a TEXT-only surface (a + // streaming transcriber). There is no AUDIO turn to grade — unavailability, + // not drift. The provider's own words are logged so the cause is legible. + console.warn( + "[gemini-live drift] every bidiGenerateContent model refused AUDIO — skipping: " + + resolved.models.join(", "), + ); + ctx.skip(); + return null; + } + + // Candidates the provider disqualified are NOT silent: they are the signal + // that the listing's shape moved under this leg. + for (const { model, reason } of drive.refused) { + console.warn(`[gemini-live drift] ${model} refused AUDIO (${reason}) — advanced past it`); + } + + return { model: drive.model, realResult: drive.result }; } describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { @@ -144,36 +242,12 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { // signal, and the assertions that remain can all actually go red. it("WS audio event sequence and shapes match", async (ctx) => { - const resolved = await getLiveCapableModel(config.apiKey); - if ("infra" in resolved) { - console.warn(`[gemini-live drift] listing infra status ${resolved.infra} — skipping`); - ctx.skip(); - return; - } - if ("unavailable" in resolved) { - console.warn("[gemini-live drift] listing exposes no bidiGenerateContent model — skipping"); - ctx.skip(); - return; - } - const model = `models/${resolved.model}`; + const turn = await driveRealAudioTurn(config, ctx, AUDIO_PROMPT); + if (!turn) return; + const { model, realResult } = turn; const sdkEvents = [geminiLiveSetupCompleteShape(), ...geminiLiveAudioEventShapes()]; - // Real API — a WS handshake-level infra status (auth/rate-limit/5xx) is an - // HONEST SKIP, never a hard-fail that would quarantine the shared drift - // baseline (mirrors ws-responses.drift.ts's handling of the same error type). - let realResult; - try { - realResult = await geminiLiveWS(config, AUDIO_PROMPT, undefined, model); - } catch (err) { - if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { - console.warn(`[gemini-live drift] WS handshake infra status ${err.status} — skipping`); - ctx.skip(); - return; - } - throw err; - } - // Mock — replicate Gemini Live protocol const mockWs = await connectWebSocket(instance.url, GEMINI_WS_PATH); @@ -263,19 +337,6 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { }); it("WS tool call event sequence matches", async (ctx) => { - const resolved = await getLiveCapableModel(config.apiKey); - if ("infra" in resolved) { - console.warn(`[gemini-live drift] listing infra status ${resolved.infra} — skipping`); - ctx.skip(); - return; - } - if ("unavailable" in resolved) { - console.warn("[gemini-live drift] listing exposes no bidiGenerateContent model — skipping"); - ctx.skip(); - return; - } - const model = `models/${resolved.model}`; - const sdkEvents = [geminiLiveSetupCompleteShape(), ...geminiLiveToolCallEventShapes()]; const tools = [ @@ -294,19 +355,11 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { }, ]; - // Real API — WS handshake-level infra status → honest skip (see the - // first test above for the full rationale). - let realResult; - try { - realResult = await geminiLiveWS(config, "Weather in Paris", tools, model); - } catch (err) { - if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { - console.warn(`[gemini-live drift] WS handshake infra status ${err.status} — skipping`); - ctx.skip(); - return; - } - throw err; - } + // Real API — every honest-skip condition (including a listing whose Live + // models all refuse AUDIO) is handled by the shared driver above. + const turn = await driveRealAudioTurn(config, ctx, "Weather in Paris", tools); + if (!turn) return; + const { model, realResult } = turn; // Mock — replicate Gemini Live protocol with tools const mockWs = await connectWebSocket(instance.url, GEMINI_WS_PATH); diff --git a/src/__tests__/drift/ws-providers.ts b/src/__tests__/drift/ws-providers.ts index 3d634289..8394ac52 100644 --- a/src/__tests__/drift/ws-providers.ts +++ b/src/__tests__/drift/ws-providers.ts @@ -21,7 +21,7 @@ interface ProviderConfig { apiKey: string; } -interface WSResult { +export interface WSResult { events: SSEEventShape[]; rawMessages: unknown[]; } @@ -778,19 +778,32 @@ export async function openaiRealtimeWS( // --------------------------------------------------------------------------- /** - * The ONE response modality every Live (`bidiGenerateContent`) model supports. + * The response modality this leg drives: the one a native-audio Live model + * actually emits. * - * Google's Live API permits exactly one modality per session, and every model - * currently exposing `bidiGenerateContent` is a native-audio model that accepts - * only `AUDIO`. Requesting `TEXT` is refused out-of-band with an RFC 6455 CLOSE - * frame — observed verbatim from the live endpoint as + * Google's Live API permits exactly ONE modality per session, and a + * native-audio model accepts only `AUDIO`. Requesting `TEXT` on one is refused + * out-of-band with an RFC 6455 CLOSE frame — observed verbatim from the live + * endpoint as * * code=1007 reason="The requested combination of response modalities (TEXT) * is not supported by the model. models/gemini-3.1-flash-live-preview" * - * so this leg drives the AUDIO shape, which is the shape the model actually - * emits. See ws-gemini-live-modality.test.ts for the local reproduction of that - * refusal and the red/green pair over this function. + * so this leg drives the AUDIO shape, which is the shape those models emit. + * + * `bidiGenerateContent` does NOT imply AUDIO, though, and the original form of + * this comment claimed it did. Google shipped `gemini-3.5-transcribe-live` on + * 2026-08-26: a streaming SPEECH-TO-TEXT model that declares + * `bidiGenerateContent` and emits only TEXT. It refused the mirror-image + * session with the mirror-image frame + * + * code=1007 reason="The requested combination of response modalities (AUDIO) + * is not supported by the model. models/gemini-3.5-transcribe-live" + * + * (drift run 33296393200). Modality is therefore a PER-MODEL capability the + * listing does not express, which is what {@link driveGeminiLiveAudio} exists + * to resolve. See ws-gemini-live-modality.test.ts for the local reproduction of + * both refusals and the red/green pairs over this function. */ export const GEMINI_LIVE_RESPONSE_MODALITIES = ["AUDIO"]; @@ -862,6 +875,83 @@ export function summarizeGeminiLiveTurn(messages: unknown[]): { return { setupCompleteCount, audioPartCount, textPartCount, turnCompleteCount, toolCallCount }; } +/** + * True when a WS failure is the provider REFUSING the requested response + * modality for that model — RFC 6455 code 1007 with Google's verbatim reason. + * + * This is a CAPABILITY answer, not an error: the provider is stating that this + * model does not serve the modality asked for, which is exactly the fact the + * `/models` listing omits. {@link driveGeminiLiveAudio} treats it as + * "disqualify this candidate and try the next", never as drift. + * + * Matched on the reason PREFIX Google emits, which is modality-agnostic — the + * TEXT refusal (2026-08, native-audio models) and the AUDIO refusal (2026-08-30, + * `gemini-3.5-transcribe-live`) are the same sentence with the modality + * substituted. Deliberately NOT matched on the model name in the reason: the + * name is the one part of that string that carries no capability signal. + */ +export function isModalityRefusal(err: unknown): err is WSClosedError { + return ( + err instanceof WSClosedError && + err.code === 1007 && + /requested combination of response modalities/i.test(err.reason) + ); +} + +/** A candidate the provider disqualified, with the reason it gave. */ +export interface RefusedGeminiLiveCandidate { + model: string; + reason: string; +} + +/** Outcome of {@link driveGeminiLiveAudio}. */ +export interface GeminiLiveAudioDrive { + /** The model that actually served the AUDIO turn (fully qualified). */ + model: string; + result: WSResult; + /** Candidates the provider refused AUDIO on, in the order it refused them. */ + refused: RefusedGeminiLiveCandidate[]; +} + +/** + * Drive an AUDIO Live turn against the first candidate that ACTUALLY SERVES + * AUDIO, learning that capability from the provider rather than from the id. + * + * `candidates` is the ordered list of `bidiGenerateContent` models the live + * listing exposed (see `resolveLiveModelCandidates`). Declaring that method is + * necessary but not sufficient: a streaming transcriber declares it and emits + * only TEXT. So each candidate is tried in listing order and a + * {@link isModalityRefusal} close advances to the next — the provider's own + * refusal is the authority, and no capability is ever re-derived from the model + * NAME (the mis-classification this leg has already been bitten by twice). + * + * Returns `null` when EVERY candidate refused AUDIO. That is an honest + * unavailability — the listing exposes no model this leg can grade — and the + * caller skips on it rather than reporting drift, matching how it treats an + * empty listing. Every other failure (handshake infra status, a mute provider, + * a timeout) propagates UNCHANGED so this helper can never mask one. + */ +export async function driveGeminiLiveAudio( + config: ProviderConfig, + text: string, + tools: object[] | undefined, + candidates: string[], + transport?: GeminiLiveTransportOptions, +): Promise { + const refused: RefusedGeminiLiveCandidate[] = []; + for (const candidate of candidates) { + const model = candidate.startsWith("models/") ? candidate : `models/${candidate}`; + try { + const result = await geminiLiveWS(config, text, tools, model, transport); + return { model, result, refused }; + } catch (err) { + if (!isModalityRefusal(err)) throw err; + refused.push({ model, reason: err.reason }); + } + } + return null; +} + export async function geminiLiveWS( config: ProviderConfig, text: string,