diff --git a/.changeset/skills-telemetry-stamp.md b/.changeset/skills-telemetry-stamp.md new file mode 100644 index 000000000..ccd936b0b --- /dev/null +++ b/.changeset/skills-telemetry-stamp.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Record which agent skills `clerk init` installed, and whether the install succeeded, was declined, or found no package runner. The telemetry disclosure notice now names this alongside the fields it already listed. diff --git a/packages/cli-core/src/commands/init/skills.test.ts b/packages/cli-core/src/commands/init/skills.test.ts index a04796491..70af8b0e3 100644 --- a/packages/cli-core/src/commands/init/skills.test.ts +++ b/packages/cli-core/src/commands/init/skills.test.ts @@ -1,5 +1,21 @@ -import { test, expect, describe } from "bun:test"; -import { formatSkillsPromptMessage, resolveUpstreamSkills } from "./skills.ts"; +import { test, expect, describe, mock, spyOn, beforeEach, afterAll } from "bun:test"; +import { setMode } from "../../mode.ts"; +import * as telemetryMod from "../../lib/telemetry.ts"; + +// Stub the layer that would shell out to `bunx skills add`, so these tests +// exercise installSkills' branching without spawning a subprocess. +let runnerStub: () => unknown = () => ({ id: "bunx", display: "bunx" }); +let addSucceeds = true; +mock.module("../../lib/skills.ts", () => ({ + resolveSkillsRunner: async () => runnerStub(), + runSkillsAdd: async () => addSucceeds, +})); + +let confirmAnswer = true; +mock.module("../../lib/prompts.ts", () => ({ confirm: async () => confirmAnswer })); + +const { formatSkillsPromptMessage, resolveUpstreamSkills, installSkills } = + await import("./skills.ts"); const DEFAULTS = [ "clerk-cli", @@ -60,3 +76,46 @@ describe("formatSkillsPromptMessage", () => { ); }); }); + +describe("installSkills telemetry", () => { + const recorded = spyOn(telemetryMod, "setTelemetrySkills"); + + beforeEach(() => { + recorded.mockClear(); + runnerStub = () => ({ id: "bunx", display: "bunx" }); + addSucceeds = true; + confirmAnswer = true; + setMode("agent"); + }); + + afterAll(() => recorded.mockRestore()); + + test("records the resolved skill list when the install succeeds", async () => { + await installSkills("/tmp/proj", "next", "bun", true); + expect(recorded).toHaveBeenCalledWith([...resolveUpstreamSkills("next")], "installed"); + }); + + test("records a failed install rather than staying silent", async () => { + addSucceeds = false; + await installSkills("/tmp/proj", undefined, "bun", true); + expect(recorded.mock.calls[0]?.[1]).toBe("failed"); + }); + + // No runner on PATH is a different story from a user saying no, and the + // warehouse could not tell them apart before this. + test("records runner_missing when no package runner is available", async () => { + runnerStub = () => undefined; + await installSkills("/tmp/proj", undefined, "bun", true); + expect(recorded.mock.calls[0]?.[1]).toBe("runner_missing"); + }); + + test("records a decline, and never reaches the runner", async () => { + setMode("human"); + confirmAnswer = false; + runnerStub = () => { + throw new Error("runner must not be probed after a decline"); + }; + await installSkills("/tmp/proj", undefined, "bun", false); + expect(recorded.mock.calls[0]?.[1]).toBe("declined"); + }); +}); diff --git a/packages/cli-core/src/commands/init/skills.ts b/packages/cli-core/src/commands/init/skills.ts index 78723fb7c..2a11e88fa 100644 --- a/packages/cli-core/src/commands/init/skills.ts +++ b/packages/cli-core/src/commands/init/skills.ts @@ -16,6 +16,7 @@ import { isHuman } from "../../mode.js"; import { log } from "../../lib/log.js"; import { confirm } from "../../lib/prompts.js"; import { resolveSkillsRunner, runSkillsAdd } from "../../lib/skills.js"; +import { setTelemetrySkills } from "../../lib/telemetry.js"; import type { ProjectContext } from "./frameworks/types.js"; /** Upstream skills from clerk/skills — installed on every project. */ @@ -94,14 +95,20 @@ export async function installSkills( message: formatSkillsPromptMessage(frameworkSkills), default: true, }); - if (!install) return; + if (!install) { + setTelemetrySkills(upstreamSkills, "declined"); + return; + } } const interactive = isHuman() && !skipPrompt; // Detect runner after the user accepts — no point probing PATH if they decline. const runner = await resolveSkillsRunner(packageManager, interactive); - if (!runner) return; + if (!runner) { + setTelemetrySkills(upstreamSkills, "runner_missing"); + return; + } log.debug(`skills: upstream install — ${upstreamSkills.join(", ")}`); const upstreamOk = await runSkillsAdd( @@ -113,6 +120,8 @@ export async function installSkills( formatSkillsSummary(frameworkSkills), ); + setTelemetrySkills(upstreamSkills, upstreamOk ? "installed" : "failed"); + if (upstreamOk) { log.blank(); log.success("Agent skills installed. AI agents now have Clerk context in this project."); diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 283b9eaa7..035041f4d 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -6,6 +6,7 @@ import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from ". import { finalizeAndSendTelemetry, getTelemetryStatus, + setTelemetrySkills, setTelemetryStage, startCommandTelemetry, telemetryEnabled, @@ -460,31 +461,31 @@ describe("finalizeAndSendTelemetry", () => { }); }); - describe("stage", () => { - /** Captures the payload of the single event a finalize call sends. */ - async function sendAndCapturePayload( - run: () => void | Promise, - result: TelemetryResult, - ): Promise> { - await markTelemetryNoticeShown(); // past the grace run — reach the send path - process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; - let sent: string | undefined; - globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { - sent = init.body; - return new Response("{}"); - }) as unknown as typeof fetch; + /** Captures the payload of the single event a finalize call sends. */ + async function sendAndCapturePayload( + run: () => void | Promise, + result: TelemetryResult, + ): Promise> { + await markTelemetryNoticeShown(); // past the grace run — reach the send path + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + let sent: string | undefined; + globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { + sent = init.body; + return new Response("{}"); + }) as unknown as typeof fetch; - startCommandTelemetry(fakeCommand()); - await run(); - await finalizeAndSendTelemetry(result); - - expect(sent).toBeDefined(); - const parsed = JSON.parse(sent as string) as { - events: { payload: Record }[]; - }; - return parsed.events[0]!.payload; - } + startCommandTelemetry(fakeCommand()); + await run(); + await finalizeAndSendTelemetry(result); + + expect(sent).toBeDefined(); + const parsed = JSON.parse(sent as string) as { + events: { payload: Record }[]; + }; + return parsed.events[0]!.payload; + } + describe("stage", () => { test("reports the furthest stage reached on success", async () => { const payload = await sendAndCapturePayload( () => { @@ -529,4 +530,41 @@ describe("finalizeAndSendTelemetry", () => { expect(() => setTelemetryStage("flags")).not.toThrow(); }); }); + + describe("skills", () => { + test("reports the skills an install actually landed", async () => { + const payload = await sendAndCapturePayload( + () => setTelemetrySkills(["clerk-cli", "clerk-orgs"], "installed"), + { outcome: "success", exitCode: 0 }, + ); + expect(payload.skills).toBe("clerk-cli,clerk-orgs"); + expect(payload.skills_outcome).toBe("installed"); + }); + + // `skills` must mean "this project has them", not "it was offered them", + // or a `skills LIKE '%clerk-orgs%'` filter silently counts declines. + test.each([["declined"], ["runner_missing"], ["failed"]] as const)( + "leaves skills empty when the install ended as %s", + async (outcome) => { + const payload = await sendAndCapturePayload( + () => setTelemetrySkills(["clerk-cli", "clerk-orgs"], outcome), + { outcome: "success", exitCode: 0 }, + ); + expect(payload.skills).toBe(""); + expect(payload.skills_outcome).toBe(outcome); + }, + ); + + // A command that never reaches the skills step is distinguishable from one + // whose user declined — that difference is the whole point of the field. + test("outcome is null when the command never reaches the skills step", async () => { + const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); + expect(payload.skills).toBe(""); + expect(payload.skills_outcome).toBeNull(); + }); + + test("recording skills with no active context is a no-op", () => { + expect(() => setTelemetrySkills(["clerk-orgs"], "installed")).not.toThrow(); + }); + }); }); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 5880a435d..241de2f38 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -73,6 +73,13 @@ export type TelemetryStage = // shared terminal marker | "done"; +/** + * How the optional agent-skills install at the end of `clerk init` ended. + * A closed union for the same reason as [TelemetryStage]: a renamed call site + * fails to compile rather than quietly splitting the funnel. + */ +export type TelemetrySkillsOutcome = "installed" | "declined" | "runner_missing" | "failed"; + /** Structural slice of Commander's Command — avoids its generic types. */ export type TelemetryCommand = { name(): string; @@ -87,6 +94,9 @@ type TelemetryContext = { startedAt: number; /** Last stage set — see setTelemetryStage. */ stage: TelemetryStage | null; + /** Skills actually installed, comma-joined — see setTelemetrySkills. */ + skills: string; + skillsOutcome: TelemetrySkillsOutcome | null; }; let context: TelemetryContext | null = null; @@ -186,6 +196,8 @@ export function startCommandTelemetry(actionCommand: TelemetryCommand): void { flags: collectSetFlagNames(actionCommand).join(","), startedAt: Date.now(), stage: null, + skills: "", + skillsOutcome: null, }; } catch (error) { log.debug(`telemetry: failed to start context: ${error}`); @@ -208,6 +220,26 @@ export function currentTelemetryStage(): TelemetryStage | null { return context?.stage ?? null; } +/** + * Record the agent skills `clerk init` installed, and how the attempt ended. + * + * Only a successful install populates `skills`, so a warehouse filter like + * `skills LIKE '%clerk-orgs%'` means "this project has that skill" rather than + * "it was offered one". The names come from the CLI's own constants, never + * from user input, so nothing unbounded reaches the payload. + * + * `skillsOutcome` is what separates "the user said no" from "init never got + * that far" — before this, both looked identical from the warehouse. + */ +export function setTelemetrySkills( + names: readonly string[], + outcome: TelemetrySkillsOutcome, +): void { + if (!context) return; + context.skills = outcome === "installed" ? names.join(",") : ""; + context.skillsOutcome = outcome; +} + export function telemetryResultForError(error: unknown): TelemetryResult { if (error instanceof UserAbortError) { return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS }; @@ -297,6 +329,8 @@ async function buildAndSend( exit_code: result.exitCode, error_code: result.errorCode ?? null, stage: current.stage, + skills: current.skills, + skills_outcome: current.skillsOutcome, duration_ms: Date.now() - current.startedAt, machine_uuid: machineUuid, install_method: detectInstallMethod(process.env, process.execPath), @@ -348,9 +382,11 @@ async function maybeShowTelemetryNotice(): Promise { "The Clerk CLI collects usage telemetry to help improve the CLI: command name, flag names,", ); log.info( - "duration, outcome, the step a multi-step command reached, a random machine identifier —", + "duration, outcome, the step a multi-step command reached, the agent skills init installed,", + ); + log.info( + "a random machine identifier — and your workspace and app IDs when a project is linked.", ); - log.info("and your workspace and app IDs when a project is linked."); log.info("Nothing has been sent during this run."); log.info("Opt out: `clerk telemetry disable` — details: https://clerk.com/docs/telemetry"); log.blank();