From 4e7ec344a87608b58ca715836bc86e0510af1fcb Mon Sep 17 00:00:00 2001 From: gabrielmeloc22 Date: Tue, 8 Sep 2026 18:25:53 -0300 Subject: [PATCH] feat(telemetry): record which agent skills init installed There is currently no way to tell whether a project has a given Clerk skill, so the return on any skill change is unmeasurable: every run looks the same whether the developer installed the skills, declined them, or had no package runner on PATH. The `skills` stage only proved init reached the step. Adds a `skills` field carrying the installed names and a `skills_outcome` union covering all four exits. Only a successful install populates `skills`, so filtering on a skill name means the project has it rather than that it was offered one. Names come from the CLI's own constants, so nothing unbounded reaches the payload. The disclosure notice names the new field. --- .changeset/skills-telemetry-stamp.md | 5 ++ .../cli-core/src/commands/init/skills.test.ts | 63 +++++++++++++- packages/cli-core/src/commands/init/skills.ts | 13 ++- packages/cli-core/src/lib/telemetry.test.ts | 84 ++++++++++++++----- packages/cli-core/src/lib/telemetry.ts | 40 ++++++++- 5 files changed, 176 insertions(+), 29 deletions(-) create mode 100644 .changeset/skills-telemetry-stamp.md diff --git a/.changeset/skills-telemetry-stamp.md b/.changeset/skills-telemetry-stamp.md new file mode 100644 index 000000000..ccd936b0b --- /dev/null +++ b/.changeset/skills-telemetry-stamp.md @@ -0,0 +1,5 @@ +--- +"clerk": minor +--- + +Record which agent skills `clerk init` installed, and whether the install succeeded, was declined, or found no package runner. The telemetry disclosure notice now names this alongside the fields it already listed. diff --git a/packages/cli-core/src/commands/init/skills.test.ts b/packages/cli-core/src/commands/init/skills.test.ts index a04796491..70af8b0e3 100644 --- a/packages/cli-core/src/commands/init/skills.test.ts +++ b/packages/cli-core/src/commands/init/skills.test.ts @@ -1,5 +1,21 @@ -import { test, expect, describe } from "bun:test"; -import { formatSkillsPromptMessage, resolveUpstreamSkills } from "./skills.ts"; +import { test, expect, describe, mock, spyOn, beforeEach, afterAll } from "bun:test"; +import { setMode } from "../../mode.ts"; +import * as telemetryMod from "../../lib/telemetry.ts"; + +// Stub the layer that would shell out to `bunx skills add`, so these tests +// exercise installSkills' branching without spawning a subprocess. +let runnerStub: () => unknown = () => ({ id: "bunx", display: "bunx" }); +let addSucceeds = true; +mock.module("../../lib/skills.ts", () => ({ + resolveSkillsRunner: async () => runnerStub(), + runSkillsAdd: async () => addSucceeds, +})); + +let confirmAnswer = true; +mock.module("../../lib/prompts.ts", () => ({ confirm: async () => confirmAnswer })); + +const { formatSkillsPromptMessage, resolveUpstreamSkills, installSkills } = + await import("./skills.ts"); const DEFAULTS = [ "clerk-cli", @@ -60,3 +76,46 @@ describe("formatSkillsPromptMessage", () => { ); }); }); + +describe("installSkills telemetry", () => { + const recorded = spyOn(telemetryMod, "setTelemetrySkills"); + + beforeEach(() => { + recorded.mockClear(); + runnerStub = () => ({ id: "bunx", display: "bunx" }); + addSucceeds = true; + confirmAnswer = true; + setMode("agent"); + }); + + afterAll(() => recorded.mockRestore()); + + test("records the resolved skill list when the install succeeds", async () => { + await installSkills("/tmp/proj", "next", "bun", true); + expect(recorded).toHaveBeenCalledWith([...resolveUpstreamSkills("next")], "installed"); + }); + + test("records a failed install rather than staying silent", async () => { + addSucceeds = false; + await installSkills("/tmp/proj", undefined, "bun", true); + expect(recorded.mock.calls[0]?.[1]).toBe("failed"); + }); + + // No runner on PATH is a different story from a user saying no, and the + // warehouse could not tell them apart before this. + test("records runner_missing when no package runner is available", async () => { + runnerStub = () => undefined; + await installSkills("/tmp/proj", undefined, "bun", true); + expect(recorded.mock.calls[0]?.[1]).toBe("runner_missing"); + }); + + test("records a decline, and never reaches the runner", async () => { + setMode("human"); + confirmAnswer = false; + runnerStub = () => { + throw new Error("runner must not be probed after a decline"); + }; + await installSkills("/tmp/proj", undefined, "bun", false); + expect(recorded.mock.calls[0]?.[1]).toBe("declined"); + }); +}); diff --git a/packages/cli-core/src/commands/init/skills.ts b/packages/cli-core/src/commands/init/skills.ts index 78723fb7c..2a11e88fa 100644 --- a/packages/cli-core/src/commands/init/skills.ts +++ b/packages/cli-core/src/commands/init/skills.ts @@ -16,6 +16,7 @@ import { isHuman } from "../../mode.js"; import { log } from "../../lib/log.js"; import { confirm } from "../../lib/prompts.js"; import { resolveSkillsRunner, runSkillsAdd } from "../../lib/skills.js"; +import { setTelemetrySkills } from "../../lib/telemetry.js"; import type { ProjectContext } from "./frameworks/types.js"; /** Upstream skills from clerk/skills — installed on every project. */ @@ -94,14 +95,20 @@ export async function installSkills( message: formatSkillsPromptMessage(frameworkSkills), default: true, }); - if (!install) return; + if (!install) { + setTelemetrySkills(upstreamSkills, "declined"); + return; + } } const interactive = isHuman() && !skipPrompt; // Detect runner after the user accepts — no point probing PATH if they decline. const runner = await resolveSkillsRunner(packageManager, interactive); - if (!runner) return; + if (!runner) { + setTelemetrySkills(upstreamSkills, "runner_missing"); + return; + } log.debug(`skills: upstream install — ${upstreamSkills.join(", ")}`); const upstreamOk = await runSkillsAdd( @@ -113,6 +120,8 @@ export async function installSkills( formatSkillsSummary(frameworkSkills), ); + setTelemetrySkills(upstreamSkills, upstreamOk ? "installed" : "failed"); + if (upstreamOk) { log.blank(); log.success("Agent skills installed. AI agents now have Clerk context in this project."); diff --git a/packages/cli-core/src/lib/telemetry.test.ts b/packages/cli-core/src/lib/telemetry.test.ts index 283b9eaa7..035041f4d 100644 --- a/packages/cli-core/src/lib/telemetry.test.ts +++ b/packages/cli-core/src/lib/telemetry.test.ts @@ -6,6 +6,7 @@ import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from ". import { finalizeAndSendTelemetry, getTelemetryStatus, + setTelemetrySkills, setTelemetryStage, startCommandTelemetry, telemetryEnabled, @@ -460,31 +461,31 @@ describe("finalizeAndSendTelemetry", () => { }); }); - describe("stage", () => { - /** Captures the payload of the single event a finalize call sends. */ - async function sendAndCapturePayload( - run: () => void | Promise, - result: TelemetryResult, - ): Promise> { - await markTelemetryNoticeShown(); // past the grace run — reach the send path - process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; - let sent: string | undefined; - globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { - sent = init.body; - return new Response("{}"); - }) as unknown as typeof fetch; + /** Captures the payload of the single event a finalize call sends. */ + async function sendAndCapturePayload( + run: () => void | Promise, + result: TelemetryResult, + ): Promise> { + await markTelemetryNoticeShown(); // past the grace run — reach the send path + process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event"; + let sent: string | undefined; + globalThis.fetch = (async (_url: unknown, init: { body?: string }) => { + sent = init.body; + return new Response("{}"); + }) as unknown as typeof fetch; - startCommandTelemetry(fakeCommand()); - await run(); - await finalizeAndSendTelemetry(result); - - expect(sent).toBeDefined(); - const parsed = JSON.parse(sent as string) as { - events: { payload: Record }[]; - }; - return parsed.events[0]!.payload; - } + startCommandTelemetry(fakeCommand()); + await run(); + await finalizeAndSendTelemetry(result); + + expect(sent).toBeDefined(); + const parsed = JSON.parse(sent as string) as { + events: { payload: Record }[]; + }; + return parsed.events[0]!.payload; + } + describe("stage", () => { test("reports the furthest stage reached on success", async () => { const payload = await sendAndCapturePayload( () => { @@ -529,4 +530,41 @@ describe("finalizeAndSendTelemetry", () => { expect(() => setTelemetryStage("flags")).not.toThrow(); }); }); + + describe("skills", () => { + test("reports the skills an install actually landed", async () => { + const payload = await sendAndCapturePayload( + () => setTelemetrySkills(["clerk-cli", "clerk-orgs"], "installed"), + { outcome: "success", exitCode: 0 }, + ); + expect(payload.skills).toBe("clerk-cli,clerk-orgs"); + expect(payload.skills_outcome).toBe("installed"); + }); + + // `skills` must mean "this project has them", not "it was offered them", + // or a `skills LIKE '%clerk-orgs%'` filter silently counts declines. + test.each([["declined"], ["runner_missing"], ["failed"]] as const)( + "leaves skills empty when the install ended as %s", + async (outcome) => { + const payload = await sendAndCapturePayload( + () => setTelemetrySkills(["clerk-cli", "clerk-orgs"], outcome), + { outcome: "success", exitCode: 0 }, + ); + expect(payload.skills).toBe(""); + expect(payload.skills_outcome).toBe(outcome); + }, + ); + + // A command that never reaches the skills step is distinguishable from one + // whose user declined — that difference is the whole point of the field. + test("outcome is null when the command never reaches the skills step", async () => { + const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 }); + expect(payload.skills).toBe(""); + expect(payload.skills_outcome).toBeNull(); + }); + + test("recording skills with no active context is a no-op", () => { + expect(() => setTelemetrySkills(["clerk-orgs"], "installed")).not.toThrow(); + }); + }); }); diff --git a/packages/cli-core/src/lib/telemetry.ts b/packages/cli-core/src/lib/telemetry.ts index 5880a435d..241de2f38 100644 --- a/packages/cli-core/src/lib/telemetry.ts +++ b/packages/cli-core/src/lib/telemetry.ts @@ -73,6 +73,13 @@ export type TelemetryStage = // shared terminal marker | "done"; +/** + * How the optional agent-skills install at the end of `clerk init` ended. + * A closed union for the same reason as [TelemetryStage]: a renamed call site + * fails to compile rather than quietly splitting the funnel. + */ +export type TelemetrySkillsOutcome = "installed" | "declined" | "runner_missing" | "failed"; + /** Structural slice of Commander's Command — avoids its generic types. */ export type TelemetryCommand = { name(): string; @@ -87,6 +94,9 @@ type TelemetryContext = { startedAt: number; /** Last stage set — see setTelemetryStage. */ stage: TelemetryStage | null; + /** Skills actually installed, comma-joined — see setTelemetrySkills. */ + skills: string; + skillsOutcome: TelemetrySkillsOutcome | null; }; let context: TelemetryContext | null = null; @@ -186,6 +196,8 @@ export function startCommandTelemetry(actionCommand: TelemetryCommand): void { flags: collectSetFlagNames(actionCommand).join(","), startedAt: Date.now(), stage: null, + skills: "", + skillsOutcome: null, }; } catch (error) { log.debug(`telemetry: failed to start context: ${error}`); @@ -208,6 +220,26 @@ export function currentTelemetryStage(): TelemetryStage | null { return context?.stage ?? null; } +/** + * Record the agent skills `clerk init` installed, and how the attempt ended. + * + * Only a successful install populates `skills`, so a warehouse filter like + * `skills LIKE '%clerk-orgs%'` means "this project has that skill" rather than + * "it was offered one". The names come from the CLI's own constants, never + * from user input, so nothing unbounded reaches the payload. + * + * `skillsOutcome` is what separates "the user said no" from "init never got + * that far" — before this, both looked identical from the warehouse. + */ +export function setTelemetrySkills( + names: readonly string[], + outcome: TelemetrySkillsOutcome, +): void { + if (!context) return; + context.skills = outcome === "installed" ? names.join(",") : ""; + context.skillsOutcome = outcome; +} + export function telemetryResultForError(error: unknown): TelemetryResult { if (error instanceof UserAbortError) { return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS }; @@ -297,6 +329,8 @@ async function buildAndSend( exit_code: result.exitCode, error_code: result.errorCode ?? null, stage: current.stage, + skills: current.skills, + skills_outcome: current.skillsOutcome, duration_ms: Date.now() - current.startedAt, machine_uuid: machineUuid, install_method: detectInstallMethod(process.env, process.execPath), @@ -348,9 +382,11 @@ async function maybeShowTelemetryNotice(): Promise { "The Clerk CLI collects usage telemetry to help improve the CLI: command name, flag names,", ); log.info( - "duration, outcome, the step a multi-step command reached, a random machine identifier —", + "duration, outcome, the step a multi-step command reached, the agent skills init installed,", + ); + log.info( + "a random machine identifier — and your workspace and app IDs when a project is linked.", ); - log.info("and your workspace and app IDs when a project is linked."); log.info("Nothing has been sent during this run."); log.info("Opt out: `clerk telemetry disable` — details: https://clerk.com/docs/telemetry"); log.blank();