feat(telemetry): record which agent skills init installed

.changeset/skills-telemetry-stamp.md

@@ -0,0 +1,5 @@
+---
+"clerk": minor
+---
+
+Record which agent skills `clerk init` installed, and whether the install succeeded, was declined, or found no package runner. The telemetry disclosure notice now names this alongside the fields it already listed.

packages/cli-core/src/commands/init/skills.test.ts

@@ -1,5 +1,21 @@
-import { test, expect, describe } from "bun:test";
-import { formatSkillsPromptMessage, resolveUpstreamSkills } from "./skills.ts";
+import { test, expect, describe, mock, spyOn, beforeEach, afterAll } from "bun:test";
+import { setMode } from "../../mode.ts";
+import * as telemetryMod from "../../lib/telemetry.ts";
+
+// Stub the layer that would shell out to `bunx skills add`, so these tests
+// exercise installSkills' branching without spawning a subprocess.
+let runnerStub: () => unknown = () => ({ id: "bunx", display: "bunx" });
+let addSucceeds = true;
+mock.module("../../lib/skills.ts", () => ({
+  resolveSkillsRunner: async () => runnerStub(),
+  runSkillsAdd: async () => addSucceeds,
+}));
+
+let confirmAnswer = true;
+mock.module("../../lib/prompts.ts", () => ({ confirm: async () => confirmAnswer }));
+
+const { formatSkillsPromptMessage, resolveUpstreamSkills, installSkills } =
+  await import("./skills.ts");
 
 const DEFAULTS = [
   "clerk-cli",
@@ -60,3 +76,46 @@ describe("formatSkillsPromptMessage", () => {
     );
   });
 });
+
+describe("installSkills telemetry", () => {
+  const recorded = spyOn(telemetryMod, "setTelemetrySkills");
+
+  beforeEach(() => {
+    recorded.mockClear();
+    runnerStub = () => ({ id: "bunx", display: "bunx" });
+    addSucceeds = true;
+    confirmAnswer = true;
+    setMode("agent");
+  });
+
+  afterAll(() => recorded.mockRestore());
+
+  test("records the resolved skill list when the install succeeds", async () => {
+    await installSkills("/tmp/proj", "next", "bun", true);
+    expect(recorded).toHaveBeenCalledWith([...resolveUpstreamSkills("next")], "installed");
+  });
+
+  test("records a failed install rather than staying silent", async () => {
+    addSucceeds = false;
+    await installSkills("/tmp/proj", undefined, "bun", true);
+    expect(recorded.mock.calls[0]?.[1]).toBe("failed");
+  });
+
+  // No runner on PATH is a different story from a user saying no, and the
+  // warehouse could not tell them apart before this.
+  test("records runner_missing when no package runner is available", async () => {
+    runnerStub = () => undefined;
+    await installSkills("/tmp/proj", undefined, "bun", true);
+    expect(recorded.mock.calls[0]?.[1]).toBe("runner_missing");
+  });
+
+  test("records a decline, and never reaches the runner", async () => {
+    setMode("human");
+    confirmAnswer = false;
+    runnerStub = () => {
+      throw new Error("runner must not be probed after a decline");
+    };
+    await installSkills("/tmp/proj", undefined, "bun", false);
+    expect(recorded.mock.calls[0]?.[1]).toBe("declined");
+  });
+});

packages/cli-core/src/commands/init/skills.ts

@@ -16,6 +16,7 @@ import { isHuman } from "../../mode.js";
 import { log } from "../../lib/log.js";
 import { confirm } from "../../lib/prompts.js";
 import { resolveSkillsRunner, runSkillsAdd } from "../../lib/skills.js";
+import { setTelemetrySkills } from "../../lib/telemetry.js";
 import type { ProjectContext } from "./frameworks/types.js";
 
 /** Upstream skills from clerk/skills — installed on every project. */
@@ -94,14 +95,20 @@ export async function installSkills(
       message: formatSkillsPromptMessage(frameworkSkills),
       default: true,
     });
-    if (!install) return;
+    if (!install) {
+      setTelemetrySkills(upstreamSkills, "declined");
+      return;
+    }
   }
 
   const interactive = isHuman() && !skipPrompt;
 
   // Detect runner after the user accepts — no point probing PATH if they decline.
   const runner = await resolveSkillsRunner(packageManager, interactive);
-  if (!runner) return;
+  if (!runner) {
+    setTelemetrySkills(upstreamSkills, "runner_missing");
+    return;
+  }
 
   log.debug(`skills: upstream install — ${upstreamSkills.join(", ")}`);
   const upstreamOk = await runSkillsAdd(
@@ -113,6 +120,8 @@ export async function installSkills(
     formatSkillsSummary(frameworkSkills),
   );
 
+  setTelemetrySkills(upstreamSkills, upstreamOk ? "installed" : "failed");
+
   if (upstreamOk) {
     log.blank();
     log.success("Agent skills installed. AI agents now have Clerk context in this project.");

packages/cli-core/src/lib/telemetry.test.ts

@@ -6,6 +6,7 @@ import { _setConfigDir, markTelemetryNoticeShown, setTelemetryDisabled } from ".
 import {
   finalizeAndSendTelemetry,
   getTelemetryStatus,
+  setTelemetrySkills,
   setTelemetryStage,
   startCommandTelemetry,
   telemetryEnabled,
@@ -460,31 +461,31 @@ describe("finalizeAndSendTelemetry", () => {
     });
   });
 
-  describe("stage", () => {
-    /** Captures the payload of the single event a finalize call sends. */
-    async function sendAndCapturePayload(
-      run: () => void | Promise<void>,
-      result: TelemetryResult,
-    ): Promise<Record<string, unknown>> {
-      await markTelemetryNoticeShown(); // past the grace run — reach the send path
-      process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event";
-      let sent: string | undefined;
-      globalThis.fetch = (async (_url: unknown, init: { body?: string }) => {
-        sent = init.body;
-        return new Response("{}");
-      }) as unknown as typeof fetch;
+  /** Captures the payload of the single event a finalize call sends. */
+  async function sendAndCapturePayload(
+    run: () => void | Promise<void>,
+    result: TelemetryResult,
+  ): Promise<Record<string, unknown>> {
+    await markTelemetryNoticeShown(); // past the grace run — reach the send path
+    process.env.CLERK_TELEMETRY_URL = "https://capture.invalid/v1/event";
+    let sent: string | undefined;
+    globalThis.fetch = (async (_url: unknown, init: { body?: string }) => {
+      sent = init.body;
+      return new Response("{}");
+    }) as unknown as typeof fetch;
 
-      startCommandTelemetry(fakeCommand());
-      await run();
-      await finalizeAndSendTelemetry(result);
-
-      expect(sent).toBeDefined();
-      const parsed = JSON.parse(sent as string) as {
-        events: { payload: Record<string, unknown> }[];
-      };
-      return parsed.events[0]!.payload;
-    }
+    startCommandTelemetry(fakeCommand());
+    await run();
+    await finalizeAndSendTelemetry(result);
+
+    expect(sent).toBeDefined();
+    const parsed = JSON.parse(sent as string) as {
+      events: { payload: Record<string, unknown> }[];
+    };
+    return parsed.events[0]!.payload;
+  }
 
+  describe("stage", () => {
     test("reports the furthest stage reached on success", async () => {
       const payload = await sendAndCapturePayload(
         () => {
@@ -529,4 +530,41 @@ describe("finalizeAndSendTelemetry", () => {
       expect(() => setTelemetryStage("flags")).not.toThrow();
     });
   });
+
+  describe("skills", () => {
+    test("reports the skills an install actually landed", async () => {
+      const payload = await sendAndCapturePayload(
+        () => setTelemetrySkills(["clerk-cli", "clerk-orgs"], "installed"),
+        { outcome: "success", exitCode: 0 },
+      );
+      expect(payload.skills).toBe("clerk-cli,clerk-orgs");
+      expect(payload.skills_outcome).toBe("installed");
+    });
+
+    // `skills` must mean "this project has them", not "it was offered them",
+    // or a `skills LIKE '%clerk-orgs%'` filter silently counts declines.
+    test.each([["declined"], ["runner_missing"], ["failed"]] as const)(
+      "leaves skills empty when the install ended as %s",
+      async (outcome) => {
+        const payload = await sendAndCapturePayload(
+          () => setTelemetrySkills(["clerk-cli", "clerk-orgs"], outcome),
+          { outcome: "success", exitCode: 0 },
+        );
+        expect(payload.skills).toBe("");
+        expect(payload.skills_outcome).toBe(outcome);
+      },
+    );
+
+    // A command that never reaches the skills step is distinguishable from one
+    // whose user declined — that difference is the whole point of the field.
+    test("outcome is null when the command never reaches the skills step", async () => {
+      const payload = await sendAndCapturePayload(() => {}, { outcome: "success", exitCode: 0 });
+      expect(payload.skills).toBe("");
+      expect(payload.skills_outcome).toBeNull();
+    });
+
+    test("recording skills with no active context is a no-op", () => {
+      expect(() => setTelemetrySkills(["clerk-orgs"], "installed")).not.toThrow();
+    });
+  });
 });

packages/cli-core/src/lib/telemetry.ts

@@ -73,6 +73,13 @@ export type TelemetryStage =
   // shared terminal marker
   | "done";
 
+/**
+ * How the optional agent-skills install at the end of `clerk init` ended.
+ * A closed union for the same reason as [TelemetryStage]: a renamed call site
+ * fails to compile rather than quietly splitting the funnel.
+ */
+export type TelemetrySkillsOutcome = "installed" | "declined" | "runner_missing" | "failed";
+
 /** Structural slice of Commander's Command — avoids its generic types. */
 export type TelemetryCommand = {
   name(): string;
@@ -87,6 +94,9 @@ type TelemetryContext = {
   startedAt: number;
   /** Last stage set — see setTelemetryStage. */
   stage: TelemetryStage | null;
+  /** Skills actually installed, comma-joined — see setTelemetrySkills. */
+  skills: string;
+  skillsOutcome: TelemetrySkillsOutcome | null;
 };
 
 let context: TelemetryContext | null = null;
@@ -186,6 +196,8 @@ export function startCommandTelemetry(actionCommand: TelemetryCommand): void {
       flags: collectSetFlagNames(actionCommand).join(","),
       startedAt: Date.now(),
       stage: null,
+      skills: "",
+      skillsOutcome: null,
     };
   } catch (error) {
     log.debug(`telemetry: failed to start context: ${error}`);
@@ -208,6 +220,26 @@ export function currentTelemetryStage(): TelemetryStage | null {
   return context?.stage ?? null;
 }
 
+/**
+ * Record the agent skills `clerk init` installed, and how the attempt ended.
+ *
+ * Only a successful install populates `skills`, so a warehouse filter like
+ * `skills LIKE '%clerk-orgs%'` means "this project has that skill" rather than
+ * "it was offered one". The names come from the CLI's own constants, never
+ * from user input, so nothing unbounded reaches the payload.
+ *
+ * `skillsOutcome` is what separates "the user said no" from "init never got
+ * that far" — before this, both looked identical from the warehouse.
+ */
+export function setTelemetrySkills(
+  names: readonly string[],
+  outcome: TelemetrySkillsOutcome,
+): void {
+  if (!context) return;
+  context.skills = outcome === "installed" ? names.join(",") : "";
+  context.skillsOutcome = outcome;
+}
+
 export function telemetryResultForError(error: unknown): TelemetryResult {
   if (error instanceof UserAbortError) {
     return { outcome: "abort", exitCode: EXIT_CODE.SUCCESS };
@@ -297,6 +329,8 @@ async function buildAndSend(
       exit_code: result.exitCode,
       error_code: result.errorCode ?? null,
       stage: current.stage,
+      skills: current.skills,
+      skills_outcome: current.skillsOutcome,
       duration_ms: Date.now() - current.startedAt,
       machine_uuid: machineUuid,
       install_method: detectInstallMethod(process.env, process.execPath),
@@ -348,9 +382,11 @@ async function maybeShowTelemetryNotice(): Promise<boolean> {
     "The Clerk CLI collects usage telemetry to help improve the CLI: command name, flag names,",
   );
   log.info(
-    "duration, outcome, the step a multi-step command reached, a random machine identifier —",
+    "duration, outcome, the step a multi-step command reached, the agent skills init installed,",
+  );
+  log.info(
+    "a random machine identifier — and your workspace and app IDs when a project is linked.",
   );
-  log.info("and your workspace and app IDs when a project is linked.");
   log.info("Nothing has been sent during this run.");
   log.info("Opt out: `clerk telemetry disable` — details: https://clerk.com/docs/telemetry");
   log.blank();