diff --git a/.github/actions/install-agent-harnesses/action.yml b/.github/actions/install-agent-harnesses/action.yml index 8b7a408f6..c9ff94c19 100644 --- a/.github/actions/install-agent-harnesses/action.yml +++ b/.github/actions/install-agent-harnesses/action.yml @@ -19,7 +19,7 @@ runs: id: engine-sha shell: pwsh env: - ENGINE_SHA: ${{ inputs.engine-sha || 'ecf8e31759d6ddd6d78e3a0b7836b40134368009' }} + ENGINE_SHA: ${{ inputs.engine-sha || 'fdc02d7020632795810057500d62cff2a61513d7' }} run: | if ($env:ENGINE_SHA -notmatch '\A[0-9a-fA-F]{40}\z') { throw "engine-sha must be a full 40-character hexadecimal commit SHA." diff --git a/docs/code-review-details.md b/docs/code-review-details.md new file mode 100644 index 000000000..5d2477485 --- /dev/null +++ b/docs/code-review-details.md @@ -0,0 +1,280 @@ +--- +layout: default +title: Code Review Advanced Metrics - BC-Bench +--- + + + +# Code Review Advanced Metrics + +This view exposes every quality, performance, configuration, and usage metric persisted in the public code-review leaderboard data. The [default leaderboard](code-review.html) keeps only the headline metrics. + +Diagnostic averages use only tasks that reported the metric. Coverage columns show what share of tasks contributed complete token or credit telemetry. + +## Aggregate metrics + +{% if site.data.code-review.aggregate and site.data.code-review.aggregate.size > 0 %} +
| Agent | +Model | +Configuration | +Runs | +Tasks | +Micro Precision | +Micro Recall | +Micro F1 | +Micro F1 95% CI | +Micro F0.5 | +Micro F2 | +Macro Precision | +Macro Recall | +Macro F1 | +Macro F1 95% CI | +Macro F0.5 | +Macro F2 | +Valid Output | +Avg Time | +Avg Prompt Tokens | +Avg Completion Tokens | +Avg Total Tokens | +Avg AI Credits | +Token Coverage | +Credit Coverage | +Usage Complete | +Avg Cached Tokens | +Avg Cache Creation Tokens | +Avg Reasoning Tokens | +Avg API Calls | +Avg Failed API Calls | +Avg Calls With Usage | +Avg Malformed Records | +Avg Articles Retained | +Avg Articles Pruned | +Avg Articles Used in Findings | +Avg Articles Suppressed | +Avg Sub-skills Executed | +Avg Sub-skills Skipped | +Judge | +BC-Bench | +Copilot CLI | +BC-ALAgents | +BCQuality | +
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {{ agg.agent_name }} | +{{ agg.model }} | +{% if agg.experiment %}{{ agg.experiment | jsonify }}{% else %}Baseline{% endif %} |
+ {{ agg.num_runs }} | +{{ agg.total }} | +{{ agg.precision | times: 100.0 | round: 1 }}% | +{{ agg.recall | times: 100.0 | round: 1 }}% | +{{ agg.f1 | times: 100.0 | round: 1 }}% | +{% if agg.f1_ci_low != null %}{{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{{ agg.f_beta_05 | times: 100.0 | round: 1 }}% | +{{ agg.f_beta_2 | times: 100.0 | round: 1 }}% | +{{ agg.macro_precision | times: 100.0 | round: 1 }}% | +{{ agg.macro_recall | times: 100.0 | round: 1 }}% | +{{ agg.macro_f1 | times: 100.0 | round: 1 }}% | +{% if agg.macro_f1_ci_low != null %}{{ agg.macro_f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.macro_f1_ci_high | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{{ agg.macro_f_beta_05 | times: 100.0 | round: 1 }}% | +{{ agg.macro_f_beta_2 | times: 100.0 | round: 1 }}% | +{% if agg.valid_review_output_rate != null %}{{ agg.valid_review_output_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{{ agg.average_duration | round: 1 }}s | +{% if agg.average_prompt_tokens != null %}{{ agg.average_prompt_tokens | round: 0 }}{% else %}—{% endif %} | +{% if agg.average_completion_tokens != null %}{{ agg.average_completion_tokens | round: 0 }}{% else %}—{% endif %} | +{% if agg.average_total_tokens != null %}{{ agg.average_total_tokens | round: 0 }}{% else %}—{% endif %} | +{% if agg.average_ai_credits != null %}{{ agg.average_ai_credits | round: 4 }}{% else %}—{% endif %} | +{% if agg.token_coverage_rate != null %}{{ agg.token_coverage_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{% if agg.credit_coverage_rate != null %}{{ agg.credit_coverage_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{% if agg.usage_complete_rate != null %}{{ agg.usage_complete_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{% if agg.average_cached_tokens != null %}{{ agg.average_cached_tokens | round: 0 }}{% else %}—{% endif %} | +{% if agg.average_cache_creation_tokens != null %}{{ agg.average_cache_creation_tokens | round: 0 }}{% else %}—{% endif %} | +{% if agg.average_reasoning_tokens != null %}{{ agg.average_reasoning_tokens | round: 0 }}{% else %}—{% endif %} | +{% if agg.average_api_calls != null %}{{ agg.average_api_calls | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_failed_api_calls != null %}{{ agg.average_failed_api_calls | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_usage_api_calls != null %}{{ agg.average_usage_api_calls | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_malformed_records != null %}{{ agg.average_malformed_records | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_knowledge_files != null %}{{ agg.average_knowledge_files | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_knowledge_pruned != null %}{{ agg.average_knowledge_pruned | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_knowledge_used != null %}{{ agg.average_knowledge_used | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_knowledge_suppressed != null %}{{ agg.average_knowledge_suppressed | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_sub_skills_executed != null %}{{ agg.average_sub_skills_executed | round: 1 }}{% else %}—{% endif %} | +{% if agg.average_sub_skills_skipped != null %}{{ agg.average_sub_skills_skipped | round: 1 }}{% else %}—{% endif %} | +{{ agg.judge_model }} | +{{ agg.benchmark_version }} | +{% if agg.copilot_cli_version %}{{ agg.copilot_cli_version }}{% else %}—{% endif %} | + {% assign engine_version = agg.agent_version | default: agg.bc_alagents_commit %} +{% if agg.agent_name == "BC PR Review" and engine_version %}{{ engine_version | slice: 0, 8 }}{% else %}—{% endif %} | +{% if agg.bcquality_commit %}{{ agg.bcquality_commit | slice: 0, 8 }}{% if agg.bcquality_version %} ({{ agg.bcquality_version }}){% endif %}{% else %}—{% endif %} | +
No aggregate results available.
+{% endif %} + +## Individual run metrics + +{% if site.data.code-review.runs and site.data.code-review.runs.size > 0 %} +| Run | +Agent | +Model | +Configuration | +Date | +Tasks | +Generated | +Expected | +Matched | +Incorrect | +Missed | +Ignored | +Micro Precision | +Micro Recall | +Micro F1 | +Micro F0.5 | +Micro F2 | +Macro Precision | +Macro Recall | +Macro F1 | +Macro F0.5 | +Macro F2 | +Severity MAE | +Valid Output | +Avg Time | +Avg LLM Time | +Avg Prompt Tokens | +Avg Completion Tokens | +Avg Total Tokens | +Avg AI Credits | +Token Coverage | +Credit Coverage | +Usage Complete | +Avg Cached Tokens | +Avg Cache Creation Tokens | +Avg Reasoning Tokens | +Avg API Calls | +Avg Failed API Calls | +Avg Calls With Usage | +Avg Malformed Records | +Avg Articles Retained | +Avg Articles Pruned | +Avg Articles Used in Findings | +Avg Articles Suppressed | +Avg Sub-skills Executed | +Avg Sub-skills Skipped | +Judge | +BC-Bench | +Copilot CLI | +BC-ALAgents | +BCQuality | +
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {% if run.github_run_id %}{{ run.github_run_id }}{% else %}—{% endif %} | +{{ run.agent_name }} | +{{ run.model }} | +{% if run.experiment %}{{ run.experiment | jsonify }}{% else %}Baseline{% endif %} |
+ {{ run.date }} | +{{ run.total }} | +{{ run.generated_comment_count }} | +{{ run.expected_comment_count }} | +{{ run.matched_comment_count }} | +{{ run.incorrect_comment_count }} | +{{ run.missed_comment_count }} | +{{ run.ignored_matched_comment_count }} | +{{ run.precision | times: 100.0 | round: 1 }}% | +{{ run.recall | times: 100.0 | round: 1 }}% | +{{ run.f1 | times: 100.0 | round: 1 }}% | +{{ run.f_beta_05 | times: 100.0 | round: 1 }}% | +{{ run.f_beta_2 | times: 100.0 | round: 1 }}% | +{{ run.macro_precision | times: 100.0 | round: 1 }}% | +{{ run.macro_recall | times: 100.0 | round: 1 }}% | +{{ run.macro_f1 | times: 100.0 | round: 1 }}% | +{{ run.macro_f_beta_05 | times: 100.0 | round: 1 }}% | +{{ run.macro_f_beta_2 | times: 100.0 | round: 1 }}% | +{{ run.severity_mae | round: 3 }} | +{% if run.valid_review_output_rate != null %}{{ run.valid_review_output_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{{ run.average_duration | round: 1 }}s | +{% if run.average_llm_duration != null %}{{ run.average_llm_duration | round: 1 }}s{% else %}—{% endif %} | +{% if run.average_prompt_tokens != null %}{{ run.average_prompt_tokens | round: 0 }}{% else %}—{% endif %} | +{% if run.average_completion_tokens != null %}{{ run.average_completion_tokens | round: 0 }}{% else %}—{% endif %} | +{% if run.average_total_tokens != null %}{{ run.average_total_tokens | round: 0 }}{% else %}—{% endif %} | +{% if run.average_ai_credits != null %}{{ run.average_ai_credits | round: 4 }}{% else %}—{% endif %} | +{% if run.token_coverage_rate != null %}{{ run.token_coverage_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{% if run.credit_coverage_rate != null %}{{ run.credit_coverage_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{% if run.usage_complete_rate != null %}{{ run.usage_complete_rate | times: 100.0 | round: 1 }}%{% else %}—{% endif %} | +{% if run.average_cached_tokens != null %}{{ run.average_cached_tokens | round: 0 }}{% else %}—{% endif %} | +{% if run.average_cache_creation_tokens != null %}{{ run.average_cache_creation_tokens | round: 0 }}{% else %}—{% endif %} | +{% if run.average_reasoning_tokens != null %}{{ run.average_reasoning_tokens | round: 0 }}{% else %}—{% endif %} | +{% if run.average_api_calls != null %}{{ run.average_api_calls | round: 1 }}{% else %}—{% endif %} | +{% if run.average_failed_api_calls != null %}{{ run.average_failed_api_calls | round: 1 }}{% else %}—{% endif %} | +{% if run.average_usage_api_calls != null %}{{ run.average_usage_api_calls | round: 1 }}{% else %}—{% endif %} | +{% if run.average_malformed_records != null %}{{ run.average_malformed_records | round: 1 }}{% else %}—{% endif %} | +{% if run.average_knowledge_files != null %}{{ run.average_knowledge_files | round: 1 }}{% else %}—{% endif %} | +{% if run.average_knowledge_pruned != null %}{{ run.average_knowledge_pruned | round: 1 }}{% else %}—{% endif %} | +{% if run.average_knowledge_used != null %}{{ run.average_knowledge_used | round: 1 }}{% else %}—{% endif %} | +{% if run.average_knowledge_suppressed != null %}{{ run.average_knowledge_suppressed | round: 1 }}{% else %}—{% endif %} | +{% if run.average_sub_skills_executed != null %}{{ run.average_sub_skills_executed | round: 1 }}{% else %}—{% endif %} | +{% if run.average_sub_skills_skipped != null %}{{ run.average_sub_skills_skipped | round: 1 }}{% else %}—{% endif %} | +{{ run.judge_model }} | +{{ run.benchmark_version }} | +{% if run.copilot_cli_version %}{{ run.copilot_cli_version }}{% else %}—{% endif %} | + {% assign engine_version = run.agent_version | default: run.bc_alagents_commit %} +{% if run.agent_name == "BC PR Review" and engine_version %}{{ engine_version | slice: 0, 8 }}{% else %}—{% endif %} | +{% if run.bcquality_commit %}{{ run.bcquality_commit | slice: 0, 8 }}{% if run.bcquality_version %} ({{ run.bcquality_version }}){% endif %}{% else %}—{% endif %} | +
No individual run results available.
+{% endif %} + +[← Back to Code Review](code-review.html) diff --git a/docs/code-review.md b/docs/code-review.md index 8d7c92057..9eaa6a46d 100644 --- a/docs/code-review.md +++ b/docs/code-review.md @@ -47,11 +47,11 @@ The `pr-review` workflow also accepts an `engine-sha` input — a full 40-charac Either way, hold everything else fixed: the benchmark version, the model, the Copilot CLI version the engine uses internally, and the configured minimum severity. Locally, `bcbench evaluate pr-review --engine-path| Recall | Valid Output | Avg Time | -Ver | +Version | - {% assign sorted_results = site.data.code-review.aggregate | sort: "f1" | reverse %} + {% assign production_results = site.data.code-review.aggregate | where: "agent_name", "BC PR Review" %} + {% assign sorted_results = production_results | sort: "f1" | reverse %} {% for agg in sorted_results %} {% if agg.experiment == null or agg.experiment.is_experiment == false %}|||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| Avg Completion Tokens | Avg Total Tokens | Avg AI Credits | -Ver | +Version | - {% assign performance_results = site.data.code-review.aggregate | sort: "average_duration" %} + {% assign production_results = site.data.code-review.aggregate | where: "agent_name", "BC PR Review" %} + {% assign performance_results = production_results | sort: "average_duration" %} {% for agg in performance_results %}|||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| {{ agg.agent_name }} | @@ -125,12 +127,49 @@ Unavailable AI credits remain `null` in bceval exports; observed zero remains ze
| Agent | +Model | +Micro F1 | +Precision | +Recall | +Avg Time | +BC-Bench | +
|---|---|---|---|---|---|---|
| {{ agg.agent_name }} | +{{ agg.model }} | +{{ agg.f1 | times: 100.0 | round: 1 }}% | +{{ agg.precision | times: 100.0 | round: 1 }}% | +{{ agg.recall | times: 100.0 | round: 1 }}% | +{{ agg.average_duration | round: 1 }}s | +{{ agg.benchmark_version }} | +